{"cells":[{"metadata":{},"cell_type":"markdown","source":"- Validate submission with TestIterator: https://www.kaggle.com/authman/lgbm-model-1?scriptVersionId=49016504\n- History: Keep last 100 interactions per user in memory\n- Build test sequence on-fly from current test_df and history for each user\n- Pad on the fly... might have to cache # items to predict per user or simply look at the test_df..."},{"metadata":{"papermill":{"duration":0.016225,"end_time":"2020-11-12T09:45:16.415602","exception":false,"start_time":"2020-11-12T09:45:16.399377","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Setup"},{"metadata":{"trusted":false},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-12T09:45:16.452401Z","iopub.status.busy":"2020-11-12T09:45:16.451753Z","iopub.status.idle":"2020-11-12T09:45:17.939066Z","shell.execute_reply":"2020-11-12T09:45:17.938339Z"},"papermill":{"duration":1.510191,"end_time":"2020-11-12T09:45:17.939185","exception":false,"start_time":"2020-11-12T09:45:16.428994","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc, joblib, random\n\nfrom tqdm.auto import tqdm\nfrom pathlib import Path\nfrom typing import List\nfrom collections import defaultdict\nfrom bitarray import bitarray\nfrom time import time\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport scipy\n\nTRAIN_DTYPES = {\n # 'row_id': np.uint32,\n 'timestamp': np.uint64,\n 'user_id': np.uint32,\n 'content_id': np.uint16,\n 'content_type_id': np.uint8,\n 'task_container_id': np.uint16,\n 'user_answer': np.int8,\n 'answered_correctly': np.int8,\n 'prior_question_elapsed_time': np.float32,\n 'prior_question_had_explanation': 'boolean'\n}\n\nDATA_DIR = Path('../input/riiid-test-answer-prediction')\nTRAIN_PATH = DATA_DIR / 'train.csv'\nQUESTIONS_PATH = DATA_DIR / 'questions.csv'\nLECTURES_PATH = DATA_DIR / 'lectures.csv'\n\nDEVICE = 'cuda'\nBATCH_SIZE = 512\nWINDOW_SIZE = 128\nPAD = 0\nGREAT_SEED = 1337\n\ndef seed_everything(seed_value):\n random.seed(seed_value)\n np.random.seed(seed_value)\n torch.manual_seed(seed_value)\n\n if torch.cuda.is_available(): \n torch.cuda.manual_seed(seed_value)\n torch.cuda.manual_seed_all(seed_value)\n torch.backends.cudnn.deterministic = False\n torch.backends.cudnn.benchmark = False\n \nseed_everything(GREAT_SEED)\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# NOTE: Not sure if we're going to make use of this..\n\n# Start col id labeling from this row below\nCOL_OFFSET_CUT = 1\n\ndataset_cols = [\n # Leave user_id first, so we can truncate it\n 'timestamp',\n 'user_id',\n 'task_container_id',\n \n #############\n # We start indexing from here, since the rest get cut off:\n 'content_id',\n 'part_id',\n 'prior_question_elapsed_time',\n 'prior_question_had_explanation',\n 'incorrect_rank', # decoder feature that needs help..........\n 'content_type_id',\n 'bundle_id',\n \n # only these of the 4 have signal (0-index)\n 'answer_ratio1',\n 'answer_ratio2',\n 'correct_streak_u',\n 'incorrect_streak_u',\n 'correct_streak_alltime_u',\n 'incorrect_streak_alltime_u',\n \n # our session information + a lifetime field (elapsed time sum on platform—probably should be capped)\n 'session_content_num_u',\n 'session_duration_u',\n 'session_ans_duration_u',\n 'lifetime_ans_duration_u',\n \n # First block of continuous - timestamp intervals\n 'lag_ts_u_recency',\n 'last_ts_u_recency1',\n 'last_ts_u_recency2',\n 'last_ts_u_recency3',\n 'last_correct_ts_u_recency',\n 'last_incorrect_ts_u_recency',\n \n # Second block of continuous - how often user is right on global/part/session basis\n 'correctness_u_recency',\n 'part_correctness_u_recency',\n 'session_correctness_u',\n \n # Have we been served this content before?\n 'encountered',\n \n # Average score of most frequent asked questions\n 'diagnostic_u_recency_1',\n 'diagnostic_u_recency_2',\n 'diagnostic_u_recency_3',\n 'diagnostic_u_recency_4',\n 'diagnostic_u_recency_5',\n 'diagnostic_u_recency_6',\n\n 'answered_correctly',\n]\n\nCOL_USER_ID = dataset_cols.index('user_id') - COL_OFFSET_CUT\nCOL_TASK_CONTAINER_ID = dataset_cols.index('task_container_id') - COL_OFFSET_CUT\nCOL_CONTENT_ID = dataset_cols.index('content_id') - COL_OFFSET_CUT\nCOL_PART_ID = dataset_cols.index('part_id') - COL_OFFSET_CUT\nCOL_PRIOR_QUESTION_ELAPSED_TIME = dataset_cols.index('prior_question_elapsed_time') - COL_OFFSET_CUT\nCOL_PRIOR_QUESTION_EXPLANATION = dataset_cols.index('prior_question_had_explanation') - COL_OFFSET_CUT\nCOL_INCORRECT_RANK = dataset_cols.index('incorrect_rank') - COL_OFFSET_CUT\nCOL_CONTENT_TYPE_ID = dataset_cols.index('content_type_id') - COL_OFFSET_CUT\nCOL_BUNDLE_ID = dataset_cols.index('bundle_id') - COL_OFFSET_CUT\n\nCOL_ANSWER_RATIO1 = dataset_cols.index('answer_ratio1') - COL_OFFSET_CUT\nCOL_ANSWER_RATIO2 = dataset_cols.index('answer_ratio2') - COL_OFFSET_CUT\nCOL_CORRECT_STREAK_U = dataset_cols.index('correct_streak_u') - COL_OFFSET_CUT\nCOL_INCORRECT_STREAK_U = dataset_cols.index('incorrect_streak_u') - COL_OFFSET_CUT\nCOL_CORRECT_STREAK_ALLTIME_U = dataset_cols.index('correct_streak_alltime_u') - COL_OFFSET_CUT\nCOL_INCORRECT_STREAK_ALLTIME_U = dataset_cols.index('incorrect_streak_alltime_u') - COL_OFFSET_CUT\n \nCOL_SESSION_CONTENT_NUM_U = dataset_cols.index('session_content_num_u') - COL_OFFSET_CUT\nCOL_SESSION_DURATION_U = dataset_cols.index('session_duration_u') - COL_OFFSET_CUT\nCOL_SESSION_CORRECTNESS_U = dataset_cols.index('session_correctness_u') - COL_OFFSET_CUT\nCOL_SESSION_ANS_DURATION_U = dataset_cols.index('session_ans_duration_u') - COL_OFFSET_CUT\nCOL_LIFETIME_ANS_DURATION_U = dataset_cols.index('lifetime_ans_duration_u') - COL_OFFSET_CUT\n\nCOL_LAG_TS_RECENCY = dataset_cols.index('lag_ts_u_recency') - COL_OFFSET_CUT\nCOL_LAST_TS_RECENCY1 = dataset_cols.index('last_ts_u_recency1') - COL_OFFSET_CUT\nCOL_LAST_TS_RECENCY2 = dataset_cols.index('last_ts_u_recency2') - COL_OFFSET_CUT\nCOL_LAST_TS_RECENCY3 = dataset_cols.index('last_ts_u_recency3') - COL_OFFSET_CUT\nCOL_CORRECTNESS_U_RECENCY = dataset_cols.index('correctness_u_recency') - COL_OFFSET_CUT\nCOL_PART_CORRECTNESS_U_RECENCY = dataset_cols.index('part_correctness_u_recency') - COL_OFFSET_CUT\nCOL_LAST_CORRECT_TS_U_RECENCY = dataset_cols.index('last_correct_ts_u_recency') - COL_OFFSET_CUT\nCOL_LAST_INCORRECT_TS_U_RECENCY = dataset_cols.index('last_incorrect_ts_u_recency') - COL_OFFSET_CUT\n\nCOL_ENCOUNTERED = dataset_cols.index('encountered') - COL_OFFSET_CUT\n\nCOL_DIAGNOSTIC_RECENCY_1 = dataset_cols.index('diagnostic_u_recency_1') - COL_OFFSET_CUT\nCOL_DIAGNOSTIC_RECENCY_2 = dataset_cols.index('diagnostic_u_recency_2') - COL_OFFSET_CUT\nCOL_DIAGNOSTIC_RECENCY_3 = dataset_cols.index('diagnostic_u_recency_3') - COL_OFFSET_CUT\nCOL_DIAGNOSTIC_RECENCY_4 = dataset_cols.index('diagnostic_u_recency_4') - COL_OFFSET_CUT\nCOL_DIAGNOSTIC_RECENCY_5 = dataset_cols.index('diagnostic_u_recency_5') - COL_OFFSET_CUT\nCOL_DIAGNOSTIC_RECENCY_6 = dataset_cols.index('diagnostic_u_recency_6') - COL_OFFSET_CUT\n\nCOL_ANSWERED_CORRECTLY = dataset_cols.index('answered_correctly') - COL_OFFSET_CUT\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":false},"cell_type":"code","source":"def count_parameters(model):\n table = PrettyTable([\"Modules\", \"Parameters\"])\n total_params = 0\n for name, parameter in model.named_parameters():\n if not parameter.requires_grad: continue\n param = parameter.numel()\n table.add_row([name, param])\n total_params+=param\n print(table)\n print(f\"Total Trainable Params: {total_params}\")\n return total_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def generate_mask(size, diagonal=1): \n return torch.triu(torch.ones(size, size)==1, diagonal=diagonal)\n\ndef tasks_mask(tasks, seq_length, diagonal=1):\n future_mask = generate_mask(seq_length, diagonal=diagonal).to(tasks.device)\n container_mask= torch.ones((seq_length, seq_length)).to(tasks.device)\n container_mask=(container_mask*tasks.reshape(1,-1))==(container_mask*tasks.reshape(-1,1))\n future_mask=future_mask+container_mask\n future_mask = future_mask.fill_diagonal_(False)\n return future_mask\n\ndef tasks_3d_mask(tasks, seq_length=WINDOW_SIZE, nhead=1, diagonal=1):\n #https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/206620\n mask_3d = [tasks_mask(t, seq_length, diagonal=diagonal) for t in tasks]\n mask_3d = torch.stack(mask_3d, dim=0)\n # Need BS*num_heads shape\n repeat_3d = [mask_3d for t in range(nhead)]\n repeat_3d = torch.cat(repeat_3d)\n return repeat_3d","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class BartSinusoidalPositionalEmbedding(nn.Embedding):\n \"\"\"This module produces sinusoidal positional embeddings of any length.\"\"\"\n\n def __init__(self, num_positions: int, embedding_dim: int, padding_idx=None):\n super().__init__(num_positions+1, embedding_dim)\n self.weight = self._init_weight(self.weight)\n self.positions = None\n \n @staticmethod\n def _init_weight(out: nn.Parameter):\n \"\"\"\n Identical to the XLM create_sinusoidal_embeddings except features are not interleaved. The cos features are in\n the 2nd half of the vector. [dim // 2:]\n \"\"\"\n n_pos, dim = out.shape\n position_enc = np.array(\n [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]\n )\n out.requires_grad = False # set early to avoid an error in pytorch-1.8+\n sentinel = dim // 2 if dim % 2 == 0 else (dim // 2) + 1\n out[:, :sentinel] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))\n out[:, sentinel:] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))\n \n # Scale this bad boy\n out *= 0.1 # 10% of whatever it is\n \n out.detach_()\n return out\n\n @torch.no_grad()\n def forward(self, start:int=0):\n if self.positions is None:\n self.positions = torch.arange(\n 0, WINDOW_SIZE+1,\n dtype=torch.long,\n device=self.weight.device\n )\n \n return super().forward(self.positions)[start:start+WINDOW_SIZE]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class LambdaLayer(nn.Module):\n def __init__(self, lambd):\n super(LambdaLayer, self).__init__()\n self.lambd = lambd\n def forward(self, x):\n return self.lambd(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class ThinTransformerEncoderLayer(nn.Module):\n def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation=\"relu\"):\n super(ThinTransformerEncoderLayer, self).__init__()\n # Implementation of Feedforward model\n self.linear1 = nn.Linear(d_model, dim_feedforward)\n self.dropout = nn.Dropout(dropout)\n self.linear2 = nn.Linear(dim_feedforward, d_model)\n\n self.norm1 = nn.LayerNorm(d_model)\n self.norm2 = nn.LayerNorm(d_model)\n self.dropout2 = nn.Dropout(dropout)\n\n\n def forward(self, src, src_mask = None, src_key_padding_mask = None):\n r\"\"\"Pass the input through the encoder layer.\n\n Args:\n src: the sequence to the encoder layer (required).\n src_mask: the mask for the src sequence (optional).\n src_key_padding_mask: the mask for the src keys per batch (optional).\n\n Shape:\n see the docs in Transformer class.\n \"\"\"\n src = self.norm1(src)\n src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))\n src = src + self.dropout2(src2)\n src = self.norm2(src)\n \n # Mask Out:\n src[src_key_padding_mask.transpose(0,1)[:,:,None].expand(src.shape)] = 0\n return src","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class SaintTransformerModel(nn.Module):\n def __init__(\n self,\n d_model:int=32,\n nhead:int=2,\n num_users:int=393657+1,\n num_layers:int=2,\n num_exercises:int=13941+2, # NOTE: 1+total number of QUESTIONS+LECTURES + Mask at the end\n num_answer_streaks:int=8+1,\n num_content_types:int=2,\n num_parts:int=7+1, # TOTAL\n num_bundles:int=9765+2,\n num_prior_question_elapsed_time:int=301+2,\n num_explanable:int=2+2,\n num_max_seq_len:int=WINDOW_SIZE, \n num_encountered:int=2+1,\n dropout:float=0.1,\n emb_dropout:float=0.2,\n initrange:float = 0.02\n ):\n super(SaintTransformerModel, self).__init__()\n\n self.d_model = d_model\n self.nhead = nhead\n self.emb_dropout = nn.Dropout(p=emb_dropout)\n \n# # These two have double dropout, due to their cardinality:\n# self.user_embeddings = nn.Sequential(\n# nn.Embedding(num_users, 16, padding_idx=0),\n# #nn.Linear(16, d_model, bias=False),\n# #nn.Dropout(p=emb_dropout)\n# )\n\n self.exercise_embeddings = nn.Embedding(num_exercises, d_model, padding_idx=0)\n self.bundle_embeddings = nn.Embedding(num_bundles, d_model, padding_idx=0)\n self.content_type_embeddings = nn.Embedding(num_content_types, d_model, padding_idx=0)\n self.sin_pos_embedder = BartSinusoidalPositionalEmbedding(num_positions=WINDOW_SIZE, embedding_dim=d_model, padding_idx=0) \n self.part_embeddings = nn.Embedding(num_parts, d_model, padding_idx=0)\n self.prior_question_elapsed_time_embeddings = nn.Embedding(num_prior_question_elapsed_time, d_model, padding_idx=0) \n self.prior_question_had_explanation_embeddings = nn.Embedding(num_explanable, d_model, padding_idx=0)\n self.correctness_embeddings = nn.Embedding(4+2, d_model, padding_idx=0)\n self.encountered_embeddings = nn.Embedding(num_encountered, d_model, padding_idx=0)\n\n self.cont_mlp = nn.Sequential(\n # LN ? BN\n nn.Linear(25, d_model//2),\n nn.Dropout(p=emb_dropout),\n nn.ReLU(inplace=True),\n nn.Linear(d_model//2, d_model, bias=False),\n nn.ReLU(inplace=True),\n )\n \n self.transformer_encoder = nn.TransformerEncoder(\n nn.TransformerEncoderLayer(\n d_model, nhead, dim_feedforward=d_model,#2048,\n dropout=dropout, activation='relu'\n ),\n num_layers\n )\n \n self.transformer_history = nn.TransformerEncoder(\n nn.TransformerEncoderLayer(\n d_model, nhead*2, dim_feedforward=d_model,#2048,\n dropout=dropout, activation='relu'\n ),\n 1 # history only has 1 layer\n )\n \n self.transformer_decoder = nn.TransformerDecoder(\n nn.TransformerDecoderLayer(\n d_model, nhead, dim_feedforward=d_model,#2048,\n dropout=dropout, activation='relu'\n ),\n num_layers\n )\n \n self.decoder = nn.Linear(d_model,1)\n self.init_weights(initrange)\n\n def init_weights(self, initrange:float = 0.02) -> None:\n self.content_type_embeddings.weight.data.normal_(0, initrange)\n self.encountered_embeddings.weight.data.normal_(0, initrange)\n self.exercise_embeddings.weight.data.normal_(0, initrange)\n self.bundle_embeddings.weight.data.normal_(0, initrange)\n self.part_embeddings.weight.data.normal_(0, initrange)\n self.prior_question_elapsed_time_embeddings.weight.data.normal_(0, initrange)\n self.prior_question_had_explanation_embeddings.weight.data.normal_(0, initrange)\n self.correctness_embeddings.weight.data.normal_(0, initrange)\n #self.decoder.bias.data.zero_()\n self.decoder.weight.data.normal_(0, initrange)\n \n @torch.cuda.amp.autocast()\n def forward(\n self,\n user_id, content_id, content_type_id, part_id, encountered_id,\n prior_question_elapsed_time, prior_question_had_explanation,\n correctness_id, bundle_id, cont_streaks,\n continuous, session, diagnostic, padding_mask, task_container_id\n ):\n batch_size = content_id.shape[0]\n seq_len = content_id.shape[-1]\n \n future_mask = torch.ones([seq_len, seq_len], device=task_container_id.device, dtype=torch.uint8)\n future_mask = future_mask.triu_(1).view(seq_len, seq_len).bool()\n history_mask = tasks_3d_mask(task_container_id, seq_length=WINDOW_SIZE, nhead=self.nhead*2)\n \n ##########\n embedded_src = self.emb_dropout(\n self.content_type_embeddings(content_type_id)\n + self.part_embeddings(part_id)\n + self.bundle_embeddings(bundle_id)\n + self.exercise_embeddings(content_id)\n + self.sin_pos_embedder(start=1)\n ).transpose(0, 1) # (S, N, E)\n \n output_src = self.emb_dropout(\n self.prior_question_elapsed_time_embeddings(prior_question_elapsed_time)\n + self.prior_question_had_explanation_embeddings(prior_question_had_explanation)\n + self.encountered_embeddings(encountered_id)\n + self.cont_mlp(\n torch.cat([\n cont_streaks,\n diagnostic,\n session,\n continuous,\n ], dim=-1)\n )\n \n + self.sin_pos_embedder(start=0)\n ).transpose(0, 1) # (S, N, E)\n\n history_src = self.emb_dropout(\n self.correctness_embeddings(correctness_id)\n + self.sin_pos_embedder(start=1)\n ).transpose(0, 1)\n \n history = self.transformer_history(\n src=history_src,\n mask=history_mask,\n src_key_padding_mask=padding_mask,\n )\n with torch.no_grad():\n history[torch.isnan(history)] = 0\n\n memory = self.transformer_encoder(\n src=embedded_src,\n mask=future_mask,\n src_key_padding_mask=padding_mask,\n )\n\n output = self.transformer_decoder(\n tgt = output_src + history,\n memory = memory,\n tgt_mask = future_mask,\n memory_mask = future_mask,\n tgt_key_padding_mask = padding_mask,\n memory_key_padding_mask = padding_mask,\n )\n \n return self.decoder(output).transpose(1, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class SiameseNetwork(nn.Module):\n def __init__(self, networks):\n super(SiameseNetwork, self).__init__()\n self.networks = nn.ModuleList(networks)\n self.power = nn.Parameter(torch.FloatTensor([1]))\n \n @torch.cuda.amp.autocast()\n def forward(\n self,\n user_id, content_id, content_type_id, part_id, encountered_id,\n prior_question_elapsed_time, prior_question_had_explanation,\n correctness_id, bundle_id, cont_streaks,\n continuous, session, diagnostic, padding_mask, task_container_id\n ):\n futures = [\n torch.jit.fork(\n model,\n user_id, content_id, content_type_id, part_id, encountered_id,\n prior_question_elapsed_time, prior_question_had_explanation,\n correctness_id, bundle_id, cont_streaks,\n continuous, session, diagnostic, padding_mask, task_container_id\n ) for model in self.networks\n ]\n outputs = [torch.jit.wait(fut) for fut in futures]\n return (torch.stack(outputs) ** self.power).mean(dim=0)","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# Classes"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"class UserDF():\n def pop_if_needed(self):\n # Ensure we're only keeping the final 100:\n if self.lensize[0] < 128:\n return True\n \n self.user_id.pop(0)\n self.task_container_id.pop(0)\n\n #############\n # We start indexing from here, since the rest get cut off:\n self.content_id.pop(0)\n self.part_id.pop(0)\n self.prior_question_elapsed_time.pop(0)\n self.prior_question_had_explanation.pop(0)\n self.incorrect_rank.pop(0) # decoder feature that needs help..........\n self.content_type_id.pop(0)\n self.bundle_id.pop(0)\n\n # only these of the 4 have signal (0-index)\n self.answer_ratio1.pop(0)\n self.answer_ratio2.pop(0)\n self.correct_streak_u.pop(0)\n self.incorrect_streak_u.pop(0)\n self.correct_streak_alltime_u.pop(0)\n self.incorrect_streak_alltime_u.pop(0)\n\n # our session information + a lifetime field (elapsed time sum on platform—probably should be capped)\n self.session_content_num_u.pop(0)\n self.session_duration_u.pop(0)\n self.session_ans_duration_u.pop(0)\n self.lifetime_ans_duration_u.pop(0)\n\n # First block of continuous - timestamp intervals\n self.lag_ts_u_recency.pop(0)\n self.last_ts_u_recency1.pop(0)\n self.last_ts_u_recency2.pop(0)\n self.last_ts_u_recency3.pop(0)\n self.last_correct_ts_u_recency.pop(0)\n self.last_incorrect_ts_u_recency.pop(0)\n\n # Second block of continuous - how often user is right on global/part/session basis\n self.correctness_u_recency.pop(0)\n self.part_correctness_u_recency.pop(0)\n self.session_correctness_u.pop(0)\n\n # Have we been served this content before?\n self.encountered.pop(0)\n\n # Average score of most frequent asked questions\n self.diagnostic_u_recency_1.pop(0)\n self.diagnostic_u_recency_2.pop(0)\n self.diagnostic_u_recency_3.pop(0)\n self.diagnostic_u_recency_4.pop(0)\n self.diagnostic_u_recency_5.pop(0)\n self.diagnostic_u_recency_6.pop(0)\n\n self.answered_correctly.pop(0)\n return False\n \n def __init__(self):\n # TODO: We may need to add some variables to keep track of how many predictions\n # And where in test_df they are\n self.lensize = [0]\n \n ##########\n self.user_id = []\n self.task_container_id = []\n\n #############\n # We start indexing from here, since the rest get cut off:\n self.content_id = []\n self.part_id = []\n self.prior_question_elapsed_time = []\n self.prior_question_had_explanation = []\n self.incorrect_rank = [] # decoder feature that needs help..........\n self.content_type_id = []\n self.bundle_id = []\n\n # only these of the 4 have signal (0-index)\n self.answer_ratio1 = []\n self.answer_ratio2 = []\n self.correct_streak_u = []\n self.incorrect_streak_u = []\n self.correct_streak_alltime_u = []\n self.incorrect_streak_alltime_u = []\n\n # our session information + a lifetime field (elapsed time sum on platform—probably should be capped)\n self.session_content_num_u = []\n self.session_duration_u = []\n self.session_ans_duration_u = []\n self.lifetime_ans_duration_u = []\n\n # First block of continuous - timestamp intervals\n self.lag_ts_u_recency = []\n self.last_ts_u_recency1 = []\n self.last_ts_u_recency2 = []\n self.last_ts_u_recency3 = []\n self.last_correct_ts_u_recency = []\n self.last_incorrect_ts_u_recency = []\n\n # Second block of continuous - how often user is right on global/part/session basis\n self.correctness_u_recency = []\n self.part_correctness_u_recency = []\n self.session_correctness_u = []\n\n # Have we been served this content before?\n self.encountered = []\n\n # Average score of most frequent asked questions\n self.diagnostic_u_recency_1 = []\n self.diagnostic_u_recency_2 = []\n self.diagnostic_u_recency_3 = []\n self.diagnostic_u_recency_4 = []\n self.diagnostic_u_recency_5 = []\n self.diagnostic_u_recency_6 = []\n\n self.answered_correctly = []","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"class UserFeatures():\n def __init__(self):\n # Everything needs to be a list so we can pass references..\n # Idx=0 current ts tcid, Idx=1 start idx of this TCID, Idx=2 counter of items in TCID\n # Idx=0 current ts tcid, Idx=1 counter, Idx=2 is [idx,idx,idx]\n self.cached_task_container_id_u = [np.nan]\n\n # These two features are only used to compute the lag feature.\n # Index: 0 = current, 1=previous\n self.task_container_num_questions_u = [0,0] # Only updated on question_id's!\n # Index: 0 = ts, 1=per-bundle1, 1 = per-bundle2,\n self.last_question_ts_u_recency2 = [np.nan,np.nan,np.nan]\n\n # Idx0=ts, Idx1=CorrectTS, Idx2=CountTS, Idx3=CorrectBundleNoLeak, Idx4=CountBundleBundleNoLeak\n # Idx5=CumAnsTimeSessionNoLeak, Idx6=CumAnsTimePlatformNoLeak\n self.session_u = [np.nan, 2,3, 2,3, 0,0]\n\n # Index: 0=ts, 1 = per-bundle1, 2 = per-bundle2, 3 = per-bundle3, ...\n self.last_ts_u = [np.nan,np.nan,np.nan,np.nan]\n\n # Index: 0 = per-ts, 1 = per-bundle\n self.last_correct_ts_u = [0,0] #[np.nan,np.nan]\n\n # Index: 0,1 = per-ts, 2,3 = per-bundle. 0=correct qs, 1=total qs\n self.last_incorrect_ts_u = [0,0] #[np.nan,np.nan]\n\n # Index: 0,1 = per-ts, 2,3 = per-bundle. 0=correct qs, 1=total qs\n self.correctness_u = [2,3,2,3] # 4right,2wrong=66.7%, mean target\n\n # 7*4, [7parts][0=correct_ts,1=count_ts,2=correct_bundle,3=count_bundle]\n self.part_correctness_u = 7 * [[2,3,2,3]]\n\n # diagnostic_content_id_order_map\n # diagnostic_content_id_mean_map\n # diagnostic_content_id_mean_map_values\n self.diagnostic_u_1 = [\n # Target Grouping rather than association grouping for now:\n #26 \t6911 \t0.818782\n #33 \t7901 \t0.824779\n\n # First two indices are 'update indicators', last two are default values that are set at end of each TCID\n np.nan, np.nan, 0.818782, 0.824779,\n\n # Last value is default mean\n 0.8217805\n ]\n\n self.diagnostic_u_2 = [\n # 30 \t7219 \t0.594133\n # 25 \t6910 \t0.602537\n # 4 \t2066 \t0.614043\n # 21 \t6879 \t0.619330\n # 3 \t2065 \t0.621588\n # 1 \t1279 \t0.635821\n np.nan, np.nan, np.nan, np.nan, np.nan, np.nan,\n 0.594133, 0.602537, 0.614043, 0.619330, 0.621588, 0.635821,\n\n 0.6145753333333334\n ]\n\n self.diagnostic_u_3 = [\n # 12 \t3365 \t0.532004\n # 16 \t4697 \t0.545093\n # 5 \t2594 \t0.557091\n np.nan, np.nan, np.nan,\n 0.532004, 0.545093, 0.557091,\n\n 0.5447293333333334\n ]\n\n self.diagnostic_u_4 = [\n # 20 \t6878 \t0.430956\n # 27 \t6912 \t0.431397\n # 15 \t4493 \t0.440798\n # 7 \t2596 \t0.468018\n # 29 \t7218 \t0.485773\n np.nan, np.nan, np.nan, np.nan, np.nan,\n 0.430956, 0.431397, 0.440798, 0.468018, 0.485773,\n\n 0.4513884\n ]\n\n self.diagnostic_u_5 = [\n # 24 \t6909 \t0.390869\n # 32 \t7877 \t0.393220\n # 22 \t6880 \t0.404521\n # 28 \t7217 \t0.405293\n # 6 \t2595 \t0.419734\n np.nan, np.nan, np.nan, np.nan, np.nan,\n 0.390869, 0.393220, 0.404521, 0.405293, 0.419734,\n 0.4027274\n ]\n\n self.diagnostic_u_6 = [\n # 10 \t2949 \t0.211211\n # 9 \t2948 \t0.226543\n # 14 \t4121 \t0.230986\n # 23 \t6881 \t0.252001\n # 18 \t6174 \t0.255422\n # 31 \t7220 \t0.262637\n # 11 \t3364 \t0.268132\n np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan,\n 0.211211, 0.226543, 0.230986, 0.252001, 0.255422, 0.262637, 0.268132,\n 0.24384742857142858,\n ]\n \n #user_answer_streak_map\n self.answer_ratio = [\n # Idx0 = last user answer, Idx1 = streak count\n # We don't have to limit to bundles since we have access to this at read time\n 1,1, # UA=1 counter, UA=1 bundle \n 1,1, # UA=2 counter, UA=2 bundle\n 4,4, # total q asked\n ]\n \n self.streak_u = [\n 0,0,0, # correct ts, bundle, all-time bundle\n 0,0,0, # incorrect ts, bundle, all-time bundle\n ]\n\n # We don't need to update @ Bundle because we'll only see a question once in a TCID\n self.content_encounter = bitarray(13550+425, endian='little')\n self.content_encounter.setall(0)\n \n def get_features(self):\n return (\n self.cached_task_container_id_u,\n self.task_container_num_questions_u,\n self.last_question_ts_u_recency2,\n self.session_u,\n self.last_ts_u,\n self.last_correct_ts_u,\n self.last_incorrect_ts_u,\n self.correctness_u,\n self.part_correctness_u,\n self.diagnostic_u_1,\n self.diagnostic_u_2,\n self.diagnostic_u_3,\n self.diagnostic_u_4,\n self.diagnostic_u_5,\n self.diagnostic_u_6,\n self.answer_ratio,\n self.streak_u,\n self.content_encounter,\n )","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"# For pickling\ndef newUserFeatures(): return UserFeatures()\ndef newUserDF(): return UserDF()","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# Load up all mappings"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"(\n embedded_user_ids_map, mask_user_ids, part_ids_map, lecture_ids_map, bundle_id_map,\n question_incorrect_ranks,\n answer_ratio1_mean, answer_ratio1_std,\n answer_ratio2_mean, answer_ratio2_std,\n correct_streak_u_mean, correct_streak_u_std,\n incorrect_streak_u_mean, incorrect_streak_u_std,\n correct_streak_alltime_u_mean, correct_streak_alltime_u_std,\n incorrect_streak_alltime_u_mean, incorrect_streak_alltime_u_std,\n diagnostic_u_recency_1_mean, diagnostic_u_recency_1_std,\n diagnostic_u_recency_2_mean, diagnostic_u_recency_2_std,\n diagnostic_u_recency_3_mean, diagnostic_u_recency_3_std,\n diagnostic_u_recency_4_mean, diagnostic_u_recency_4_std,\n diagnostic_u_recency_5_mean, diagnostic_u_recency_5_std,\n diagnostic_u_recency_6_mean, diagnostic_u_recency_6_std,\n session_content_num_u_mean, session_content_num_u_std,\n session_duration_u_mean, session_duration_u_std,\n session_correctness_u_mean, session_correctness_u_std,\n session_ans_duration_u_mean, session_ans_duration_u_std,\n lifetime_ans_duration_u_mean, lifetime_ans_duration_u_std,\n correctness_u_recency_mean, correctness_u_recency_std,\n part_correctness_u_recency_mean, part_correctness_u_recency_std,\n lag_ts_u_recency_mean, lag_ts_u_recency_std,\n last_ts_u_recency1_mean, last_ts_u_recency1_std,\n last_ts_u_recency2_mean, last_ts_u_recency2_std,\n last_ts_u_recency3_mean, last_ts_u_recency3_std,\n last_incorrect_ts_u_recency_mean, last_incorrect_ts_u_recency_std,\n last_correct_ts_u_recency_mean, last_correct_ts_u_recency_std,\n) = joblib.load('../input/uthman-riiid/mappings.jlib')\n\n# Build this out:\nuser_features = joblib.load('../input/uthman-riiid/user_features_full.jlib')\n\n# Also load up cached 128 last records of users....\nuser_dfs = joblib.load('../input/uthman-riiid/user_dfs_full.jlib')\n\n# NOTE: We aaren't embedding user_ids, even though I think it;d be a great idea...\n# TODO: This needs to include validation data.....\n# NOTE: User IDs already mapped.... but we add 1 so that we can use 0 for unknown users\nembedded_user_ids_map = joblib.load('../input/uthman-riiid/mapped_embedded_user_ids.jlib')\nembedded_user_ids_map = {i:v+1 for i,v in embedded_user_ids_map.items()}\n\nquestion_incorrect_ranks = joblib.load(f'../input/uthman-riiid/question_incorrect_ranks.pkl')","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# add_user_feats_without_update"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"def add_user_feats_without_update(df, user_features): \n # Index: 0 = per-ts, 1 = per-bundle1, 2 = per-bundle2, 3 = per-bundle3 \n session_content_num_u = np.zeros(df.shape[0], dtype=np.float32)\n session_duration_u = np.zeros(df.shape[0], dtype=np.float64)\n session_correctness_u = np.zeros(df.shape[0], dtype=np.float32)\n session_ans_duration_u= np.zeros(df.shape[0], dtype=np.float64)\n lifetime_ans_duration_u= np.zeros(df.shape[0], dtype=np.float64)\n lag_ts_u_recency = np.zeros(df.shape[0], dtype=np.float64)\n last_ts_u_recency1 = np.zeros(df.shape[0], dtype=np.float64)\n last_ts_u_recency2 = np.zeros(df.shape[0], dtype=np.float64)\n last_ts_u_recency3 = np.zeros(df.shape[0], dtype=np.float64)\n last_incorrect_ts_u_recency = np.zeros(df.shape[0], dtype=np.float64)\n last_correct_ts_u_recency = np.zeros(df.shape[0], dtype=np.float64)\n correctness_u_recency = np.zeros(df.shape[0], dtype=np.float32)\n part_correctness_u_recency = np.zeros(df.shape[0], dtype=np.float32)\n diagnostic_u_recency = np.zeros((df.shape[0],6), dtype=np.float32)\n answer_ratio1 = np.zeros(df.shape[0], dtype=np.float32)\n answer_ratio2 = np.zeros(df.shape[0], dtype=np.float32)\n correct_streak_u = np.zeros(df.shape[0], dtype=np.uint16)\n incorrect_streak_u = np.zeros(df.shape[0], dtype=np.uint16)\n correct_streak_alltime_u = np.zeros(df.shape[0], dtype=np.uint16)\n incorrect_streak_alltime_u = np.zeros(df.shape[0], dtype=np.uint16)\n encountered = np.zeros(df.shape[0], dtype=np.uint8)\n \n # Ideally, we can pull less\n for idx, (_, user_id, task_container_id, content_type_id, part_id, timestamp, prior_question_elapsed_time_cont) in enumerate(tqdm(df[[\n 'user_id', 'task_container_id', 'content_type_id', 'part_id', 'timestamp', 'prior_question_elapsed_time_cont'\n ]].itertuples())):\n # Adjustments\n part_id -= 1 # So we can index at 0\n (\n cached_task_container_id_u,\n task_container_num_questions_u,\n last_question_ts_u_recency2,\n session_u,\n last_ts_u,\n last_correct_ts_u,\n last_incorrect_ts_u,\n correctness_u,\n part_correctness_u,\n diagnostic_u_1,\n diagnostic_u_2,\n diagnostic_u_3,\n diagnostic_u_4,\n diagnostic_u_5,\n diagnostic_u_6,\n answer_ratio,\n streak_u,\n content_encounter,\n ) = user_features[user_id].get_features()\n part_correctness_u__part = part_correctness_u[part_id]\n \n # Step 1) Bundle Alignment operation - in kernel submission, this will run each time\n if cached_task_container_id_u[0] != task_container_id:\n # Initialize:\n cached_task_container_id_u[0] = task_container_id\n \n # Index: 0 = per-ts, 1 = per-bundle\n last_correct_ts_u[1] = last_correct_ts_u[0]\n last_incorrect_ts_u[1] = last_incorrect_ts_u[0]\n\n # Index: 0,1 = per-ts, 2,3 = per-bundle. 0=correct qs, 1=total qs\n correctness_u[2] = correctness_u[0]\n correctness_u[3] = correctness_u[1]\n\n for part_id in range(7):\n # 7*4, [7parts][0=correct_ts,1=count_ts,2=correct_bundle,3=count_bundle]\n part_correctness_u[part_id][2] = part_correctness_u[part_id][0]\n part_correctness_u[part_id][3] = part_correctness_u[part_id][1]\n \n # Index: 0=current-ts, 1 = per-bundle1, 2 = per-bundle2, 3 = per-bundle3 \n # Order is important\n last_ts_u[3] = last_ts_u[2]\n last_ts_u[2] = last_ts_u[1]\n last_ts_u[1] = last_ts_u[0]\n last_ts_u[0] = timestamp\n \n # Streak\n streak_u[1] = streak_u[0] # Update bundle\n streak_u[4] = streak_u[3] # Update bundle\n if streak_u[2] < streak_u[1]: streak_u[2] = streak_u[1] # Update all-time\n if streak_u[5] < streak_u[4]: streak_u[5] = streak_u[4] # Update all-time\n \n # Answer Ratios\n answer_ratio[1] = answer_ratio[0]\n answer_ratio[3] = answer_ratio[2]\n answer_ratio[5] = answer_ratio[4]\n \n if task_container_id < 40:\n # Diagnostic Updates\n for i in range(2):\n if np.isnan(diagnostic_u_1[i]): continue\n diagnostic_u_1[i+2] = diagnostic_u_1[i]\n diagnostic_u_1[i] = np.nan\n\n for i in range(6):\n if np.isnan(diagnostic_u_2[i]): continue\n diagnostic_u_2[i+6] = diagnostic_u_2[i]\n diagnostic_u_2[i] = np.nan\n\n for i in range(3):\n if np.isnan(diagnostic_u_3[i]): continue\n diagnostic_u_3[i+3] = diagnostic_u_3[i]\n diagnostic_u_3[i] = np.nan\n\n for i in range(5):\n if not np.isnan(diagnostic_u_4[i]):\n diagnostic_u_4[i+5] = diagnostic_u_4[i]\n diagnostic_u_4[i] = np.nan\n if not np.isnan(diagnostic_u_5[i]):\n diagnostic_u_5[i+5] = diagnostic_u_5[i]\n diagnostic_u_5[i] = np.nan\n\n for i in range(7):\n if np.isnan(diagnostic_u_6[i]): continue\n diagnostic_u_6[i+7] = diagnostic_u_6[i]\n diagnostic_u_6[i] = np.nan\n \n # The means:\n diagnostic_u_1[-1] = np.mean(diagnostic_u_1[2:-1])\n diagnostic_u_2[-1] = np.mean(diagnostic_u_2[6:-1])\n diagnostic_u_3[-1] = np.mean(diagnostic_u_3[3:-1])\n diagnostic_u_4[-1] = np.mean(diagnostic_u_4[5:-1])\n diagnostic_u_5[-1] = np.mean(diagnostic_u_5[5:-1])\n diagnostic_u_6[-1] = np.mean(diagnostic_u_6[7:-1])\n # End diagnostic features\n ###\n \n ###\n # Lag features\n if content_type_id==0:\n # We only reset for questions; lectures will ffill()\n task_container_num_questions_u[1] = task_container_num_questions_u[0]\n task_container_num_questions_u[0] = 0\n \n # Same claculation as last_ts_u_recency2\n last_question_ts_u_recency2[2] = last_question_ts_u_recency2[1]\n last_question_ts_u_recency2[1] = last_question_ts_u_recency2[0]\n last_question_ts_u_recency2[0] = timestamp\n\n if not np.isnan(prior_question_elapsed_time_cont):\n session_u[5] += prior_question_elapsed_time_cont * task_container_num_questions_u[1]\n session_u[6] += prior_question_elapsed_time_cont * task_container_num_questions_u[1]\n else:\n # Entering a lecture:\n diff = timestamp - last_question_ts_u_recency2[0]\n last_question_ts_u_recency2[2] += diff\n last_question_ts_u_recency2[1] += diff\n last_question_ts_u_recency2[0] = timestamp\n # End lag features\n \n # Copy the session stuff over\n session_u[3] = session_u[1]\n session_u[4] = session_u[2]\n \n ###\n \n # Bake values - note, none of these change per task_container_id,\n # even though we're recalculating them all every iteration......\n lag_ts_u_recency[idx] = last_question_ts_u_recency2[1] - last_question_ts_u_recency2[2] - prior_question_elapsed_time_cont * task_container_num_questions_u[1]\n last_ts_u_recency1[idx] = timestamp - last_ts_u[1]\n last_ts_u_recency2[idx] = last_ts_u[1] - last_ts_u[2]\n last_ts_u_recency3[idx] = last_ts_u[2] - last_ts_u[3]\n last_correct_ts_u_recency[idx] = timestamp - last_correct_ts_u[1]\n last_incorrect_ts_u_recency[idx] = timestamp - last_incorrect_ts_u[1]\n correctness_u_recency[idx] = correctness_u[2] / correctness_u[3]\n part_correctness_u_recency[idx] = part_correctness_u__part[2] / part_correctness_u__part[3]\n session_content_num_u[idx] = session_u[4] - 2 # we start at 6....\n session_duration_u[idx] = 0 if np.isnan(session_u[0]) else timestamp - session_u[0]\n session_correctness_u[idx] = session_u[3] / session_u[4]\n session_ans_duration_u[idx] = session_u[5]\n lifetime_ans_duration_u[idx] = session_u[6]\n diagnostic_u_recency[idx,0] = diagnostic_u_1[-1]\n diagnostic_u_recency[idx,1] = diagnostic_u_2[-1]\n diagnostic_u_recency[idx,2] = diagnostic_u_3[-1]\n diagnostic_u_recency[idx,3] = diagnostic_u_4[-1]\n diagnostic_u_recency[idx,4] = diagnostic_u_5[-1]\n diagnostic_u_recency[idx,5] = diagnostic_u_6[-1]\n answer_ratio1[idx] = answer_ratio[1] / answer_ratio[5]\n answer_ratio2[idx] = answer_ratio[3] / answer_ratio[5]\n correct_streak_u[idx] = streak_u[1]\n incorrect_streak_u[idx] = streak_u[4]\n correct_streak_alltime_u[idx] = streak_u[2]\n incorrect_streak_alltime_u[idx] = streak_u[5]\n \n\n user_feats_df = pd.DataFrame({\n 'lag_ts_u_recency': np.log1p(lag_ts_u_recency.clip(0,86400000) / 1000),\n 'last_ts_u_recency1': np.log1p(last_ts_u_recency1.clip(650,1209600000)),\n 'last_ts_u_recency2': np.log1p(last_ts_u_recency2.clip(650,1209600000)),\n 'last_ts_u_recency3': np.log1p(last_ts_u_recency3.clip(650,1209600000)),\n 'last_correct_ts_u_recency': np.log1p(last_correct_ts_u_recency.clip(650,1209600000)),\n 'last_incorrect_ts_u_recency': np.log1p(last_incorrect_ts_u_recency.clip(650,1209600000)),\n 'correctness_u_recency': correctness_u_recency,\n 'part_correctness_u_recency': part_correctness_u_recency,\n 'session_content_num_u': np.log1p(session_content_num_u),\n 'session_duration_u': np.log1p(session_duration_u.clip(4100,18000000)),\n 'session_correctness_u': session_correctness_u,\n 'session_ans_duration_u': np.log1p(session_ans_duration_u.clip(4100,18000000)),\n 'lifetime_ans_duration_u': np.log1p(lifetime_ans_duration_u.clip(4100,None)),\n 'diagnostic_u_recency_1': diagnostic_u_recency[:, 0],\n 'diagnostic_u_recency_2': diagnostic_u_recency[:, 1],\n 'diagnostic_u_recency_3': diagnostic_u_recency[:, 2],\n 'diagnostic_u_recency_4': diagnostic_u_recency[:, 3],\n 'diagnostic_u_recency_5': diagnostic_u_recency[:, 4],\n 'diagnostic_u_recency_6': diagnostic_u_recency[:, 5],\n 'answer_ratio1': answer_ratio1,\n 'answer_ratio2': answer_ratio2,\n 'correct_streak_u': np.log1p(correct_streak_u),\n 'incorrect_streak_u': np.log1p(incorrect_streak_u),\n 'correct_streak_alltime_u': np.log1p(correct_streak_alltime_u),\n 'incorrect_streak_alltime_u': np.log1p(incorrect_streak_alltime_u),\n 'encountered': encountered,\n })\n \n user_feats_df.lag_ts_u_recency = ((user_feats_df.lag_ts_u_recency - lag_ts_u_recency_mean) / lag_ts_u_recency_std).astype(np.float32) \n user_feats_df.last_ts_u_recency1 = ((user_feats_df.last_ts_u_recency1 - last_ts_u_recency1_mean) / last_ts_u_recency1_std).astype(np.float32) \n user_feats_df.last_ts_u_recency2 = ((user_feats_df.last_ts_u_recency2 - last_ts_u_recency2_mean) / last_ts_u_recency2_std).astype(np.float32) \n user_feats_df.last_ts_u_recency3 = ((user_feats_df.last_ts_u_recency3 - last_ts_u_recency3_mean) / last_ts_u_recency3_std).astype(np.float32) \n user_feats_df.last_incorrect_ts_u_recency = ((user_feats_df.last_incorrect_ts_u_recency - last_incorrect_ts_u_recency_mean) / last_incorrect_ts_u_recency_std).astype(np.float32)\n user_feats_df.last_correct_ts_u_recency = ((user_feats_df.last_correct_ts_u_recency - last_correct_ts_u_recency_mean ) / last_correct_ts_u_recency_std).astype(np.float32) \n user_feats_df.correctness_u_recency = ((user_feats_df.correctness_u_recency - correctness_u_recency_mean ) / correctness_u_recency_std).astype(np.float32) \n user_feats_df.part_correctness_u_recency = ((user_feats_df.part_correctness_u_recency - part_correctness_u_recency_mean ) / part_correctness_u_recency_std).astype(np.float32) \n user_feats_df.session_content_num_u = ((user_feats_df.session_content_num_u - session_content_num_u_mean ) / session_content_num_u_std).astype(np.float32) \n user_feats_df.session_duration_u = ((user_feats_df.session_duration_u - session_duration_u_mean ) / session_duration_u_std).astype(np.float32) \n user_feats_df.session_correctness_u = ((user_feats_df.session_correctness_u - session_correctness_u_mean ) / session_correctness_u_std).astype(np.float32) \n user_feats_df.session_ans_duration_u = ((user_feats_df.session_ans_duration_u - session_ans_duration_u_mean ) / session_ans_duration_u_std).astype(np.float32) \n user_feats_df.lifetime_ans_duration_u= ((user_feats_df.lifetime_ans_duration_u - lifetime_ans_duration_u_mean) / lifetime_ans_duration_u_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_1 = ((user_feats_df.diagnostic_u_recency_1 - diagnostic_u_recency_1_mean ) / diagnostic_u_recency_1_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_2 = ((user_feats_df.diagnostic_u_recency_2 - diagnostic_u_recency_2_mean ) / diagnostic_u_recency_2_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_3 = ((user_feats_df.diagnostic_u_recency_3 - diagnostic_u_recency_3_mean ) / diagnostic_u_recency_3_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_4 = ((user_feats_df.diagnostic_u_recency_4 - diagnostic_u_recency_4_mean ) / diagnostic_u_recency_4_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_5 = ((user_feats_df.diagnostic_u_recency_5 - diagnostic_u_recency_5_mean ) / diagnostic_u_recency_5_std).astype(np.float32) \n user_feats_df.diagnostic_u_recency_6 = ((user_feats_df.diagnostic_u_recency_6 - diagnostic_u_recency_6_mean ) / diagnostic_u_recency_6_std).astype(np.float32) \n user_feats_df.answer_ratio1 = ((user_feats_df.answer_ratio1 - answer_ratio1_mean ) / answer_ratio1_std).astype(np.float32) \n user_feats_df.answer_ratio2 = ((user_feats_df.answer_ratio2 - answer_ratio2_mean ) / answer_ratio2_std).astype(np.float32) \n user_feats_df.correct_streak_u = ((user_feats_df.correct_streak_u - correct_streak_u_mean ) / correct_streak_u_std).astype(np.float32) \n user_feats_df.incorrect_streak_u = ((user_feats_df.incorrect_streak_u - incorrect_streak_u_mean ) / incorrect_streak_u_std).astype(np.float32) \n user_feats_df.correct_streak_alltime_u = ((user_feats_df.correct_streak_alltime_u - correct_streak_alltime_u_mean ) / correct_streak_alltime_u_std).astype(np.float32) \n user_feats_df.incorrect_streak_alltime_u = ((user_feats_df.incorrect_streak_alltime_u - incorrect_streak_alltime_u_mean ) / incorrect_streak_alltime_u_std).astype(np.float32) \n\n user_feats_df.loc[user_feats_df.lag_ts_u_recency.isna(), 'lag_ts_u_recency'] = 0\n user_feats_df.loc[user_feats_df.last_ts_u_recency1.isna(), 'last_ts_u_recency1'] = 0\n user_feats_df.loc[user_feats_df.last_ts_u_recency2.isna(), 'last_ts_u_recency2'] = 0\n user_feats_df.loc[user_feats_df.last_ts_u_recency3.isna(), 'last_ts_u_recency3'] = 0\n user_feats_df.loc[user_feats_df.last_incorrect_ts_u_recency.isna(), 'last_incorrect_ts_u_recency'] = 0\n user_feats_df.loc[user_feats_df.last_correct_ts_u_recency.isna(), 'last_correct_ts_u_recency'] = 0\n user_feats_df.loc[user_feats_df.correctness_u_recency.isna(), 'correctness_u_recency'] = 0.653417715747257 # mean answered correctly\n user_feats_df.loc[user_feats_df.part_correctness_u_recency.isna(), 'part_correctness_u_recency'] = 0.653417715747257 # mean answered correctly across dset \n user_feats_df.loc[user_feats_df.session_content_num_u.isna(), 'session_content_num_u'] = 0\n user_feats_df.loc[user_feats_df.session_duration_u.isna(), 'session_duration_u'] = 0\n user_feats_df.loc[user_feats_df.session_correctness_u.isna(), 'session_correctness_u'] = 0.653417715747257\n user_feats_df.loc[user_feats_df.session_ans_duration_u.isna(), 'session_ans_duration_u'] = 0\n user_feats_df.loc[user_feats_df.lifetime_ans_duration_u.isna(), 'lifetime_ans_duration_u'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_1.isna(), 'diagnostic_u_recency_1'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_2.isna(), 'diagnostic_u_recency_2'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_3.isna(), 'diagnostic_u_recency_3'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_4.isna(), 'diagnostic_u_recency_4'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_5.isna(), 'diagnostic_u_recency_5'] = 0\n user_feats_df.loc[user_feats_df.diagnostic_u_recency_6.isna(), 'diagnostic_u_recency_6'] = 0\n user_feats_df.loc[user_feats_df.answer_ratio1.isna(), 'answer_ratio1'] = 0\n user_feats_df.loc[user_feats_df.answer_ratio2.isna(), 'answer_ratio2'] = 0\n user_feats_df.loc[user_feats_df.correct_streak_u.isna(), 'correct_streak_u'] = 0\n user_feats_df.loc[user_feats_df.incorrect_streak_u.isna(), 'incorrect_streak_u'] = 0\n user_feats_df.loc[user_feats_df.correct_streak_alltime_u.isna(), 'correct_streak_alltime_u'] = 0\n user_feats_df.loc[user_feats_df.incorrect_streak_alltime_u.isna(), 'incorrect_streak_alltime_u'] = 0\n\n # TODO: Let's do this outside?\n df = pd.concat([df, user_feats_df], axis=1)\n return df","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# update_user_feats"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"def update_user_feats(df, user_features):\n # Ideally, we can pull less\n for idx, (_, user_id, content_id, task_container_id, content_type_id, part_id, timestamp, answered_correctly, user_answer) in enumerate(tqdm(df[[\n 'user_id', 'content_id', 'task_container_id', 'content_type_id', 'part_id', 'timestamp', 'answered_correctly', 'user_answer'\n ]].itertuples())):\n # Adjustments\n part_id -= 1 # So we can index at 0\n (\n cached_task_container_id_u,\n task_container_num_questions_u,\n last_question_ts_u_recency2,\n session_u,\n last_ts_u,\n last_correct_ts_u,\n last_incorrect_ts_u,\n correctness_u,\n part_correctness_u,\n diagnostic_u_1,\n diagnostic_u_2,\n diagnostic_u_3,\n diagnostic_u_4,\n diagnostic_u_5,\n diagnostic_u_6,\n answer_ratio,\n streak_u,\n content_encounter,\n ) = user_features[user_id].get_features()\n part_correctness_u__part = part_correctness_u[part_id]\n \n # Step 0) Handle session stuff\n if np.isnan(last_ts_u[0]) or timestamp - last_ts_u[0] > 3600000 // 2:\n # Reset the session\n session_u[0] = timestamp\n session_u[1] = 2 # correct\n session_u[2] = 3 # count\n session_u[3] = 2 # correct\n session_u[4] = 3 # count\n session_u[5] = 0 # CumAnsTimeSessionNoLeak\n \n if content_encounter[content_id] == 1:\n encountered[idx] = 2\n else:\n # 0 for pad\n encountered[idx] = 1\n content_encounter[content_id] = 1\n \n if content_type_id==0:\n # Lag: Increase for each question we see\n task_container_num_questions_u[0] += 1\n answer_ratio[5] += 1\n \n # Answer Selection Ratio for questions only\n if user_answer==1:\n answer_ratio[0] += 1\n elif user_answer==2:\n answer_ratio[2] += 1\n\n # Content Counters\n correctness_u[1] += 1\n part_correctness_u__part[1] += 1\n session_u[2] += 1\n \n if answered_correctly==0:\n last_incorrect_ts_u[0] = timestamp\n streak_u[0] = 0 # reset correct counter\n streak_u[3] += 1 # increment incorrect counter\n else:\n last_correct_ts_u[0] = timestamp\n streak_u[0] += 1 # increment correct counter\n streak_u[3] = 0 # reset incorrect counter\n\n # Correct Content Counters\n correctness_u[0] += 1\n part_correctness_u__part[0] += 1\n session_u[1] += 1\n \n # Diagnostic Features\n if task_container_id < 40:\n if content_id == 6911: diagnostic_u_1[0] = answered_correctly\n elif content_id == 7901: diagnostic_u_1[1] = answered_correctly\n elif content_id == 7219: diagnostic_u_2[0] = answered_correctly\n elif content_id == 6910: diagnostic_u_2[1] = answered_correctly\n elif content_id == 2066: diagnostic_u_2[2] = answered_correctly\n elif content_id == 6879: diagnostic_u_2[3] = answered_correctly\n elif content_id == 2065: diagnostic_u_2[4] = answered_correctly\n elif content_id == 1279: diagnostic_u_2[5] = answered_correctly\n elif content_id == 3365: diagnostic_u_3[0] = answered_correctly\n elif content_id == 4697: diagnostic_u_3[1] = answered_correctly\n elif content_id == 2594: diagnostic_u_3[2] = answered_correctly\n elif content_id == 6878: diagnostic_u_4[0] = answered_correctly\n elif content_id == 6912: diagnostic_u_4[1] = answered_correctly\n elif content_id == 4493: diagnostic_u_4[2] = answered_correctly\n elif content_id == 2596: diagnostic_u_4[3] = answered_correctly\n elif content_id == 7218: diagnostic_u_4[4] = answered_correctly\n elif content_id == 6909: diagnostic_u_5[0] = answered_correctly\n elif content_id == 7877: diagnostic_u_5[1] = answered_correctly\n elif content_id == 6880: diagnostic_u_5[2] = answered_correctly\n elif content_id == 7217: diagnostic_u_5[3] = answered_correctly\n elif content_id == 2595: diagnostic_u_5[4] = answered_correctly\n elif content_id == 2949: diagnostic_u_6[0] = answered_correctly\n elif content_id == 2948: diagnostic_u_6[1] = answered_correctly\n elif content_id == 4121: diagnostic_u_6[2] = answered_correctly\n elif content_id == 6881: diagnostic_u_6[3] = answered_correctly\n elif content_id == 6174: diagnostic_u_6[4] = answered_correctly\n elif content_id == 7220: diagnostic_u_6[5] = answered_correctly\n elif content_id == 3364: diagnostic_u_6[6] = answered_correctly","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# update_user_df"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"def update_user_df(df, user_dfs):\n # We build the batch ourselves:\n _user_id, _task_container_id, _content_id, _part_id, _prior_question_elapsed_time,\n _prior_question_had_explanation, _incorrect_rank, _content_type_id, _bundle_id,\n _answer_ratio1, _answer_ratio2, _correct_streak_u, _incorrect_streak_u, _correct_streak_alltime_u,\n _incorrect_streak_alltime_u, _session_content_num_u, _session_duration_u, _session_ans_duration_u,\n _lifetime_ans_duration_u, _lag_ts_u_recency, _last_ts_u_recency1, _last_ts_u_recency2, _last_ts_u_recency3,\n _last_correct_ts_u_recency, _last_incorrect_ts_u_recency, _correctness_u_recency,\n _part_correctness_u_recency, _session_correctness_u, _encountered, _diagnostic_u_recency_1,\n _diagnostic_u_recency_2, _diagnostic_u_recency_3, _diagnostic_u_recency_4, _diagnostic_u_recency_5,\n _diagnostic_u_recency_6, _answered_correctly = [],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[] \n \n for idx, (\n _, user_id, task_container_id, content_id, part_id, prior_question_elapsed_time,\n prior_question_had_explanation, incorrect_rank, content_type_id, bundle_id,\n answer_ratio1, answer_ratio2, correct_streak_u, incorrect_streak_u, correct_streak_alltime_u,\n incorrect_streak_alltime_u, session_content_num_u, session_duration_u, session_ans_duration_u,\n lifetime_ans_duration_u, lag_ts_u_recency, last_ts_u_recency1, last_ts_u_recency2, last_ts_u_recency3,\n last_correct_ts_u_recency, last_incorrect_ts_u_recency, correctness_u_recency,\n part_correctness_u_recency, session_correctness_u, encountered, diagnostic_u_recency_1,\n diagnostic_u_recency_2, diagnostic_u_recency_3, diagnostic_u_recency_4, diagnostic_u_recency_5,\n diagnostic_u_recency_6, answered_correctly\n ) in enumerate(tqdm(df[dataset_cols[COL_OFFSET_CUT:]].itertuples())):\n # TODO: We may have to cache some variables to record:\n # 1) How many items we're predicting in for this users TCID\n # 2) Where in test_df these values should map to (what line = idx??)\n \n # Find or create the user df:\n state = user_dfs[user_id]\n if state.pop_if_needed():\n state.lensize[0] += 1\n \n state.user_id.append(user_id)\n state.task_container_id.append(task_container_id)\n state.content_id.append(content_id)\n state.part_id.append(part_id)\n state.prior_question_elapsed_time.append(prior_question_elapsed_time)\n state.prior_question_had_explanation.append(prior_question_had_explanation)\n state.incorrect_rank.append(incorrect_rank) # decoder feature that needs help..........\n state.content_type_id.append(content_type_id)\n state.bundle_id.append(bundle_id)\n\n # only these of the 4 have signal (0-index)\n state.answer_ratio1.append(answer_ratio1)\n state.answer_ratio2.append(answer_ratio2)\n state.correct_streak_u.append(correct_streak_u)\n state.incorrect_streak_u.append(incorrect_streak_u)\n state.correct_streak_alltime_u.append(correct_streak_alltime_u)\n state.incorrect_streak_alltime_u.append(incorrect_streak_alltime_u)\n\n # our session information + a lifetime field (elapsed time sum on platform—probably should be capped)\n state.session_content_num_u.append(session_content_num_u)\n state.session_duration_u.append(session_duration_u)\n state.session_ans_duration_u.append(session_ans_duration_u)\n state.lifetime_ans_duration_u.append(lifetime_ans_duration_u)\n\n # First block of continuous - timestamp intervals\n state.lag_ts_u_recency.append(lag_ts_u_recency)\n state.last_ts_u_recency1.append(last_ts_u_recency1)\n state.last_ts_u_recency2.append(last_ts_u_recency2)\n state.last_ts_u_recency3.append(last_ts_u_recency3)\n state.last_correct_ts_u_recency.append(last_correct_ts_u_recency)\n state.last_incorrect_ts_u_recency.append(last_incorrect_ts_u_recency)\n\n # Second block of continuous - how often user is right on global/part/session basis\n state.correctness_u_recency.append(correctness_u_recency)\n state.part_correctness_u_recency.append(part_correctness_u_recency)\n state.session_correctness_u.append(session_correctness_u)\n\n # Have we been served this content before?\n state.encountered.append(encountered)\n\n # Average score of most frequent asked questions\n state.diagnostic_u_recency_1.append(diagnostic_u_recency_1)\n state.diagnostic_u_recency_2.append(diagnostic_u_recency_2)\n state.diagnostic_u_recency_3.append(diagnostic_u_recency_3)\n state.diagnostic_u_recency_4.append(diagnostic_u_recency_4)\n state.diagnostic_u_recency_5.append(diagnostic_u_recency_5)\n state.diagnostic_u_recency_6.append(diagnostic_u_recency_6)\n\n state.answered_correctly.append(answered_correctly)\n \n ################################################\n # Finally, add to our BATCH, which will be fed into a data loader to do padding\n _user_id.append(state.user_id)\n _task_container_id.append(state.task_container_id)\n _content_id.append(state.content_id)\n _part_id.append(state.part_id)\n _prior_question_elapsed_time.append(state.prior_question_elapsed_time)\n _prior_question_had_explanation.append(state.prior_question_had_explanation)\n _incorrect_rank.append(state.incorrect_rank)\n _content_type_id.append(state.content_type_id)\n _bundle_id.append(state.bundle_id)\n _answer_ratio1.append(state.answer_ratio1)\n _answer_ratio2.append(state.answer_ratio2)\n _correct_streak_u.append(state.correct_streak_u)\n _incorrect_streak_u.append(state.incorrect_streak_u)\n _correct_streak_alltime_u.append(state.correct_streak_alltime_u)\n _incorrect_streak_alltime_u.append(state.incorrect_streak_alltime_u)\n _session_content_num_u.append(state.session_content_num_u)\n _session_duration_u.append(state.session_duration_u)\n _session_ans_duration_u.append(state.session_ans_duration_u)\n _lifetime_ans_duration_u.append(state.lifetime_ans_duration_u)\n _lag_ts_u_recency.append(state.lag_ts_u_recency)\n _last_ts_u_recency1.append(state.last_ts_u_recency1)\n _last_ts_u_recency2.append(state.last_ts_u_recency2)\n _last_ts_u_recency3.append(state.last_ts_u_recency3)\n _last_correct_ts_u_recency.append(state.last_correct_ts_u_recency)\n _last_incorrect_ts_u_recency.append(state.last_incorrect_ts_u_recency)\n _correctness_u_recency.append(state.correctness_u_recency)\n _part_correctness_u_recency.append(state.part_correctness_u_recency)\n _session_correctness_u.append(state.session_correctness_u)\n _encountered.append(state.encountered)\n _diagnostic_u_recency_1.append(state.diagnostic_u_recency_1)\n _diagnostic_u_recency_2.append(state.diagnostic_u_recency_=2)\n _diagnostic_u_recency_3.append(state.diagnostic_u_recency_3)\n _diagnostic_u_recency_4.append(state.diagnostic_u_recency_4)\n _diagnostic_u_recency_5.append(state.diagnostic_u_recency_5)\n _diagnostic_u_recency_6.append(state.diagnostic_u_recency_6)\n _answered_correctly.append(state.answered_correctly)\n \n return (\n _user_id, _task_container_id, _content_id, _part_id, _prior_question_elapsed_time,\n _prior_question_had_explanation, _incorrect_rank, _content_type_id, _bundle_id,\n _answer_ratio1, _answer_ratio2, _correct_streak_u, _incorrect_streak_u, _correct_streak_alltime_u,\n _incorrect_streak_alltime_u, _session_content_num_u, _session_duration_u, _session_ans_duration_u,\n _lifetime_ans_duration_u, _lag_ts_u_recency, _last_ts_u_recency1, _last_ts_u_recency2, _last_ts_u_recency3,\n _last_correct_ts_u_recency, _last_incorrect_ts_u_recency, _correctness_u_recency,\n _part_correctness_u_recency, _session_correctness_u, _encountered, _diagnostic_u_recency_1,\n _diagnostic_u_recency_2, _diagnostic_u_recency_3, _diagnostic_u_recency_4, _diagnostic_u_recency_5,\n _diagnostic_u_recency_6, _answered_correctly\n )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":false},"cell_type":"code","source":"class Riiid(torch.utils.data.Dataset):\n def __init__(self, test_df):\n global user_dfs\n \n (\n self.user_id, self.task_container_id, self.content_id, self.part_id, self.prior_question_elapsed_time,\n self.prior_question_had_explanation, self.incorrect_rank, self.content_type_id, self.bundle_id,\n self.answer_ratio1, self.answer_ratio2, self.correct_streak_u, self.incorrect_streak_u, self.correct_streak_alltime_u,\n self.incorrect_streak_alltime_u, self.session_content_num_u, self.session_duration_u, self.session_ans_duration_u,\n self.lifetime_ans_duration_u, self.lag_ts_u_recency, self.last_ts_u_recency1, self.last_ts_u_recency2, self.last_ts_u_recency3,\n self.last_correct_ts_u_recency, self.last_incorrect_ts_u_recency, self.correctness_u_recency,\n self.part_correctness_u_recency, self.session_correctness_u, self.encountered, self.diagnostic_u_recency_1,\n self.diagnostic_u_recency_2, self.diagnostic_u_recency_3, self.diagnostic_u_recency_4, self.diagnostic_u_recency_5,\n self.diagnostic_u_recency_6, self.answered_correctly\n \n ) = update_user_df(test_df, user_dfs)\n \n def __len__(self):\n return len(self.user_id)\n \n def __getitem__(self, idx):\n cont_feats = [[0,0, 0,0, 0,0, 0,0, 0,0]]\n sess_feats = [[0,0,0,0]]\n diag_feats = [[0,0,0,0,0,0]]\n strk_feats = [[0,0]]\n \n row_len = len(self.user_id[idx]) \n pad_len = WINDOW_SIZE - row_len\n padding = [PAD] * pad_len\n neg_pad = [-2] * pad_len\n \n return (\n # These are the features we need to build:\n # user_id, content_id, content_type_id, part_id, encountered_id,\n # prior_question_elapsed_time, prior_question_had_explanation,\n # correctness_id, bundle_id, cont_streaks,\n # continuous, session, diagnostic, padding_mask, task_container_id\n self.user_id[idx] + padding,\n np.array(self.content_id[idx] + padding).astype(int),\n np.array(self.content_type_id[idx] + padding).astype(int),\n np.array(self.part_id[idx] + padding).astype(int),\n np.array(self.encountered[idx] + padding).astype(int),\n np.array(self.prior_question_elapsed_time[idx] + padding).astype(int),\n np.array(self.prior_question_had_explanation[idx] + padding).astype(int),\n \n # TODO Check to see if we append something to this??? might be empty\n # NOTE: We don't get the actual incorrect rank until after our prediction\n # So at this point, incorrect rank will equal the previous Q's rank\n # We pad with a 1?? TODO Figure this out!!!!\n # I think we always want to use whatever data we have - but whatever data we have will\n # always be an item short. so the first item should be a [1] then we add...\n # Q. how do we build this feature? currently, we pretend its already there and dont\n # store it in our feature creation process.................. so we don't have this\n # data.\n np.array([1] + self.incorrect_rank[:-1] + padding).astype(int),\n \n np.array(self.bundle_id[idx] + padding).astype(int),\n\n np.vstack((\n # cont_streaks:\n self.answer_ratio1 + padding,\n self.answer_ratio2 + padding,\n self.correct_streak_u + padding,\n self.incorrect_streak_u + padding,\n self.correct_streak_alltime_u + padding,\n self.incorrect_streak_alltime_u + padding,\n \n # diagnostic:\n self.diagnostic_u_recency_1 + padding,\n self.diagnostic_u_recency_2 + padding,\n self.diagnostic_u_recency_3 + padding,\n self.diagnostic_u_recency_4 + padding,\n self.diagnostic_u_recency_5 + padding,\n self.diagnostic_u_recency_6 + padding,\n\n # session:\n self.session_content_num_u + padding,\n self.session_duration_u + padding,\n self.session_ans_duration_u + padding,\n self.lifetime_ans_duration_u + padding,\n \n # continuous\n self.lag_ts_u_recency + padding,\n self.last_ts_u_recency1 + padding,\n self.last_ts_u_recency2 + padding,\n self.last_ts_u_recency3 + padding,\n self.last_correct_ts_u_recency + padding,\n self.last_incorrect_ts_u_recency + padding,\n self.correctness_u_recency + padding,\n self.part_correctness_u_recency + padding,\n self.session_correctness_u + padding,\n )).astype(np.float32).T,\n \n # padding mask:\n np.array([0]*row_len + [1]*pad_len).astype(np.uint8),\n\n # task container id:\n np.array(self.task_container_id[idx] + padding).astype(int),\n )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def collate_fn(batch):\n (\n user_id, content_id, content_type_id, part_id, encountered_id,\n prior_question_elapsed_time, prior_question_had_explanation,\n correctness_id, bundle_id, fe_stuff, padding_mask, task_container_id\n ) = zip(*batch)\n \n user_id = torch.LongTensor(user_id)\n content_id = torch.LongTensor(content_id)\n content_type_id = torch.LongTensor(content_type_id)\n part_id = torch.LongTensor(part_id)\n encountered_id = torch.LongTensor(encountered_id)\n prior_question_elapsed_time = torch.LongTensor(prior_question_elapsed_time)\n prior_question_had_explanation = torch.LongTensor(prior_question_had_explanation)\n correctness_id = torch.LongTensor(correctness_id)\n bundle_id = torch.LongTensor(bundle_id)\n \n fe_stuff = torch.FloatTensor(fe_stuff)\n padding_mask = torch.BoolTensor(padding_mask)\n task_container_id = torch.LongTensor(task_container_id)\n \n # remember the order\n return (\n user_id, content_id, content_type_id, part_id, encountered_id,\n prior_question_elapsed_time, prior_question_had_explanation,\n correctness_id, bundle_id, fe_stuff, padding_mask, task_container_id\n )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":false},"cell_type":"code","source":"DEVICE = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\nprint(DEVICE)\n\n# creating the model\nmodel = SaintTransformerModel(\n d_model=128, # minimally\n nhead=4, # leave it\n num_layers=2, # maybe increase it 2-4\n dropout=0.1, # can be increased....\n emb_dropout=0.1,\n initrange=0.02\n)\n\n# model = SiameseNetwork([\n# SaintTransformerModel(\n# d_model=128, # minimally\n# nhead=4, # leave it\n# num_layers=2, # maybe increase it 2-4\n# dropout=0.1, # can be increased....\n# emb_dropout=0.15,\n# initrange=0.02\n# ),\n# SaintTransformerModel(\n# d_model=128, # minimally\n# nhead=4, # leave it\n# num_layers=2, # maybe increase it 2-4\n# dropout=0.15, # can be increased....\n# emb_dropout=0.1,\n# initrange=0.02\n# )\n# ])\n \n# Load up:\n\n# TODO: This can be spruced up for siamese network:\nstate = torch.load(f'../input/uthman-riiid/best-loss-leanmodel_1.pth')\nfor key in list(state.keys()):\n# if 'user_embeddings' in key:\n# state.pop(key)\n# continue\n state[key[len('module.'):]] = state[key]\n del state[key]\nprint(model.load_state_dict(state, strict=False))\n\n\nmodel.to(DEVICE)\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"previous_test_df = None\nfor counta, (test_df, sample_prediction_df) in enumerate(tqdm(iter_test)):\n if counta % 255 == 0:\n # Keep things fresh\n torch.cuda.empty_cache()\n gc.collect()\n \n if previous_test_df is not None:\n # Update answered_correctly and user_answer\n answered_correctly = np.array(eval(test_df[\"prior_group_answers_correct\"].iloc[0]))\n user_answer = np.array(eval(test_df[\"prior_group_responses\"].iloc[0]))\n user_answer[user_answer==-1] = 0 # Patch Lectures\n answered_correctly[answered_correctly==-1] = 1 # Patch Lectures\n previous_test_df['answered_correctly'] = answered_correctly\n previous_test_df['user_answer'] = user_answer\n update_user_feats(previous_test_df, user_features)\n \n # TODO: Update the DATASET with last 128 historical values for this user\n \n \n ############################################################################\n # Step 1) Make updates to the DF per our pre-processing and any joins we need:\n test_df.content_id += 1\n test_df.prior_question_had_explanation = (test_df.prior_question_had_explanation.astype(np.float16).fillna(-2) + 2).astype(np.uint8)\n mask_lectures = (test_df.content_type_id == 1)\n test_df.loc[mask_lectures, 'content_id'] = test_df[mask_lectures].content_id.map(lecture_ids_map)\n test_df.loc[mask_lectures, 'answered_correctly'] = 1\n test_df['part_id'] = test_df.content_id.map(part_ids_map).astype(np.uint8)\n test_df['bundle_id'] = test_df.content_id.map(bundle_id_map).fillna(0).astype(np.uint16)\n \n # TODO: To create incorrect rank, we also need the user_answer.....\n pd.merge(test_df, question_incorrect_ranks, how='left', on=['user_answer','content_id'], copy=False)\n test_df.loc[test_df.incorrect_rank.isna(), 'incorrect_rank'] = 2 # = \"correct\", for fillna\n test_df.incorrect_rank = test_df.incorrect_rank.astype(np.uint8)\n \n \n test_df['prior_question_elapsed_time_cont'] = test_df.prior_question_elapsed_time\n test_df.prior_question_elapsed_time //= 1000\n test_df.loc[test_df.prior_question_elapsed_time.isna(), 'prior_question_elapsed_time'] = -2\n test_df.prior_question_elapsed_time += 2\n # We are NOT mapping!!! We created a df that did NOT map since we dont embed users\n #df.user_id = df.user_id.map(embedded_user_ids_map).astype(np.uint32)\n \n # Step 2) Build out features and cache those that are necessary\n # Cache the df so that we can make updates after prediction\n test_df = add_user_feats_without_update(test_df, user_features)\n previous_test_df = test_df[\n # Minimal features used in `update_user_feats`\n ['user_id', 'content_id', 'task_container_id', 'content_type_id', 'part_id', 'timestamp', 'answered_correctly', 'user_answer']\n ].copy()\n\n # Step 3) Create a data loader! using test_df + user_dfs consisting of last 128 records for this user\n (\n _user_id, _task_container_id, _content_id, _part_id, _prior_question_elapsed_time,\n _prior_question_had_explanation, _incorrect_rank, _content_type_id, _bundle_id,\n _answer_ratio1, _answer_ratio2, _correct_streak_u, _incorrect_streak_u, _correct_streak_alltime_u,\n _incorrect_streak_alltime_u, _session_content_num_u, _session_duration_u, _session_ans_duration_u,\n _lifetime_ans_duration_u, _lag_ts_u_recency, _last_ts_u_recency1, _last_ts_u_recency2, _last_ts_u_recency3,\n _last_correct_ts_u_recency, _last_incorrect_ts_u_recency, _correctness_u_recency,\n _part_correctness_u_recency, _session_correctness_u, _encountered, _diagnostic_u_recency_1,\n _diagnostic_u_recency_2, _diagnostic_u_recency_3, _diagnostic_u_recency_4, _diagnostic_u_recency_5,\n _diagnostic_u_recency_6, _answered_correctly\n ) = update_user_df(test_df, user_dfs)\n \n\n # TODO: On the fly padding in the DataLoader\n \n # TODO: Predict using the model\n # TODO: Store predictions in the right location on test_df\n # And in the order they arrived; not in the order we created the dataset (by user, with this TCID last)\n test_df[TARGET] = model.predict(test_df[FEATS])\n \n set_predict(\n # We only predict questions\n test_df.loc[\n test_df.content_type_id == 0,\n ['row_id', TARGET]\n ]\n )","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# Gym"},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"def save(path, model, optimizer, best_loss, epoch, step_scheduler=None, scheduler=None, lean=False):\n model.eval()\n \n if lean:\n torch.save(model.state_dict(), path)\n return\n\n params = {\n 'model_state_dict': model.state_dict(),\n 'optimizer_state_dict': optimizer.state_dict(),\n 'best_loss': best_loss,\n 'epoch': epoch,\n }\n if scheduler is not None: params['scheduler_state_dict'] = scheduler.state_dict()\n if step_scheduler is not None: params['step_scheduler_state_dict'] = step_scheduler.state_dict()\n torch.save(params, path)\n\ndef load(path, model, optimizer, step_scheduler=None, scheduler=None):\n checkpoint = torch.load(path)\n \n if 'step_scheduler' in checkpoint:\n step_scheduler.load_state_dict(checkpoint['step_scheduler_state_dict'])\n scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n\n model.load_state_dict(checkpoint['model_state_dict'])\n optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n \n return (\n model, optimizer,\n checkpoint['best_loss'], checkpoint['epoch'],\n step_scheduler, scheduler\n )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fin~"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}