{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Ignite Starter\nSimple starter notebook, which uses for prediction the [Ignite](https://github.com/pytorch/ignite) library."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nimport datatable as dt\n\nimport statsmodels.api as sm\nfrom sklearn.metrics import roc_auc_score\n\nfrom matplotlib import pyplot as plt\nimport riiideducation\nfrom pathlib import Path\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss, RunningAverage, ConfusionMatrix\nfrom ignite.handlers import ModelCheckpoint, EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input')\nassert path.exists()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndata_types_dict = {\n    'user_id': 'int32', \n    'content_id': 'int16', \n    'answered_correctly': 'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\ntarget = 'answered_correctly'\ntrain_df = dt.fread(path/\"riidtrainjay/train.jay\", columns=set(data_types_dict.keys())).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df['task_container_id']\ndel train_df['content_type_id']\ndel train_df['row_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = train_df[train_df[target] != -1].reset_index(drop=True)\ntrain_df.drop(columns=['timestamp'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature generation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_had_explanation'].fillna(False, inplace=True)\ntrain_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['lag'] = train_df.groupby('user_id')[target].shift()\ntrain_df['lag'].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ncum = train_df.groupby(['user_id'])['lag'].agg(['cumsum', 'cumcount'])\ntrain_df['user_correctness'] = cum['cumsum'] / cum['cumcount']\ntrain_df.drop(columns=['lag'], inplace=True)\ndel cum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])\ncontent_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['prior_question_elapsed_time']:\n    train_df[f] = pd.to_numeric(train_df[f], downcast='float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df['residual'] =  train_df[target] - train_df['content_id'].map(content_agg['sum'] / content_agg['count'])\nresidual_agg = train_df.groupby('user_id')['residual'].agg(['sum'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_agg = train_df.groupby('user_id').agg({'prior_question_elapsed_time': ['sum', lambda x: len(x)]})\nprior_question_elapsed_time_agg.columns = ['sum', 'count']\nprior_question_elapsed_time_agg['count'] = prior_question_elapsed_time_agg['count'].astype('int32')\nprior_question_elapsed_time_agg.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"USER_TRIES = 70\n\nimport math\nVALID_TRIES = math.ceil(USER_TRIES / 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.groupby('user_id').tail(USER_TRIES).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Question related"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_types_dict = {'question_id': 'int16', 'part': 'int8', 'bundle_id': 'int16', 'tags': 'string'}\n\nquestions_df = pd.read_csv(\n    path/'riiid-test-answer-prediction/questions.csv', \n    usecols=data_types_dict.keys(),\n    dtype=data_types_dict\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_tags_combos_keys = {v:i for i,v in enumerate(questions_df['tags'].unique())}\nquestions_df['tags_encoded'] = questions_df['tags'].apply(lambda x : unique_tags_combos_keys[x])\nquestions_df['tags_encoded'] = pd.to_numeric(questions_df['tags_encoded'], downcast='integer')\nquestions_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_tag_factory(tag_pos):\n    def extract_tag(x):\n        if isinstance(x, str) and tag_pos < len(x.split()):\n            splits = x.split()\n            splits.sort()\n            return int(splits[tag_pos])\n        else:\n            return 255\n    return extract_tag\n        \nfor i in range(0, 3):\n    questions_df[f'tag_{i + 1}'] = questions_df['tags'].apply(extract_tag_factory(i))\n    questions_df[f'tag_{i + 1}'] = questions_df[f'tag_{i + 1}'].astype('uint8')\n    unique_tag_keys = {v:i for i,v in enumerate(questions_df[f'tag_{i + 1}'].unique())}\n    questions_df[f'tag_{i + 1}'] = questions_df[f'tag_{i + 1}'].apply(lambda x : unique_tag_keys[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')\ntrain_df.drop(columns=['question_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['content_count'] = train_df['content_id'].map(content_agg['count']).astype('int32')\ntrain_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_elapsed_time_mean'] = train_df['user_id'].map(prior_question_elapsed_time_agg['sum'] / prior_question_elapsed_time_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['residual_user_mean'] = train_df['user_id'].map(residual_agg['sum'] / user_agg['count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean(), inplace=True)\ntrain_df['user_correctness'].fillna(train_df['user_correctness'].mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in ['user_correctness', 'content_id']:\n    train_df[f] = pd.to_numeric(train_df[f], downcast='float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = train_df.groupby('user_id').tail(VALID_TRIES)\ntrain_df.drop(valid_df.index, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['user_correctness'] = train_df['user_correctness'].replace(train_df['user_correctness'].mean(), 0.0)\nvalid_df['user_correctness'] = valid_df['user_correctness'].replace(valid_df['user_correctness'].mean(), 0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    'user_correctness',\n    'part',\n    'content_id',\n    'content_count',\n    'tags_encoded',\n    'tag_1',\n    'tag_2',\n    'prior_question_elapsed_time_mean',\n    'residual_user_mean'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Ignite"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn_pandas import DataFrameMapper\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom pandas.api.types import is_string_dtype, is_numeric_dtype","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features = ['part', 'tags_encoded', 'tag_1', 'tag_2']\ncont_features = [x for x in features if x not in cat_features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Riiid(torch.utils.data.Dataset):\n    \n    def __init__(self, df, cat_fields, cont_fields, target):\n        df_cat = df[cat_fields]\n        df_cont = df[cont_fields]\n        \n        cats = [c.values for _, c in df_cat.items()]\n        conts = [c.values for _, c in df_cont.items()]\n        \n        n = len(cats[0])\n        self.cats = np.stack(cats, 1).astype(np.int64)\n        self.conts = np.stack(conts, 1).astype(np.float32)\n        self.y = df[target].values.astype(np.float32) if target is not None else np.zeros((n,1))\n        \n    def __len__(self): return len(self.y)\n    \n    def __getitem__(self, idx):\n        return [self.cats[idx], self.conts[idx], self.y[idx]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale_vars(df, mapper, cols):\n    if mapper is None:\n        map_f = [([n],StandardScaler()) for n in df.columns if\n                 is_numeric_dtype(df[n]) and n in cols]\n        mapper = DataFrameMapper(map_f).fit(df)\n    df[mapper.transformed_names_] = mapper.transform(df)\n    return mapper","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nmapper = scale_vars(train_df, None, cont_features)\nmapper.transform(train_df)\nmapper = scale_vars(valid_df, None, cont_features)\nmapper.transform(valid_df).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = Riiid(train_df, cat_features, cont_features, target)\nvalid_ds = Riiid(valid_df, cat_features, cont_features, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_WORKERS = 6\nBATCH_SIZE = 8192 * 16\nEPOCHS = 5\nMODEL_PATH = \"riid-output\"\ntrain_dl = DataLoader(train_ds, BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)\nvalid_dl = DataLoader(valid_ds, BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for v in cat_features: \n    train_df[v] = train_df[v].astype('category').cat.as_ordered()\ncat_sz = [(c, len(train_df[c].cat.categories)+1) for c in cat_features]\nembed_sizes = [(c, min(50, (c+1)//2)) for _,c in cat_sz]\nembed_sizes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RiiidModel(nn.Module):\n    def __init__(self, embed_sizes, n_cont, emb_drop, out_sz, sizes, \n                 drops, use_bn=False):\n        super().__init__()\n        for i,(c,s) in enumerate(embed_sizes): \n            assert c > 1, f\"cardinality must be >=2, got embed_sizes[{i}]: ({c},{s})\"\n        self.embs = nn.ModuleList([nn.Embedding(c, s) \n                                      for c,s in embed_sizes])\n        for emb in self.embs: \n            self.emb_init(emb)\n        n_emb = sum(e.embedding_dim for e in self.embs)\n        self.n_emb, self.n_cont = n_emb, n_cont\n        sizes = [n_emb + n_cont] + sizes\n        self.linears = nn.ModuleList([\n            nn.Linear(sizes[i], sizes[i+1]) for i in range(len(sizes)-1)])\n        self.batch_norms = nn.ModuleList([\n            nn.BatchNorm1d(sz) for sz in sizes[1:]])\n        for o in self.linears: \n            nn.init.kaiming_normal_(o.weight.data)\n        self.outp = nn.Linear(sizes[-1], out_sz)\n        nn.init.kaiming_normal_(self.outp.weight.data)\n        self.emb_drop = nn.Dropout(emb_drop)\n        self.drops = nn.ModuleList([nn.Dropout(drop) \n                                        for drop in drops])\n        self.bn = nn.BatchNorm1d(n_cont)\n        self.use_bn = use_bn\n        \n    def forward(self, x_cat, x_cont):\n        if self.n_emb > 0:\n            x = [e(x_cat[:,i]) for i,e in enumerate(self.embs)]\n            x = torch.cat(x, 1)\n            x = self.emb_drop(x)\n        if self.n_cont > 0:\n            x2 = self.bn(x_cont)\n            x = torch.cat([x, x2], 1) if self.n_emb != 0 else x2\n        for l,d,b in zip(self.linears, self.drops, self.batch_norms):\n            x = F.relu(l(x))\n            if self.use_bn: \n                x = b(x)\n            x = d(x)\n        x = self.outp(x)\n        x = torch.sigmoid(x)\n        return x\n        \n        \n    def emb_init(self, x):\n        x = x.weight.data\n        sc = 2 / (x.size(1)+1)\n        x.uniform_(-sc,sc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    riiid_model = RiiidModel(embed_sizes, len(cont_features), emb_drop = 0.04, out_sz = 1,\n          sizes = [200, 100], drops = [0.001,0.01], use_bn=True)\n    riiid_model.to(device)\n    return riiid_model\n\nriid_model = create_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss\nfrom ignite.contrib.metrics.roc_auc import ROC_AUC\nfrom tqdm.notebook import tqdm\n\nLR = 0.006\n\ncriterion = nn.BCELoss()\n\noptimizer = torch.optim.Adam(riid_model.parameters(), LR, weight_decay=0.01)\n\n# Decay LR by a factor of 0.2 every 1 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.2)\n\nval_metrics = {\n    \"auc\": ROC_AUC(),\n    \"loss\": Loss(criterion)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_batch(batch, device):\n    x_cat, x_cont, t = batch\n    return x_cat.to(device), x_cont.to(device), t.to(device)\n\ndef train_step(trainer, batch):\n    riid_model.train()\n    optimizer.zero_grad()\n    x_cat, x_cont, y = prepare_batch(batch, device=device)\n    y_pred = riid_model(x_cat, x_cont)\n    loss = criterion(y_pred, y.unsqueeze(1))\n    loss.backward()\n    optimizer.step()\n    return loss.item()\n\ndef predict_on_batch(engine, batch):\n    riid_model.eval()\n    with torch.no_grad():\n        x_cat, x_cont, y = prepare_batch(batch, device=device)\n        y_pred = riid_model(x_cat, x_cont)\n\n    return y_pred, y.unsqueeze(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ignite.engine.engine import Engine\nfrom ignite.handlers import ModelCheckpoint, global_step_from_engine\n\ntrainer = Engine(train_step)\nevaluator = Engine(predict_on_batch)\n\n# Checkpoint to store n_saved best models wrt score function\nmodel_checkpoint = ModelCheckpoint(\n    MODEL_PATH,\n    n_saved=1,\n    filename_prefix=\"best\",\n    score_function=lambda engine : engine.state.metrics[\"auc\"],\n    score_name=\"auc\",\n    global_step_transform=global_step_from_engine(trainer),\n)\n\nfor name, metric in val_metrics.items():\n    metric.attach(evaluator, name)\n    \n# Save the model (if relevant) every epoch completed of evaluator\nevaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {\"model\": riid_model});","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndesc = \"ITERATION - loss: {:.5f}\"\npbar = tqdm(initial=0, leave=False, total=len(train_dl), desc=desc.format(0))\nlog_interval = 5\n\n@trainer.on(Events.ITERATION_COMPLETED(every=log_interval))\ndef log_training_loss(engine):\n    pbar.desc = desc.format(engine.state.output)\n    pbar.update(log_interval)\n\n# Un-coment this if you want to have training evaluation. This turned out to be a bit slow\n# @trainer.on(Events.EPOCH_COMPLETED)\n# def log_training_results(engine):\n#     exp_lr_scheduler.step()\n#     tqdm.write(f\"Optimizer learning rate: {optimizer.param_groups[0]['lr']}\")\n#     pbar.refresh()\n#     evaluator.run(train_dl)\n#     metrics = evaluator.state.metrics\n#     auc = metrics[\"auc\"]\n#     loss = metrics[\"loss\"]\n#     tqdm.write(\n#         f\"Training Results - Epoch: {engine.state.epoch}  AUC: {auc:.5f} Loss: {loss:.5f}\"\n#     )\n\n@trainer.on(Events.EPOCH_COMPLETED)\ndef log_validation_results(engine):\n    evaluator.run(valid_dl)\n    metrics = evaluator.state.metrics\n    auc = metrics[\"auc\"]\n    loss = metrics[\"loss\"]\n    tqdm.write(\n        \"Validation Results - Epoch: {}  AUC: {:.5f} Loss: {:.5f}\".format(\n            engine.state.epoch, auc, loss\n        )\n    )\n    pbar.n = pbar.last_print_n = 0\n\n@trainer.on(Events.EPOCH_COMPLETED | Events.COMPLETED)\ndef log_time(engine):\n    tqdm.write(\n        \"{} took {} seconds\".format(trainer.last_event_name.name, trainer.state.times[trainer.last_event_name.name])\n    )\n    \ntrainer.run(train_dl, max_epochs=EPOCHS)\npbar.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(df):\n    mapper = scale_vars(df, None, cont_features)\n    mapper.transform(df)\n    ds = Riiid(df, cat_features, cont_features, None)\n    dl = DataLoader(ds, len(df), shuffle=False, num_workers=1)\n    return predict_on_batch(None, next(iter(dl)))[0].squeeze().cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load best model from disk\n\nimport os\n\nriid_model = create_model()\nriid_model.load_state_dict(torch.load(Path(MODEL_PATH)/os.listdir(MODEL_PATH)[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))\nresidual_sum_dict = residual_agg['sum'].astype('float32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time_sum_dict = prior_question_elapsed_time_agg['sum'].astype('int32').to_dict(defaultdict(int))\nprior_question_elapsed_time_count_dict = prior_question_elapsed_time_agg['count'].astype('int32').to_dict(defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clip(count): return np.clip(count, 1e-8, np.inf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        prior_question_elapsed_times = prior_test_df['prior_question_elapsed_time'].values\n        targets = prior_test_df[target].values\n        \n        for user_id, content_id, prior_question_elapsed_time, answered_correctly in zip(user_ids, content_ids, prior_question_elapsed_times, targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1\n            content_sum_dict[content_id] += answered_correctly\n            content_count_dict[content_id] += 1\n            mean_accuracy = content_sum_dict[content_id] / clip(content_count_dict[content_id])\n            residual_sum_dict[user_id] += answered_correctly - mean_accuracy\n            \n            prior_question_elapsed_time_sum_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time) else prior_question_elapsed_time\n            prior_question_elapsed_time_count_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time) else 1\n    \n    prior_test_df = test_df.copy()\n    \n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    \n    test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False).astype('uint8')\n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    res_sum = np.zeros(len(test_df), dtype=np.float32)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    prior_question_elapsed_time_sum = np.zeros(len(test_df), dtype=np.int32)\n    prior_question_elapsed_time_count = np.zeros(len(test_df), dtype=np.int32)\n    \n    for i, (user_id, content_id) in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)):\n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        res_sum[i] = residual_sum_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        prior_question_elapsed_time_sum[i] = prior_question_elapsed_time_sum_dict[user_id]\n        prior_question_elapsed_time_count[i] = prior_question_elapsed_time_count_dict[user_id]\n\n    content_count = clip(content_count)\n    user_count = clip(user_count)\n    prior_question_elapsed_time_count = clip(prior_question_elapsed_time_count)\n    test_df['user_correctness'] = user_sum / user_count\n    test_df['residual_user_mean'] = res_sum / user_count\n    test_df['content_count'] = content_count\n    test_df['content_id'] = content_sum / content_count\n    test_df['prior_question_elapsed_time_mean'] = prior_question_elapsed_time_sum / prior_question_elapsed_time_count\n    \n    test_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean(), inplace=True)\n    \n    test_df.fillna(0, inplace=True)\n    test_df[cat_features] = test_df[cat_features].apply(pd.to_numeric, downcast='integer')\n    test_df[target] = predict(test_df)\n    \n    env.predict(test_df[['row_id', target]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}