{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Version notes\n\nTo download the pickled file you go to Version 8 directly.\n\n- Ver 1: testing if the features can be dumped into a pickle, train is last 50m rows\n- Ver 6 (interactive): just dumping train dictionary without add features to the df \n- Ver 7 (quick-save): update the workflow to be `itertuples()` to be the fastest\n- Ver 9: getting rid of feature columns to save even more memory\n\n## main contribution\n- Un-encapsulating the functions to be more debuggable.\n- Pickling and loading nested dictionaries for fast inference.\n\n\n### TO-DO:\n\n- Add a rolling mean of `time_recency`? (too much for inference?)\n\nReference:\n- https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n- https://www.kaggle.com/ragnar123/riiid-model-lgbm"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport datatable as dt\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom contextlib import contextmanager\nimport psutil\nimport math\nfrom time import time\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pickle\nimport random\nimport os\nimport sys\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def get_memory(num_var=10):\n    for name, size in sorted(((name, sys.getsizeof(value)) for name, value in globals().items()), \n                             key= lambda x: -x[1])[:num_var]:\n        print(color(f\"{name:>30}:\", color=Colors.green), \n              color(f\"{get_size(size):>8}\", color=Colors.magenta))\n\nget_system()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True\nFOLD = 1\nSEED = 802","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_parquet = '../input/cv-strategy-in-the-kaggle-environment/cv1_train.parquet'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\n\n# Read data\nfeatures = ['timestamp', \n           'user_id', \n           'answered_correctly',\n           'content_id', \n           'content_type_id', \n           'prior_question_elapsed_time', \n           'prior_question_had_explanation']\n\ntrain_dtypes = {\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'answered_correctly': 'int8', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n#     'task_container_id': 'int16',\n    #'user_answer': 'int8',\n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\n\nwith timer(\"Loading train\"):\n    train = pd.read_parquet(train_parquet)[features].astype(train_dtypes)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Delete some trianing data to don't have ram problems\nif DEBUG:\n    train = train.iloc[:10_000_000]\n\n# Filter by content_type_id to discard lectures\ntrain = train.loc[train.content_type_id == False].reset_index(drop = True)\n\n# Changing dtype to avoid lightgbm error\ntrain['prior_question_had_explanation'] = \\\ntrain.prior_question_had_explanation.fillna(False).astype('int8')\n\n# Fill prior question elapsed time with the mean\n# prior_question_elapsed_time_mean = \\\n# train['prior_question_elapsed_time'].dropna().mean()\nprior_question_elapsed_time_mean = 13005.081\ntrain['prior_question_elapsed_time']\\\n.fillna(prior_question_elapsed_time_mean, inplace = True)\n\n##### Merge with question dataframe, not needed for feature dump for inference\nquestions_df = pd.read_csv(question_file)\nquestions_df['part'] = questions_df['part'].astype(np.int32)\nquestions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n\n# train = pd.merge(train, questions_df[['question_id', 'part']], \n#                  left_on = 'content_id', right_on = 'question_id', how = 'left')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If it is just for user dict dump, no need to store the features."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops\ndef add_features(row):\n    \n    '''\n    after re-assignment\n    row[0]: 'user_id',\n    row[1]: 'answered_correctly', \n    row[2]: 'content_id', \n    row[3]: 'prior_question_elapsed_time', \n    row[4]: 'prior_question_had_explanation',\n    row[5]: 'timestamp'\n    \n    '''\n   \n    num = row[0] # index\n    row = row[1:]\n    user_id = row[0]\n\n    # Client features assignation\n    # ------------------------------------------------------------------\n#     if answered_correctly_u_count[user_id] != 0:\n#         answered_correctly_u_avg[num] = \\\n#         answered_correctly_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n#         elapsed_time_u_avg[num] = \\\n#         elapsed_time_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n#         explanation_u_avg[num] = \\\n#         explanation_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n#     else:\n#         answered_correctly_u_avg[num] = np.nan\n\n#         elapsed_time_u_avg[num] = np.nan\n\n#         explanation_u_avg[num] = np.nan\n\n#     if len(timestamp_u[user_id]) == 0:\n#         timestamp_u_recency_1[num] = np.nan\n#         timestamp_u_recency_2[num] = np.nan\n#         timestamp_u_recency_3[num] = np.nan\n\n#     elif len(timestamp_u[user_id]) == 1:\n#         timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][0]\n#         timestamp_u_recency_2[num] = np.nan\n#         timestamp_u_recency_3[num] = np.nan\n\n#     elif len(timestamp_u[user_id]) == 2:\n#         timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][1]\n#         timestamp_u_recency_2[num] = row[5] - timestamp_u[user_id][0]\n#         timestamp_u_recency_3[num] = np.nan\n\n#     elif len(timestamp_u[user_id]) == 3:\n#         timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][2]\n#         timestamp_u_recency_2[num] = row[5] - timestamp_u[user_id][1]\n#         timestamp_u_recency_3[num] = row[5] - timestamp_u[user_id][0]\n\n#     if len(timestamp_u_incorrect[user_id]) == 0:\n#         timestamp_u_incorrect_recency[num] = np.nan\n#     else:\n#         timestamp_u_incorrect_recency[num] = \\\n#         row[5] - timestamp_u_incorrect[user_id][0]\n\n#     # ------------------------------------------------------------------\n#     # Question features assignation\n#     if answered_correctly_q_count[row[2]] != 0:\n#         answered_correctly_q_avg[num] = \\\n#         answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n#         elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n#         explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n#     else:\n#         answered_correctly_q_avg[num] = np.nan\n#         elapsed_time_q_avg[num] = np.nan\n#         explanation_q_avg[num] = np.nan\n#     # ------------------------------------------------------------------\n#     # Client Question assignation\n#     answered_correctly_uq_count[num] = answered_correctly_uq[user_id][row[2]]\n    # ------------------------------------------------------------------\n\n    # ------------------------------------------------------------------\n    # Client features updates\n    answered_correctly_u_count[user_id] += 1\n    elapsed_time_u_sum[user_id] += row[3]\n    explanation_u_sum[user_id] += int(row[4])\n\n    if len(timestamp_u[user_id]) == 3:\n        timestamp_u[user_id].pop(0)\n        timestamp_u[user_id].append(row[5])\n    else:\n        timestamp_u[user_id].append(row[5])\n\n    # ------------------------------------------------------------------\n    # Question features updates\n    answered_correctly_q_count[row[2]] += 1\n    elapsed_time_q_sum[row[2]] += row[3]\n    explanation_q_sum[row[2]] += int(row[4])\n    # ------------------------------------------------------------------\n    # Client Question updates\n    answered_correctly_uq[user_id][row[2]] += 1\n\n    # ------------------------------------------------------------------\n    # Flag for training and inference\n    # ------------------------------------------------------------------\n    # Client features updates\n    answered_correctly_u_sum[user_id] += row[1]\n    if row[1] == 0:\n        if len(timestamp_u_incorrect[user_id]) == 1:\n            timestamp_u_incorrect[user_id].pop(0)\n            timestamp_u_incorrect[user_id].append(row[5])\n        else:\n            timestamp_u_incorrect[user_id].append(row[5])\n\n    # ------------------------------------------------------------------\n    # Question features updates\n    answered_correctly_q_sum[row[2]] += row[1]\n    # ------------------------------------------------------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_len = len(train)\n\n# # -----------------------------------------------------------------------\n# # Client features\n# answered_correctly_u_avg = np.zeros(train_len, dtype = np.float32)\n# elapsed_time_u_avg = np.zeros(train_len, dtype = np.float32)\n# explanation_u_avg = np.zeros(train_len, dtype = np.float32)\n# timestamp_u_recency_1 = np.zeros(train_len, dtype = np.float32)\n# timestamp_u_recency_2 = np.zeros(train_len, dtype = np.float32)\n# timestamp_u_recency_3 = np.zeros(train_len, dtype = np.float32)\n# timestamp_u_incorrect_recency = np.zeros(train_len, dtype = np.float32)\n# # -----------------------------------------------------------------------\n# # Question features\n# answered_correctly_q_avg = np.zeros(train_len, dtype = np.float32)\n# elapsed_time_q_avg = np.zeros(train_len, dtype = np.float32)\n# explanation_q_avg = np.zeros(train_len, dtype = np.float32)\n\n# # -----------------------------------------------------------------------\n# # User Question\n# answered_correctly_uq_count = np.zeros(train_len, dtype = np.int32)\n\n# # -----------------------------------------------------------------------","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Client dictionaries, global var to be updated\nanswered_correctly_u_count = defaultdict(int)\nanswered_correctly_u_sum = defaultdict(int)\nelapsed_time_u_sum = defaultdict(int)\nexplanation_u_sum = defaultdict(int)\ntimestamp_u = defaultdict(list)\ntimestamp_u_incorrect = defaultdict(list)\n\n# Question dictionaries, global var to be updated\nanswered_correctly_q_count = defaultdict(int)\nanswered_correctly_q_sum = defaultdict(int)\nelapsed_time_q_sum = defaultdict(int)\nexplanation_q_sum = defaultdict(int)\n\n# Client Question dictionary, if the user has not answer a questions, then the value is a defaultdict(int)\nanswered_correctly_uq = defaultdict(lambda: defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"iters = train[['user_id',\n          'answered_correctly', \n          'content_id', \n          'prior_question_elapsed_time', \n          'prior_question_had_explanation',\n          'timestamp']].itertuples()\ntrain_len = len(train)\n\nwith timer(\"User feature calculation\"):\n    for _row in tqdm(iters, total=train_len):\n        add_features(_row)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dumping features"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_ids = train['user_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for item in answered_correctly_u_sum.items():\n    print(item)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(answered_correctly_u_sum)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dumping regular dicts"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_u_count.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_u_count, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel answered_correctly_u_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_u_sum.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel answered_correctly_u_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('elapsed_time_u_sum.pickle', 'wb') as f:\n    pickle.dump(elapsed_time_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel elapsed_time_u_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('explanation_u_sum.pickle', 'wb') as f:\n    pickle.dump(explanation_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel explanation_u_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_q_count.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_q_count, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel answered_correctly_q_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_q_sum.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel answered_correctly_q_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('elapsed_time_q_sum.pickle', 'wb') as f:\n    pickle.dump(elapsed_time_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel elapsed_time_q_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    \nwith open('explanation_q_sum.pickle', 'wb') as f:\n    pickle.dump(explanation_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel explanation_q_sum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('timestamp_u.pickle', 'wb') as f:\n    pickle.dump(timestamp_u, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel timestamp_u","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('timestamp_u_incorrect.pickle', 'wb') as f:\n    pickle.dump(timestamp_u_incorrect, f, protocol=pickle.HIGHEST_PROTOCOL)\ndel timestamp_u_incorrect","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dumping nested dictionary\n`answered_correctly_uq` is a nested dict using `user_id` as keys to apply as a lambda function, a straightforward application of pickling is not possible. After applied `user_id`, this user's correctly answered questions are the keys to this dict and the values are just 1."},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_uq[926573062] # user_id == 926573062","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The way to dump it is first to convert it to a regular dict as follows."},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_uq_dict = defaultdict(int)\n\nwith tqdm(total=len(user_ids)) as pbar:\n    for num, user in enumerate(user_ids):\n        answered_correctly_uq_dict[user] = answered_correctly_uq[user]\n        if num % 50 == 0:\n            pbar.update(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_memory(num_var=5) # lambda function is so small because it has not been applied","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_uq_dict.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_uq_dict, f, protocol=pickle.HIGHEST_PROTOCOL)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After dumping we will find its file size is relative big."},{"metadata":{"trusted":true},"cell_type":"code","source":"files = find_files('pickle', '../working/')\nprint_file_size(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# answered_correctly_u_count[898778487] # total q for a user\n# answered_correctly_u_sum[898778487] # correct for a user","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_uq_dict[898778487] ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load models\nJust make sure we can do inference, we load the models to do a mock run."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\n# Features to train and predict\nFEATURES = ['prior_question_elapsed_time', \n            'prior_question_had_explanation', \n            'part', \n            'answered_correctly_u_avg', \n            'elapsed_time_u_avg', \n            'explanation_u_avg',\n            'answered_correctly_q_avg', \n            'elapsed_time_q_avg', \n            'explanation_q_avg', \n            'answered_correctly_uq_count', \n            'timestamp_u_recency_1',\n            'timestamp_u_recency_2', \n            'timestamp_u_recency_3', \n            'timestamp_u_incorrect_recency']\n\nmodel_file = '../input/riiid-lgb-models/lgb_loop_fold_0_auc_0.7739.txt'\nmodel = lgb.Booster(model_file=model_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops for test\ndef add_features_test(df):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    \n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    \n    # -----------------------------------------------------------------------\n    \n    for num, row in enumerate(df[['user_id',\n                                  'answered_correctly', \n                                  'content_id', \n                                  'prior_question_elapsed_time', \n                                  'prior_question_had_explanation',\n                                  'timestamp']].values):\n\n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = \\\n            answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n            elapsed_time_u_avg[num] = \\\n            elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n            explanation_u_avg[num] = \\\n            explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n        else:\n            answered_correctly_u_avg[num] = np.nan\n\n            elapsed_time_u_avg[num] = np.nan\n\n            explanation_u_avg[num] = np.nan\n\n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n\n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = \\\n            row[5] - timestamp_u_incorrect[row[0]][0]\n\n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = \\\n            answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n\n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, \n                            'elapsed_time_u_avg': elapsed_time_u_avg, \n                            'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, \n                            'elapsed_time_q_avg': elapsed_time_q_avg, \n                            'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, \n                            'timestamp_u_recency_1': timestamp_u_recency_1, \n                            'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, \n                            'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading a nested dict\nHere loading the nested dict is a bit tricky, we simply re-allocate it into a dict with a default value as a lambda function."},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('../working/answered_correctly_uq_dict.pickle', 'rb') as f:\n    answered_correctly_uq_dict = pickle.load(f)\n    \nanswered_correctly_uq = defaultdict(lambda: defaultdict(int))\nfor key in tqdm(answered_correctly_uq_dict.keys()):\n    answered_correctly_uq[key] = answered_correctly_uq_dict[key]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get feature dicts\nwith open('../working/answered_correctly_u_count.pickle', 'rb') as f:\n    answered_correctly_u_count = pickle.load(f)\n    \nwith open('../working/answered_correctly_u_sum.pickle', 'rb') as f:\n    answered_correctly_u_sum = pickle.load(f)\n\nwith open('../working/elapsed_time_u_sum.pickle', 'rb') as f:\n    elapsed_time_u_sum = pickle.load(f)\n\nwith open('../working/explanation_u_sum.pickle', 'rb') as f:\n    explanation_u_sum = pickle.load(f)    \n\nwith open('../working/answered_correctly_q_count.pickle', 'rb') as f:\n    answered_correctly_q_count = pickle.load(f)  \n    \nwith open('../working/answered_correctly_q_sum.pickle', 'rb') as f:\n    answered_correctly_q_sum = pickle.load(f)  \n    \nwith open('../working/elapsed_time_q_sum.pickle', 'rb') as f:\n    elapsed_time_q_sum = pickle.load(f)  \n\nwith open('../working/explanation_q_sum.pickle', 'rb') as f:\n    explanation_q_sum = pickle.load(f)     \n    \nwith open('../working/timestamp_u.pickle', 'rb') as f:\n    timestamp_u = pickle.load(f)         \n    \nwith open('../working/timestamp_u_incorrect.pickle', 'rb') as f:\n    timestamp_u_incorrect = pickle.load(f)     ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def update_features(df):\n    for row in df[['user_id', \n                   'answered_correctly', \n                   'content_id', \n                   'content_type_id', \n                   'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_features(previous_test_df)\n    previous_test_df = test_df.copy()\n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n    \n    test_df['prior_question_had_explanation'] = \\\n    test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    test_df['prior_question_elapsed_time'].\\\n    fillna(prior_question_elapsed_time_mean, inplace = True)\n    test_df = pd.merge(test_df, questions_df[['question_id', 'part']], \n                       left_on = 'content_id', \n                       right_on = 'question_id', \n                       how = 'left')\n    test_df[TARGET] = 0.67\n    \n    test_df = add_features_test(test_df)\n    \n    test_df[TARGET] =  model.predict(test_df[FEATURES])\n    set_predict(test_df[['row_id', TARGET]])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../working/submission.csv')\nsub['answered_correctly'].hist(bins=15);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`answered_correctly_uq` is now a dict with a lambda function as default value again."},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_uq[705741139.0][7922.0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## debug\n# previous_test_df = None\n# test_df, sample_prediction_df = next(iter_test)\n# if previous_test_df is not None:\n#     previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n#     update_features(previous_test_df)\n# previous_test_df = test_df.copy()\n\n# test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n\n# test_df['prior_question_had_explanation'] = \\\n# test_df.prior_question_had_explanation.fillna(False).astype('int8')\n\n# test_df['prior_question_elapsed_time'].\\\n# fillna(prior_question_elapsed_time_mean, inplace = True)\n# test_df = pd.merge(test_df, questions_df[['question_id', 'part']], \n#                    left_on = 'content_id', \n#                    right_on = 'question_id', \n#                    how = 'left')\n# test_df[TARGET] = 0.66\n\n# test_df = add_features_test(test_df)\n\n# test_df[TARGET] =  model.predict(test_df[FEATURES])\n# set_predict(test_df[['row_id', TARGET]])\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}