{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:51:45.965251Z","iopub.status.busy":"2021-01-07T16:51:45.964535Z","iopub.status.idle":"2021-01-07T16:51:53.042409Z","shell.execute_reply":"2021-01-07T16:51:53.041878Z"},"papermill":{"duration":7.10312,"end_time":"2021-01-07T16:51:53.042517","exception":false,"start_time":"2021-01-07T16:51:45.939397","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from tensorflow.python.client import device_lib\ndevice_lib.list_local_devices()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-01-07T16:51:53.085598Z","iopub.status.busy":"2021-01-07T16:51:53.084962Z","iopub.status.idle":"2021-01-07T16:51:53.105691Z","shell.execute_reply":"2021-01-07T16:51:53.105173Z"},"papermill":{"duration":0.044438,"end_time":"2021-01-07T16:51:53.105785","exception":false,"start_time":"2021-01-07T16:51:53.061347","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:51:53.152184Z","iopub.status.busy":"2021-01-07T16:51:53.151549Z","iopub.status.idle":"2021-01-07T16:52:29.754327Z","shell.execute_reply":"2021-01-07T16:52:29.752976Z"},"papermill":{"duration":36.62888,"end_time":"2021-01-07T16:52:29.754481","exception":false,"start_time":"2021-01-07T16:51:53.125601","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures_df = pd.read_csv('../input/nadare-processed-2/lectures.csv')\nquestions_df = pd.read_csv('../input/nadare-processed-2/questions.csv')\nchoices, timestamps, elapsed_times, orders, explanations, user_id_map = pd.read_pickle(\"../input/nadare-processed-2/raw_items.pickle\")\ntimestamp_bins, elapsedtime_bins = pd.read_pickle(\"../input/nadare-processed-2/bins.pickle\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:29.826882Z","iopub.status.busy":"2021-01-07T16:52:29.826134Z","iopub.status.idle":"2021-01-07T16:52:29.853902Z","shell.execute_reply":"2021-01-07T16:52:29.853172Z"},"papermill":{"duration":0.069072,"end_time":"2021-01-07T16:52:29.854073","exception":false,"start_time":"2021-01-07T16:52:29.785001","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"timestamp_bins = tf.constant(timestamp_bins, dtype=\"float32\")\nelapsedtime_bins = tf.constant(elapsedtime_bins, dtype=\"float32\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:29.915675Z","iopub.status.busy":"2021-01-07T16:52:29.914438Z","iopub.status.idle":"2021-01-07T16:52:29.922521Z","shell.execute_reply":"2021-01-07T16:52:29.92327Z"},"papermill":{"duration":0.042343,"end_time":"2021-01-07T16:52:29.923469","exception":false,"start_time":"2021-01-07T16:52:29.881126","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"delta_bins =pd.read_pickle(\"../input/nadare-processed-2/delta_bins.pickle\")\ndelta_bins[-2:] = 1e18","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:29.9888Z","iopub.status.busy":"2021-01-07T16:52:29.98792Z","iopub.status.idle":"2021-01-07T16:52:30.786549Z","shell.execute_reply":"2021-01-07T16:52:30.785353Z"},"papermill":{"duration":0.832522,"end_time":"2021-01-07T16:52:30.786677","exception":false,"start_time":"2021-01-07T16:52:29.954155","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qp_original_sum = pd.read_pickle(\"../input/nadare-processed-2/question_part_agg.pickle\").sum()\n\npart_mean = tf.constant([qp_original_sum[f\"part{i}_sum\"] / qp_original_sum[f\"part{i}_count\"] for i in range(7)], dtype=\"float32\")\npart_mean = tf.concat([part_mean, [qp_original_sum[f\"question_sum\"] / qp_original_sum[f\"question_count\"]]], axis=0)\npart_rate = tf.constant([qp_original_sum[f\"part{i}_count\"] / qp_original_sum[\"question_count\"] for i in range(7)], dtype=\"float32\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:30.829793Z","iopub.status.busy":"2021-01-07T16:52:30.828826Z","iopub.status.idle":"2021-01-07T16:52:31.052568Z","shell.execute_reply":"2021-01-07T16:52:31.053457Z"},"papermill":{"duration":0.247962,"end_time":"2021-01-07T16:52:31.053669","exception":false,"start_time":"2021-01-07T16:52:30.805707","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qp_agg_df = pd.read_pickle(\"../input/nadare-processed-2/question_part_agg.pickle\")\nqp_agg_df = qp_agg_df.set_index(\"user_ix\").astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:31.112869Z","iopub.status.busy":"2021-01-07T16:52:31.112014Z","iopub.status.idle":"2021-01-07T16:52:31.137354Z","shell.execute_reply":"2021-01-07T16:52:31.141457Z"},"papermill":{"duration":0.060753,"end_time":"2021-01-07T16:52:31.14163","exception":false,"start_time":"2021-01-07T16:52:31.080877","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:31.211385Z","iopub.status.busy":"2021-01-07T16:52:31.208641Z","iopub.status.idle":"2021-01-07T16:52:31.216809Z","shell.execute_reply":"2021-01-07T16:52:31.217454Z"},"papermill":{"duration":0.044247,"end_time":"2021-01-07T16:52:31.217629","exception":false,"start_time":"2021-01-07T16:52:31.173382","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures_df[\"type_of\"] = lectures_df[\"type_of\"].map({k: v for v, k in enumerate(lectures_df[\"type_of\"].unique())})","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:31.281183Z","iopub.status.busy":"2021-01-07T16:52:31.280326Z","iopub.status.idle":"2021-01-07T16:52:44.72977Z","shell.execute_reply":"2021-01-07T16:52:44.728663Z"},"papermill":{"duration":13.485412,"end_time":"2021-01-07T16:52:44.729892","exception":false,"start_time":"2021-01-07T16:52:31.24448","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"choice_is_correct = [False] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_is_wrong = [False] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_is_question = [False] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_accuracy_mean = [questions_df[\"content_4_accuracy\"].mean()] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_accuracy2_mean = [questions_df[\"content_2_accuracy\"].mean()] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_elapsed_time_mean = [questions_df[\"content_elapsed_time_mean\"].mean()] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_correct_elapssed_time_mean = [questions_df[\"correct_elapsed_time_mean\"].mean()] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_correct_et_rate = [1.] * ((lectures_df[\"content_id\"].max()+1) * 4)\nanswer_elapsed_time_std = [questions_df[\"content_elapsed_time_std\"].mean()] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_rate = [[0]*4 for _ in range((lectures_df[\"content_id\"].max()+1) * 4)] \ncorrect_choice = [-1] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_parts = [-1] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_typeof = [-1] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_tags = [[] for _ in range((lectures_df[\"content_id\"].max()+1) * 4)]\n\nfor i, row in questions_df.iterrows():\n    tags = [] if pd.isna(row[\"tags\"]) else list(map(int, row[\"tags\"].split()))\n    for i in range(4):\n        choice_is_correct[i + row[\"content_id\"]*4] = (i == row[\"correct_answer\"])\n        choice_is_wrong[i + row[\"content_id\"]*4] = (i != row[\"correct_answer\"])\n        choice_is_question[i + row[\"content_id\"]*4] = True\n        answer_accuracy_mean[i + row[\"content_id\"]*4] = row[\"content_4_accuracy\"]\n        answer_accuracy2_mean[i + row[\"content_id\"]*4] = row[\"content_2_accuracy\"]\n        answer_elapsed_time_mean[i + row[\"content_id\"]*4] = row[\"content_elapsed_time_mean\"]\n        answer_correct_elapssed_time_mean[i + row[\"content_id\"]*4] = row[\"correct_elapsed_time_mean\"]\n        answer_correct_et_rate[i + row[\"content_id\"]*4] = row[\"correct_elapsed_time_mean\"] / row[\"content_elapsed_time_mean\"]\n        answer_elapsed_time_std[i + row[\"content_id\"]*4] = row[\"content_elapsed_time_std\"]\n        for j in range(4):\n            choice_rate[i + row[\"content_id\"]*4][j] = row[f\"choice_rate_{j}\"]\n        correct_choice[i + row[\"content_id\"]*4] = row[\"correct_choice\"]\n        choice_tags[i + row[\"content_id\"]*4] = [t for t in tags]\n        choice_parts[i + row[\"content_id\"]*4] = int(row[\"part\"])\n\nfor i, row in lectures_df.iterrows():\n    tags = [row[\"tag\"]]\n    for i in range(4):\n        cid = int(row[\"content_id\"])\n        choice_tags[i + cid*4] = [t for t in tags]\n        choice_parts[i + cid*4] = int(row[\"part\"])\n        choice_typeof[i + cid*4] = int(row[\"type_of\"])\n        \nchoice_is_correct = tf.constant(choice_is_correct)\nchoice_is_wrong = tf.constant(choice_is_wrong)\nchoice_is_question = tf.constant(choice_is_question)\nanswer_accuracy_mean = tf.constant(answer_accuracy_mean)\nanswer_accuracy2_mean = tf.constant(answer_accuracy2_mean)\nanswer_elapsed_time_mean = tf.constant(answer_elapsed_time_mean)\nanswer_elapsed_time_std = tf.constant(answer_elapsed_time_std)\ncorrect_choice = tf.constant(correct_choice, dtype=\"int32\")\nchoice_rate = tf.constant(choice_rate, dtype=\"float32\")\nchoice_parts = tf.constant(choice_parts, dtype=\"int32\")\nchoice_typeof = tf.constant(choice_typeof, dtype=\"int32\")\nchoice_tags = tf.keras.preprocessing.sequence.pad_sequences(choice_tags, dtype=\"int16\", value=-1, padding=\"post\") + 1\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:44.780129Z","iopub.status.busy":"2021-01-07T16:52:44.779211Z","iopub.status.idle":"2021-01-07T16:52:44.782166Z","shell.execute_reply":"2021-01-07T16:52:44.781485Z"},"papermill":{"duration":0.032774,"end_time":"2021-01-07T16:52:44.782265","exception":false,"start_time":"2021-01-07T16:52:44.749491","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"question_id_map = {id_: i+1 for i, id_ in enumerate(questions_df[\"question_id\"])}\nlecture_id_map = {id_: i+questions_df.shape[0]+1 for i, id_ in enumerate(lectures_df[\"lecture_id\"])}","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:44.825694Z","iopub.status.busy":"2021-01-07T16:52:44.824356Z","iopub.status.idle":"2021-01-07T16:52:45.057303Z","shell.execute_reply":"2021-01-07T16:52:45.056773Z"},"papermill":{"duration":0.256581,"end_time":"2021-01-07T16:52:45.05747","exception":false,"start_time":"2021-01-07T16:52:44.800889","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nclass BaseModel(tf.keras.Model):\n    def __init__(self, user_in_train,\n                 content_embedding_size, choice_embedding_size,\n                 choice_is_correct, choice_is_wrong, choice_is_question,\n                 choice_tags, choice_parts, choice_typeof,\n                 answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,\n                 correct_choice, choice_rate,\n                 sample_window=500):\n        super(BaseModel, self).__init__()\n        \n        #self.user_lookup = tf.keras.layers.experimental.preprocessing.IntegerLookup(vocabulary=user_in_train, mask_value=None, oov_value=0)\n        #self.user_embedding = tf.keras.layers.Embedding(self.user_lookup.vocab_size()+2, 64)\n        self.content_embedding = tf.keras.layers.Embedding(content_embedding_size+2, 64, mask_zero=True)\n        self.choice_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 64, mask_zero=True)\n        \n        \n        self.choice_tags_vec = tf.keras.layers.Embedding(choice_tags.shape[0],\n                                                         choice_tags.shape[1],\n                                                         weights=[choice_tags],\n                                                         embeddings_initializer = None,\n                                                         dtype=\"int16\",\n                                                         trainable=False)\n        self.tag_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 16, mask_zero=True)\n        \n        self.timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)\n        self.lag_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)\n        self.delta_timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)\n        self.cooltime_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)\n        self.elapsed_embedding = tf.keras.layers.Embedding(100, 32, mask_zero=True)\n        self.positional_embedding = tf.keras.layers.Embedding(500, 32, mask_zero=True)\n        \n        self.choice_is_correct = choice_is_correct\n        self.choice_is_wrong = choice_is_wrong\n        self.choice_is_question = choice_is_question\n        self.choice_tags = choice_tags\n        self.choice_parts = choice_parts\n        self.choice_typeof = choice_typeof\n        self.answer_accuracy_mean = answer_accuracy_mean\n        self.answer_accuracy2_mean = answer_accuracy2_mean\n        self.answer_elapsed_time_mean = answer_elapsed_time_mean\n        self.answer_elapsed_time_std = answer_elapsed_time_std\n        self.answer_correct_elapssed_time_mean = answer_correct_elapssed_time_mean\n        self.correct_choice = correct_choice\n        self.choice_rate = choice_rate\n        _ = self.content_embedding(0)\n        _ = self.choice_embedding(0)\n        \n        # self.attention = tf.keras.layers.Attention()\n        self.order_in = tf.keras.layers.InputLayer(input_shape=(1,))\n        self.feat_in = tf.keras.layers.InputLayer(input_shape=(76,))\n        self.oh_in = tf.keras.layers.InputLayer(input_shape=(7,))\n        \n        self.table_bn_clipper_qc = tf.keras.Sequential([tf.keras.layers.BatchNormalization(), tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5))])\n        self.bn_clipper_history = tf.keras.Sequential([tf.keras.layers.BatchNormalization(), tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5))])\n        self.bn_clipper_0 = tf.keras.Sequential([tf.keras.layers.BatchNormalization(), tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5))])\n        self.bn_clipper_1 = tf.keras.Sequential([tf.keras.layers.BatchNormalization(), tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5))])\n        self.bn_clipper_2 = tf.keras.Sequential([tf.keras.layers.BatchNormalization(), tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5))])\n        \n        self.dense_qc_u0 = tf.keras.layers.Dense(64)\n        self.dense_qc_v0 = tf.keras.layers.Dense(305)\n        self.bn_qc0 = tf.keras.layers.BatchNormalization()\n        self.dense_qc_u1 = tf.keras.layers.Dense(64)\n        self.dense_qc_v1 = tf.keras.layers.Dense(305)\n        self.bn_qc1 = tf.keras.layers.BatchNormalization()\n        \n        self.dense_kv_u0 = tf.keras.layers.Dense(64)\n        self.dense_kv_v0 = tf.keras.layers.Dense(312)\n        self.bn_kv0 = tf.keras.layers.BatchNormalization()\n        self.dense_kv_u1 = tf.keras.layers.Dense(64)\n        self.dense_kv_v1 = tf.keras.layers.Dense(312)\n        self.bn_kv1 = tf.keras.layers.BatchNormalization()\n    \n        self.zero_embedding = tf.keras.layers.Embedding(1, 312)\n\n    \n        self.encoder_masks = []\n        for i in range(5):\n            row = tf.reshape(tf.repeat(tf.range(100*(i+1)), 100), (100*(i+1), 100))\n            col = tf.reshape(tf.tile(tf.range(100)+100*i, [100*(i+1)]), (100*(i+1), 100))\n            self.encoder_masks.append(tf.cast(tf.expand_dims(tf.transpose(tf.where(row > col, tf.float32.min/100, 0.)), axis=0), \"float32\"))\n        self.dense_sa_k0 = tf.keras.layers.Dense(64)\n        self.bn_sa_k0 = tf.keras.layers.BatchNormalization()\n        self.do_sa_k0 = tf.keras.layers.Dropout(0.1)\n        \n        self.dense_ts_k0 = tf.keras.layers.Dense(64)\n        self.bn_ts_k0 = tf.keras.layers.BatchNormalization()\n        self.do_ts_k0 = tf.keras.layers.Dropout(0.1)\n        self.do_ts_k1 = tf.keras.layers.Dropout(0.1)\n        \n        self.history_key_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation=\"selu\")\n        self.history_key_conv1 = tf.keras.layers.Conv1D(32, 1, 1)\n        self.history_correct_key_conv = tf.keras.layers.Conv1D(32, 1, 1, activation=\"relu\")\n        self.history_choice_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation=\"selu\")\n        self.history_choice_conv1 = tf.keras.layers.Conv1D(32, 1, 1)\n        self.history_wrong_choice_conv = tf.keras.layers.Conv1D(32, 1, 1, activation=\"relu\")\n        self.history_val_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation=\"selu\")\n        self.history_val_conv1 = tf.keras.layers.Conv1D(32, 1, 1)\n        self.gloval_avg_pool_key_drop = tf.keras.layers.Dropout(0.1)\n        self.gloval_avg_pool_choice_drop = tf.keras.layers.Dropout(0.1)\n        self.gloval_avg_pool_val_drop = tf.keras.layers.Dropout(0.1)\n        self.gloval_max_pool_key = tf.keras.layers.GlobalMaxPool1D()\n        self.gloval_avg_pool_key = tf.keras.layers.GlobalAvgPool1D()\n        self.gloval_max_pool_correct_key = tf.keras.layers.GlobalMaxPool1D()\n        self.gloval_max_pool_choice = tf.keras.layers.GlobalMaxPool1D()\n        self.gloval_avg_pool_choice = tf.keras.layers.GlobalAvgPool1D()\n        self.gloval_max_pool_wrong_choice = tf.keras.layers.GlobalMaxPool1D()\n        self.gloval_max_pool_val = tf.keras.layers.GlobalMaxPool1D()\n        self.gloval_avg_pool_val = tf.keras.layers.GlobalAvgPool1D()\n        \n        self.dense_u0 = tf.keras.layers.Dense(256)\n        self.dense_v0 = tf.keras.layers.Dense(1214)\n        self.dense_u1 = tf.keras.layers.Dense(256)\n        self.dense_v1 = tf.keras.layers.Dense(1214)\n        self.dense_u2 = tf.keras.layers.Dense(256)\n        self.dense_v2 = tf.keras.layers.Dense(1214)\n        self.bnc0 = tf.keras.layers.BatchNormalization()\n        self.bnc1 = tf.keras.layers.BatchNormalization()\n        self.bnc2 = tf.keras.layers.BatchNormalization()\n\n        self.dense1 = tf.keras.layers.Dense(512, activation='relu')\n        self.dense2 = tf.keras.layers.Dense(512, activation='relu')\n        self.dense3 = tf.keras.layers.Dense(512, activation='relu')\n        self.bn0 = tf.keras.layers.BatchNormalization()\n        self.bn1 = tf.keras.layers.BatchNormalization()\n        self.bn2 = tf.keras.layers.BatchNormalization()\n        self.bn3 = tf.keras.layers.BatchNormalization()\n        self.do0 = tf.keras.layers.Dropout(0)\n        self.do1 = tf.keras.layers.Dropout(3/10)\n        self.do2 = tf.keras.layers.Dropout(3/10)\n        self.do3 = tf.keras.layers.Dropout(0)\n        self.out = tf.keras.layers.Dense(1, activation='sigmoid')\n        self.sample_window = sample_window\n    \n    @tf.function(input_signature=[\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int16),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int64),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int16),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.float32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int16),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.float32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=(None, 76), dtype=tf.float32),\n        \n        tf.TensorSpec(shape=[], dtype=tf.bool),\n    ]\n    )\n    def call(self, user_ix, content_id, timestamp_cat, lag_time, order,\n             choice_history, timestamp_history, delta_timestamp_history,\n             raw_elapsed_time_history, elapsed_time_history,\n             explanation_history, cooltime_history,\n             part, feat,\n             training=None):\n        if training:\n            user_ix = tf.where(tf.random.uniform([len(user_ix)], maxval=1) < .01, 0, user_ix)\n        content_emb = self.content_embedding(content_id)\n        timestamp_emb = self.timestamp_embedding(tf.reshape(timestamp_cat, (-1,)))\n        choice_emb = tf.reduce_mean(tf.concat([tf.expand_dims(self.choice_embedding(content_id*4 + i), axis=1) for i in range(4)], axis=1), axis=1)        \n        lag_emb = self.lag_embedding(tf.reshape(lag_time, (-1,)))\n        \n        order_in = self.order_in(tf.expand_dims(tf.cast(order, \"float32\"), axis=-1))\n        order_in = tf.math.log1p(order_in)\n        feat_in = self.feat_in(feat)\n        part_oh = tf.reshape(self.oh_in(tf.one_hot(tf.cast(part, \"int32\"), 7)), (-1, 7))\n\n        content_tags_emb = self.tag_embedding(self.choice_tags_vec(tf.reshape(content_id*4, (-1, ))))\n        emb_max = tf.keras.backend.max(content_tags_emb, axis=1)\n        emb_min = tf.keras.backend.min(content_tags_emb, axis=1)\n        content_tags_emb = tf.where(emb_max > -emb_min, emb_max, emb_min)\n        \n        correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), (-1, 4))\n        choice_rate = tf.reshape(tf.gather(self.choice_rate, content_id*4), (-1, 4))\n        content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4), (-1, 1))\n        \n        content_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_emb, axis=1, keepdims=True)))\n        timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(timestamp_emb, axis=1, keepdims=True)))\n        choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(choice_emb, axis=1, keepdims=True)))\n        lag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(lag_emb, axis=1, keepdims=True)))\n        tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_tags_emb, axis=1, keepdims=True)))\n        \n        table_features = tf.reshape(tf.concat([order_in, feat_in, part_oh, content_norm, timestamp_norm, choice_norm, lag_norm, tag_norm, correct_choice, choice_rate], axis=-1), (-1, 97))\n        table_features = self.table_bn_clipper_qc(table_features)\n        \n        query_context_features_0 = tf.concat([content_emb, choice_emb, timestamp_emb, lag_emb, content_tags_emb, table_features], axis=1)\n        query_context_features_0 = tf.reshape(query_context_features_0, (-1, 305))\n        query_context_features = query_context_features_0 * self.dense_qc_v0(self.dense_qc_u0(self.bn_qc0(query_context_features_0))) + query_context_features_0\n        query_context_features = query_context_features_0 * self.dense_qc_v1(self.dense_qc_u1(self.bn_qc1(query_context_features))) + query_context_features\n        \n        content_mask = choice_history > 0\n        idx = tf.cast(tf.where(content_mask), \"int32\")\n\n        content_history = choice_history // 4\n        correct_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.), choice_history), axis=-1)\n        wrong_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.), choice_history), axis=-1)\n        question_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.), choice_history), axis=-1)\n        accuracy_history = tf.expand_dims(tf.where(tf.reshape(question_history, (-1, 500))==1., tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=-1)\n        \n        history_choice_emb = self.choice_embedding(tf.boolean_mask(choice_history, content_mask))\n        history_delta_timestamp_emb = self.delta_timestamp_embedding(tf.boolean_mask(delta_timestamp_history, content_mask))\n        history_elapsed_time_emb = self.elapsed_embedding(tf.boolean_mask(elapsed_time_history, content_mask))\n        history_cooltime_emb = self.cooltime_embedding(tf.boolean_mask(cooltime_history, content_mask))\n        history_explanation = tf.expand_dims(tf.cast(tf.boolean_mask(explanation_history, content_mask), \"float32\"), -1)\n        history_position_emb = self.positional_embedding(tf.boolean_mask(tf.reshape(tf.tile(tf.range(self.sample_window), [len(user_ix)]), (len(user_ix), 500)), content_mask))\n    \n        masked_choice = tf.boolean_mask(choice_history, content_mask)\n        history_choice = tf.reshape(tf.one_hot(tf.where(tf.gather(self.choice_is_question, masked_choice), masked_choice%4, -1), 4), (-1, 4))\n        history_correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, masked_choice), 4), (-1, 4))\n        history_choice_rate = tf.reshape(tf.gather(self.choice_rate, masked_choice), (-1, 4))\n                                    \n        history_is_correct = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.), tf.boolean_mask(choice_history, content_mask)), -1)\n        history_is_wrong = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.), tf.boolean_mask(choice_history, content_mask)), -1)\n        history_is_question = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.), tf.boolean_mask(choice_history, content_mask)), -1)\n        history_answer_mean = tf.expand_dims(tf.gather(self.answer_accuracy_mean, tf.boolean_mask(choice_history, content_mask)), -1)\n        history_answer2_mean = tf.expand_dims(tf.gather(self.answer_accuracy2_mean, tf.boolean_mask(choice_history, content_mask)), -1)\n        history_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_elapsed_time_mean, tf.boolean_mask(choice_history, content_mask)), -1)\n        history_correct_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_correct_elapssed_time_mean, tf.boolean_mask(choice_history, content_mask)), -1)\n\n        history_raw_elapsed_time =  tf.expand_dims(tf.boolean_mask(raw_elapsed_time_history, content_mask), -1)\n        history_ac_et_rate = history_correct_elapsed_time_mean / history_elapsed_time_mean\n        history_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_elapsed_time_mean, 1))\n        history_correct_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_correct_elapsed_time_mean, 1))\n        history_elapsed_time_mean = tf.math.log1p(history_elapsed_time_mean)\n        history_correct_elapsed_time_mean = tf.math.log1p(history_correct_elapsed_time_mean)        \n        \n        history_content_emb = self.content_embedding(tf.boolean_mask(choice_history//4, content_mask))\n        history_tag_emb = self.tag_embedding(self.choice_tags_vec(tf.boolean_mask(choice_history, content_mask)))\n        history_tag_emb_max = tf.keras.backend.max(history_tag_emb, axis=1)\n        history_tag_emb_min = tf.keras.backend.min(history_tag_emb, axis=1)\n        history_tag_emb_cd = tf.cast(history_tag_emb_max > history_tag_emb_min, \"float32\")\n        history_tag_emb = history_tag_emb_max * history_tag_emb_cd + history_tag_emb_min * (1 - history_tag_emb_cd)\n        history_part_oh =  tf.one_hot(tf.gather(self.choice_parts, tf.boolean_mask(choice_history, content_mask)), 7)\n        history_typeof_oh =  tf.one_hot(tf.gather(self.choice_typeof, tf.boolean_mask(choice_history, content_mask)), 4)\n        \n        history_choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_choice_emb ,axis=1, keepdims=True)))\n        history_delta_timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_delta_timestamp_emb ,axis=1, keepdims=True)))\n        history_elapsed_time_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_elapsed_time_emb ,axis=1, keepdims=True)))\n        history_cooltime_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_cooltime_emb ,axis=1, keepdims=True)))\n        history_position_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_position_emb ,axis=1, keepdims=True)))\n        history_tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_tag_emb ,axis=1, keepdims=True)))\n        \n        history_norms = self.bn_clipper_history(tf.concat([history_choice_norm, history_delta_timestamp_norm, history_elapsed_time_norm,\n                                                           history_cooltime_norm, history_position_norm, history_tag_norm], axis=1))\n\n        history_keyvalue_0 = tf.concat([history_content_emb, # +64\n                                        history_choice_emb, # +64\n                                        history_delta_timestamp_emb, # +32\n                                        history_elapsed_time_emb, # +32\n                                        history_cooltime_emb, # +32\n                                        history_position_emb, # + 32\n                                        history_norms, # + 6\n                                        history_explanation, # +1\n                                        history_choice, # +4\n                                        history_correct_choice, # + 4\n                                        history_choice_rate, # + 4\n                                        history_is_correct, # + 1\n                                        history_is_wrong, # + 1\n                                        history_is_question, # + 1\n                                        history_answer_mean, # + 1\n                                        history_answer2_mean, # + 1\n                                        history_elapsed_time_mean, # + 1\n                                        history_correct_elapsed_time_mean, # +1\n                                        history_ac_et_rate, # +1\n                                        history_elapsed_time_rate, # + 1\n                                        history_correct_elapsed_time_rate, # +1,\n                                        history_tag_emb, # + 16\n                                        history_part_oh, # +7\n                                        history_typeof_oh], # +4\n                                    axis = -1) # = 312\n        zero_embedding = self.zero_embedding(0)\n        \n        history_keyvalue = history_keyvalue_0 * self.dense_kv_v0(self.dense_kv_u0(self.bn_kv0(history_keyvalue_0))) + history_keyvalue_0\n        history_keyvalue = history_keyvalue_0 * self.dense_kv_v1(self.dense_kv_u1(self.bn_kv1(history_keyvalue))) + history_keyvalue\n        \n        history_keyvalue = tf.scatter_nd(idx, history_keyvalue, (len(user_ix), self.sample_window, 312))\n        history_keyvalue += tf.expand_dims(tf.cast(~content_mask, \"float32\"), -1) *  tf.expand_dims(zero_embedding, axis=0)\n        \n        query, context_features = tf.split(query_context_features, [64, -1], axis=1)\n        query = tf.expand_dims(query, axis=1)\n        history_key, history_choice, history_value = tf.split(history_keyvalue, [64, 64, -1], axis=2)\n        query0 = tf.identity(query)\n        history_key0 = tf.identity(history_key)\n        \n        queries = []\n        key_attentions = []\n        choice_attentions = []\n        value_attentions = []\n        add_features = []\n        \n        queries.append(query)\n        logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True) / 8\n        weight = self.do_ts_k0(tf.math.softmax(logit))\n        \n        key_attention = tf.matmul(weight, history_key)\n        choice_attention = tf.matmul(weight, history_choice)\n        sc_logit = tf.clip_by_value((tf.math.sign(logit) * tf.math.log1p(tf.abs(logit))), -4, 4)\n        logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)\n        cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True) / (tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9), (-1, 1))\n        \n        ac_div = tf.matmul(weight, question_history)\n        ac_num = tf.matmul(weight, correct_history)\n        eac_num = tf.matmul(weight, accuracy_history)\n        eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657), (-1, 1))\n        eac_rate = tf.math.maximum(0.01, eac_rate)\n        ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657), (-1, 1))\n        ac_rate = (ac_rate * tf.cast(tf.expand_dims(order, axis=1), \"float32\") + eac_rate * 10.) / (tf.cast(tf.expand_dims(order, axis=1), \"float32\") + 10.)\n        ac_eac_rate = ac_rate / eac_rate\n        h_ac_rate = 2 / (1/tf.maximum(0.01, ac_rate) + 1/tf.maximum(0.01, content_accuracy))\n        \n        cossim= tf.clip_by_value(cossim, -1, 1)\n        logit_attention = tf.clip_by_value(logit_attention, -5, 5)\n        ac_rate = tf.clip_by_value(ac_rate, 0, 1)\n        eac_rate = tf.clip_by_value(eac_rate, 0, 1)\n        ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)), -1, 5)\n        calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))\n        norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))\n        add_feature = self.bn_clipper_0(tf.concat([tf.reshape(calc_feature, (-1, 1, 6)), tf.reshape(norm_feature, (-1, 1, 3))], axis=2))\n        \n        key_attentions.append(key_attention)\n        choice_attentions.append(choice_attention)\n        add_features.append(tf.reshape(add_feature, (-1, 9)))\n        query = query0 * self.dense_ts_k0(self.bn_ts_k0(tf.concat([key_attention, choice_attention, add_feature], axis=2))) + query       \n        # local attention\n        history_keys = tf.split(history_key, [100]*5, axis=1)\n        history_keys0 = tf.split(history_key0, [100]*5, axis=1) \n        history_timestamps = tf.split(timestamp_history, [100]*5, axis=1)\n        zero_emb_key, zero_emb_choice, _ = tf.split(model.zero_embedding(0), [64, 64, -1], axis=0)\n        zero_emb_key = tf.reshape(tf.tile(zero_emb_key, [len(user_ix)]), (-1, 1, 64))\n        zero_emb_choice = tf.reshape(tf.tile(zero_emb_choice, [len(user_ix)]), (-1, 1, 64))\n        history_new_keys = []\n        attentions = []\n        history_attentions = []\n        for i in range(5):\n            key_slice = tf.slice(history_key, [0, 0, 0], [-1, 100*(i+1), -1])\n            key_slice = tf.concat([zero_emb_key, key_slice], axis=1)\n            choice_slice = tf.slice(history_choice, [0, 0, 0], [-1, 100*(i+1), -1])\n            choice_slice = tf.concat([zero_emb_choice, choice_slice], axis=1)\n            timestamp_slice = tf.slice(timestamp_history, [0, 0], [-1, 100*(i+1)])\n            timestamp_slice = tf.concat([tf.ones((len(user_ix), 1), dtype=\"int64\")*-1, timestamp_slice], axis=1)\n\n            query_timestamp = tf.maximum(tf.constant(0, dtype=\"int64\"), tf.reshape(tf.repeat(history_timestamps[i], 100*(i+1)+1, axis=1), (-1, 100, 100*(i+1)+1)))\n            key_timestamp = tf.reshape(tf.tile(timestamp_slice, [1, 100]), (-1, 100, 100*(i+1)+1))\n            encoder_mask = tf.where(key_timestamp < query_timestamp, 0., tf.float32.min/100)\n\n            logit = tf.matmul(tf.slice(history_keys[i], [0, 0, 0], [-1, -1, 64]), tf.slice(key_slice, [0, 0, 0], [-1, -1, 64]), transpose_b=True) + encoder_mask\n            weight = self.do_sa_k0(tf.math.softmax(logit/8))\n\n            key_attention = tf.matmul(weight, key_slice)\n            choice_attention = tf.matmul(weight, choice_slice)\n            \n            history_exist = tf.where(history_timestamps[0] >= 0, 1., 0.)\n            \n            sc_logit = tf.clip_by_value(tf.math.sign(logit) * tf.math.log1p(tf.abs(logit)), -4, 4)\n            history_logit = tf.reduce_sum(sc_logit * weight, axis=2) * history_exist\n            \n            history_cossim = tf.reduce_sum(key_attention * history_keys[i], axis=2) / (tf.norm(key_attention, axis=2) * tf.norm(history_keys[i], axis=2) + 1e-9)\n            history_cossim *= history_exist\n            history_cossim = tf.clip_by_value(history_cossim, -1, 1)\n            \n            history_logit = tf.stop_gradient(tf.expand_dims(history_logit, axis=2))\n            history_exist = tf.stop_gradient(tf.expand_dims(history_exist, axis=2))\n            history_cossim = tf.stop_gradient(tf.expand_dims(history_cossim, axis=2))\n            history_add = self.bn_clipper_1(tf.concat([history_logit, history_logit, history_cossim], axis=2))\n            \n            attention = tf.concat([key_attention, choice_attention, history_add], axis=2)\n            history_new_keys.append(history_keys0[i] * self.dense_sa_k0(self.bn_sa_k0(attention)) + history_keys[i])\n            history_attentions.append(attention)\n            \n        history_key = tf.concat(history_new_keys, axis=1)\n        history_value = tf.concat([history_value, tf.concat(history_attentions, axis=1)], axis=2)\n\n        queries.append(query)\n        logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True) / 8\n        weight = self.do_ts_k1(tf.math.softmax(logit))\n        key_attention = tf.matmul(weight, history_key)\n        choice_attention = tf.matmul(weight, history_choice)\n        value_attention = tf.matmul(weight, history_value)\n        sc_logit = tf.clip_by_value((tf.math.sign(logit) * tf.math.log1p(tf.abs(logit))), -4, 4)\n        logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)\n        cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True) / (tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9), (-1, 1))\n     \n        ac_div = tf.matmul(weight, question_history)\n        ac_num = tf.matmul(weight, correct_history)\n        eac_num = tf.matmul(weight, accuracy_history)\n        eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657), (-1, 1))\n        eac_rate = tf.math.maximum(0.01, eac_rate)\n        ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657), (-1, 1))\n        ac_rate = (ac_rate * tf.cast(tf.expand_dims(order, axis=1), \"float32\") + eac_rate * 10.) / (tf.cast(tf.expand_dims(order, axis=1), \"float32\") + 10.)\n        h_ac_rate = 2 / (1/tf.maximum(0.01, ac_rate) + 1/tf.maximum(0.01, content_accuracy))\n        ac_eac_rate = ac_rate / eac_rate\n        \n        cossim= tf.clip_by_value(cossim, -1, 1)\n        logit_attention = tf.clip_by_value(logit_attention, -5, 5)\n        ac_rate = tf.clip_by_value(ac_rate, 0, 1)\n        eac_rate = tf.clip_by_value(eac_rate, 0, 1)\n        ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)), -1, 5)\n        calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))\n        norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))\n        add_feature = self.bn_clipper_2(tf.concat([tf.reshape(calc_feature, (-1, 1, 6)), tf.reshape(norm_feature, (-1, 1, 3))], axis=2))\n        \n        key_attentions.append(key_attention)\n        choice_attentions.append(choice_attention)\n        value_attentions.append(value_attention)\n        add_features.append(tf.reshape(add_feature, (-1, 9)))\n\n        queries_vec = tf.reshape(tf.concat(queries, axis=2), (-1, 128))\n        key_attentions = tf.reshape(tf.concat(key_attentions, axis=2), (-1, 128))\n        choice_attentions = tf.reshape(tf.concat(choice_attentions, axis=2), (-1, 128))\n        value_attentions = tf.reduce_mean(tf.concat(value_attentions, axis=1), axis=1)\n        feature_attentions = tf.concat(add_features, axis=1)\n        \n        filter_res_key_max = self.gloval_max_pool_key(self.history_key_conv0(history_key))\n        filter_res_key_avg = self.gloval_avg_pool_key(self.gloval_avg_pool_key_drop(self.history_key_conv1(history_key)))\n        filter_res_correct_key_max = self.gloval_max_pool_correct_key(self.history_correct_key_conv(history_key) * correct_history)\n        filter_res_choice_max = self.gloval_max_pool_choice(self.history_choice_conv0(history_choice))\n        filter_res_choice_avg = self.gloval_avg_pool_choice(self.gloval_avg_pool_choice_drop(self.history_choice_conv1(history_choice)))\n        filter_res_wrong_choice_max = self.gloval_max_pool_wrong_choice(self.history_wrong_choice_conv(history_choice) * wrong_history)\n        filter_res_val_max = self.gloval_max_pool_val(self.history_val_conv0(history_value))\n        filter_res_val_avg = self.gloval_avg_pool_val(self.gloval_avg_pool_val_drop(self.history_val_conv1(history_value)))\n        filter_res = tf.concat([filter_res_key_max, filter_res_key_avg, filter_res_correct_key_max,\n                                filter_res_choice_max, filter_res_choice_avg,filter_res_wrong_choice_max,\n                                filter_res_val_max, filter_res_val_avg], axis=1)\n    \n        table_features = tf.concat([table_features, feature_attentions], axis=1)\n        X0 = tf.keras.layers.Concatenate()([context_features,\n                                            queries_vec, key_attentions, choice_attentions, value_attentions,\n                                            feature_attentions, filter_res])    \n\n        prod_out0 = self.dense_v0(self.dense_u0(self.bnc0(X0)))\n        X = X0 * prod_out0 + X0\n        prod_out1 = self.dense_v1(self.dense_u1(self.bnc1(X)))\n        X = X0 * prod_out1 + X\n        prod_out2 = self.dense_v2(self.dense_u2(self.bnc2(X)))\n        X_feat = tf.concat([X0 * prod_out2 + X, table_features], axis=1)      \n        \n        X = self.bn0(X_feat)\n        X = self.do0(X)\n        X = self.dense1(X)\n        X = self.bn1(X)\n        X = self.do1(X)\n        X = self.dense2(X)\n        X = self.bn2(X)\n        X = self.do2(X)\n        X = self.dense3(X)\n        X = self.bn3(X)\n        X = self.do3(X)\n        return self.out(X)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:45.120685Z","iopub.status.busy":"2021-01-07T16:52:45.115583Z","iopub.status.idle":"2021-01-07T16:52:45.137486Z","shell.execute_reply":"2021-01-07T16:52:45.136905Z"},"papermill":{"duration":0.0607,"end_time":"2021-01-07T16:52:45.13759","exception":false,"start_time":"2021-01-07T16:52:45.07689","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"feat_in_col = (\n['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', \"idf\"]\n+ [\"correct_streak\", \"wrong_streak\"]\n+ [f\"user_choice_{i}_rate_rate\" for i in range(4)]\n+ [f\"user_wrong_choice_{i}_rate_rate\" for i in range(4)]\n+ [f\"user_choice_rate_rate_correct\"]\n+ [f\"user_wrong_choice_rate_rate_correct\"]\n+ [\"harmonic_mean\", \"part_harmonic_mean\", \"frequency\", \"content_seen_sum\", \"acc_mean\", \"acc_per_elapse\", \"elp_mean\", \"scaled_elapsed_time_mean\", \"exp_mean\"]\n+ ['part0_sum', 'part0_count', 'part1_sum', 'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count', 'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum', 'part6_count', 'question_sum', 'question_count']\n+ [f\"part{i}_accuracy_rate\" for _ in range(7)] + [\"total_accuracy_rate\"]\n+ [f\"part{i}_count_rate\" for _ in range(7)]\n+ [\"questions_one_day\", \"questions_one_hour\", \"questions_one_week\"]\n+ [\"day_angle_cos_mean\", \"day_angle_sin_mean\", \"day_angle_norm\", \"day_angle_normed_cos\", \"day_angle_normed_sin\", \"week_angle_cos_mean\", \"week_angle_sin_mean\", \"week_angle_norm\", \"week_angle_normed_cos\", \"week_angle_normed_sin\"]\n+ [\"last_correct_ts\", \"last_wrong_ts\", \"last_correct_ts_rate\", \"last_wrong_ts_rate\"]\n+ [\"delta_last_2\", \"delta_last_1\", \"delta_last_0\"]\n)\n\ncontext_features = (\n  [f\"context_choice_emb_{i}\" for i in range(64)]\n+ [f\"context_timestamp_emb_{i}\" for i in range(32)]\n+ [f\"context_lag_emb_{i}\" for i in range(32)]\n+ [f\"content_tags_emb_{i}\" for i in range(16)]\n+ [\"context_order\"]\n+ [\"context_\" + feat for feat in feat_in_col]\n+ [f\"context_part_oh_{i}\" for i in range(7)]\n+ [\"context_content_norm\", \"context_timestamp_norm\", \"context_choice_norm\", \"context_lag_norm\", \"context_tag_norm\"]\n+ [f\"context_correct_choice_oh_{i}\" for i in range(4)]\n+ [f\"context_content_choice_rate_{i}\" for i in range(4)]\n)\n\nhistory_value_features = (\n[f\"history_delta_timestamp_emb_{i}\" for i in range(32)]\n+ [f\"history_elapsed_time_emb_{i}\" for i in range(32)]\n+ [f\"history_cooltime_emb_{i}\" for i in range(32)]\n+ [f\"history_position_emb_{i}\" for i in range(32)]\n+ [\"history_choice_norm\", \"history_delta_timestamp_norm\", \"history_elapsed_time_norm\", \"history_cooltime_norm\", \"history_position_norm\", \"history_tag_norm\"]\n+ [\"history_explanation\"]\n+ [f\"history_choice{i}\" for i in range(4)]\n+ [f\"history_correct_choice{i}\" for i in range(4)]\n+ [f\"history_choice_rate{i}\" for i in range(4)]\n+ [\"history_is_correct\", \"history_is_wrong\", \"history_is_question\"]\n+ [\"history_answer_mean\", \"history_answer2_mean\"]\n+ [\"history_elapsed_time_mean\", \"history_correct_elapsed_time_mean\", \"history_ac_et_rate\"]\n+ [\"history_elapsed_time_rate\", \"history_correct_elapsed_time_rate\"]\n+ [f\"history_tag_{i}\" for i in range(16)]\n+ [f\"history_part_oh_{i}\" for i in range(7)]\n+ [f\"history_typeof_oh_{i}\" for i in range(4)]\n+ [f\"history_encoder_key_attention_{i}\" for i in range(64)]\n+ [f\"history_encoder_choice_attention_{i}\" for i in range(64)]\n+ [f\"history_encoder_logit\", \"history_encoder_exist\", \"history_encoder_cossim\"]\n)\nfilter_cols = (\n[f\"filter_res_key_max_{i}\" for i in range(32)]\n+ [f\"filter_res_key_avg_{i}\" for i in range(32)]\n+ [f\"filter_res_correct_key_max_{i}\" for i in range(32)]\n+ [f\"filter_res_choice_max_{i}\" for i in range(32)]\n+ [f\"filter_res_choice_avg_{i}\" for i in range(32)]\n+ [f\"filter_res_wrong_choice_max_{i}\" for i in range(32)]\n+ [f\"filter_res_value_max_{i}\" for i in range(32)]\n+ [f\"filter_res_value_avg_{i}\" for i in range(32)]\n)\n\n\nX0_cols = (\ncontext_features\n+ [f\"query_emb_level0_{i}\" for i in range(64)] + [f\"query_emb_level1_{i}\" for i in range(64)]\n+ [f\"key_attention_emb_level0_{i}\" for i in range(64)] + [f\"key_attention_emb_level1_{i}\" for i in range(64)]\n+ [f\"choice_attention_emb_level0_{i}\" for i in range(64)] + [f\"choice_attention_emb_level1_{i}\" for i in range(64)]\n+ history_value_features\n+ [f\"history_level0_{name}\" for name in [\"cossim\", \"logit_attention\", \"ac_rate\", \"eac_rate\", \"ac_eac_rate\",\" h_ac_rate\"]]\n+ [f\"history_level0_{name}_norm\" for name in [\"query_attention\", \"key_attention\", \"choice_attention\"]]\n+ [f\"history_level1_{name}\" for name in [\"cossim\", \"logit_attention\", \"ac_rate\", \"eac_rate\", \"ac_eac_rate\",\" h_ac_rate\"]]\n+ [f\"history_level1_{name}_norm\" for name in [\"query_attention\", \"key_attention\", \"choice_attention\"]]\n+ filter_cols\n)\n\ntable_features = (\n[\"raw_order\"]\n+ [\"raw_\" + feat for feat in feat_in_col]\n+ [f\"raw_part_oh_{i}\" for i in range(7)]\n+ [\"raw_content_norm\", \"raw_timestamp_norm\", \"raw_choice_norm\", \"raw_lag_norm\", \"raw_tag_norm\"]\n+ [f\"raw_correct_choice_oh_{i}\" for i in range(4)]\n+ [f\"raw_content_choice_rate_{i}\" for i in range(4)]\n+ [f\"raw_history_level0_{name}\" for name in [\"cossim\", \"logit_attention\", \"ac_rate\", \"eac_rate\", \"ac_eac_rate\",\" h_ac_rate\"]]\n+ [f\"raw_history_level0_{name}_norm\" for name in [\"query_attention\", \"key_attention\", \"choice_attention\"]]\n+ [f\"raw_history_level1_{name}\" for name in [\"cossim\", \"logit_attention\", \"ac_rate\", \"eac_rate\", \"ac_eac_rate\",\" h_ac_rate\"]]\n+ [f\"raw_history_level1_{name}_norm\" for name in [\"query_attention\", \"key_attention\", \"choice_attention\"]]\n\n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:45.18243Z","iopub.status.busy":"2021-01-07T16:52:45.181755Z","iopub.status.idle":"2021-01-07T16:52:49.851473Z","shell.execute_reply":"2021-01-07T16:52:49.850507Z"},"papermill":{"duration":4.695001,"end_time":"2021-01-07T16:52:49.851595","exception":false,"start_time":"2021-01-07T16:52:45.156594","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model = BaseModel(max(user_id_map.values()), lectures_df[\"content_id\"].max(), lectures_df[\"content_id\"].max()*4 + 3,\n                  choice_is_correct, choice_is_wrong, choice_is_question,\n                  choice_tags, choice_parts, choice_typeof,\n                  answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,\n                  correct_choice, choice_rate)\nmodel.load_weights(\"../input/nadare-processed-2/model8\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:50.47264Z","iopub.status.busy":"2021-01-07T16:52:50.451804Z","iopub.status.idle":"2021-01-07T16:52:50.572643Z","shell.execute_reply":"2021-01-07T16:52:50.572132Z"},"papermill":{"duration":0.144035,"end_time":"2021-01-07T16:52:50.572749","exception":false,"start_time":"2021-01-07T16:52:50.428714","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class Predictor(tf.Module):\n    def __init__(self,\n                 timestamp_bins,\n                 delta_bins,\n                 elapsedtime_bins,\n                 choice_is_correct,\n                 choice_is_wrong,\n                 choice_is_question,\n                 answer_accuracy_mean,\n                 answer_elapsed_time_mean,\n                 answer_elapsed_time_std,\n                 correct_choice,\n                 choice_rate,\n                 part_mean,\n                 part_rate):\n        self.sample_window = 500\n        self.timestamp_bins = tf.constant(timestamp_bins, dtype=\"float32\")\n        self.delta_bins = tf.constant(delta_bins, dtype=\"float32\")\n        self.elapsedtime_bins = tf.constant(elapsedtime_bins, dtype=\"float32\")\n        self.choice_is_correct = choice_is_correct\n        self.choice_is_wrong = choice_is_wrong\n        self.choice_is_question = choice_is_question\n        self.answer_accuracy_mean = answer_accuracy_mean\n        self.answer_elapsed_time_mean = answer_elapsed_time_mean\n        self.answer_elapsed_time_std = answer_elapsed_time_std\n        self.correct_choice = correct_choice\n        self.choice_rate = choice_rate\n        self.part_mean = part_mean\n        self.part_rate = part_rate\n    \n    @tf.function(input_signature=[\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=None, dtype=tf.int64),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.int64),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.float32),\n        tf.TensorSpec(shape=(None, 500), dtype=tf.float32),\n        tf.TensorSpec(shape=None, dtype=tf.int32),\n        tf.TensorSpec(shape=(None, 4), dtype=tf.float32),\n        tf.TensorSpec(shape=(None, 16), dtype=tf.float32)\n    ]\n    )\n    def __call__(self, user_ix, content_id, timestamp, order,\n                 choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history,\n                 part, feat, raw_feat):\n        N = len(user_ix)\n        \n        timestamp_cat = tf.reshape(tf.cast(tf.searchsorted(self.timestamp_bins, tf.cast(timestamp, \"float32\")), dtype=\"int16\"), (-1,))\n        timestamp_history = tf.identity(raw_timestamp_history)\n        \n        delta_timestamp_history = tf.expand_dims(timestamp, axis=-1) - raw_timestamp_history\n        delta_timestamp_history = tf.cast(tf.searchsorted(self.delta_bins, tf.cast(tf.keras.backend.flatten(delta_timestamp_history), \"float32\"), side=\"right\"), dtype=\"int16\")\n        delta_timestamp_history = tf.reshape(delta_timestamp_history, [-1, self.sample_window])\n        \n        elapsed_time_history = tf.cast(tf.searchsorted(self.elapsedtime_bins, tf.cast(tf.keras.backend.flatten(raw_elapsed_time_history), \"float32\"), side=\"right\"), dtype=\"int16\")\n        elapsed_time_history = tf.reshape(elapsed_time_history, [-1, self.sample_window])\n        raw_elapsed_time_history = tf.cast(tf.reshape(raw_elapsed_time_history, [-1, self.sample_window]), \"float32\")\n\n        timestamp_histories = tf.split(tf.cast(timestamp_history, \"float32\"), 500, axis=1)\n        elapsed_time_histories = tf.split(raw_elapsed_time_history, 500, axis=1)\n        \n        lag_time = tf.where(tf.reshape(timestamp_histories[-1], (-1,)) < 0, 0, tf.searchsorted(self.delta_bins, tf.cast(timestamp, \"float32\") - tf.reshape(timestamp_histories[-1], (-1,))))\n        \n        ts = tf.reshape(tf.cast(timestamp, \"float32\"), (-1, 1))\n        ept = tf.ones((N, 1), dtype=\"float32\") * tf.float32.max/100\n        nept = tf.zeros((N, 1), dtype=\"float32\")\n        cooltimes = []\n\n        for i in range(499, -1, -1):\n            tsn = timestamp_histories[i]\n            if i < 499:\n                ept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], ept, nept)\n            cooltimes.append(ts - tsn - ept)\n            ts = tf.where(elapsed_time_histories[i] <= 0, ts, tsn)\n            if i < 499:\n                nept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], elapsed_time_histories[i] + nept, elapsed_time_histories[i])\n                nept = tf.where(elapsed_time_histories[i] <= 0, ept, nept)\n            else:\n                nept = tf.where(elapsed_time_histories[i] <= 0, ept, elapsed_time_histories[i])\n\n        cooltimes = tf.concat(cooltimes[::-1], axis=1)\n        cooltimes = tf.searchsorted(self.delta_bins, tf.reshape(cooltimes, (-1,)))\n        cooltime_history = tf.reshape(cooltimes, [N, self.sample_window])\n        cooltime_history = tf.where(choice_history > 0, tf.minimum(9999, cooltime_history+1), 0)\n\n        correct_answer_sum = tf.reduce_sum(tf.where(tf.gather(self.choice_is_correct, choice_history), 1., 0.), axis=1, keepdims=True)\n\n        history_is_question = tf.gather(self.choice_is_question, choice_history)\n        history_is_correct = tf.gather(self.choice_is_correct, choice_history)\n        history_is_wrong = tf.gather(self.choice_is_wrong, choice_history)   \n        explanation_sum = tf.reduce_sum(tf.where(history_is_question, explanation_history, 0), axis=1, keepdims=True)\n        question_sum = tf.reduce_sum(tf.where(history_is_question, 1., 0.), axis=1, keepdims=True)\n        wrong_sum = tf.reduce_sum(tf.where(history_is_wrong, 1., 0.), axis=1, keepdims=True)\n        elapsed_acc_sum = tf.reduce_sum(tf.where(history_is_question, tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=1, keepdims=True)\n\n        elapsed_time_content_mean = tf.gather(self.answer_elapsed_time_mean, choice_history)\n        elapsed_time_content_std = tf.gather(self.answer_elapsed_time_std, choice_history)\n        scaled_elapsed_time = tf.where(history_is_question, tf.math.log1p(raw_elapsed_time_history), 0.)\n        scaled_elapsed_time_sum = tf.reduce_sum(scaled_elapsed_time, axis=1, keepdims=True)\n\n        correct_streak = tf.zeros((N, 1))\n        correct_continue = tf.ones((N, 1))\n        wrong_streak = tf.zeros((N, 1))\n        wrong_continue = tf.ones((N, 1))\n\n        for i in range(500):\n            isc = tf.where(tf.slice(history_is_correct, [0, 500-i-1], [-1, 1]), 1., 0.)\n            isw = tf.where(tf.slice(history_is_wrong, [0, 500-i-1], [-1, 1]), 1., 0.)\n            isq = tf.where(tf.slice(history_is_question, [0, 500-i-1], [-1, 1]), 0., 1.)\n            cc = tf.maximum(isq, isc)\n            ww = tf.maximum(isq, isw)\n            correct_continue *= cc\n            wrong_continue *= ww\n            correct_streak += correct_continue * cc\n            wrong_streak += wrong_continue * ww\n\n        correct_streak = tf.math.log1p(correct_streak)\n        wrong_streak = tf.math.log1p(wrong_streak)\n\n        user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_question, choice_history%4, -1), 4), axis=1)\n        user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history), axis=1)\n        user_choice_rate_rate = user_real_choice_rate / (user_elapsed_choice_rate + 1e-9)\n        user_choice_rate_rate = (user_choice_rate_rate * question_sum + 10) / (question_sum + 10)\n        wrong_user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_wrong, choice_history%4, -1), 4), axis=1)\n        wrong_user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history) * tf.expand_dims(tf.where(history_is_wrong, 1., 0.), axis=-1), axis=1)\n        wrong_user_choice_rate_rate = wrong_user_real_choice_rate / (wrong_user_elapsed_choice_rate + 1e-9)\n        wrong_user_choice_rate_rate = (wrong_user_choice_rate_rate * wrong_sum + 10) / (wrong_sum + 10)\n        user_choice_rate_rate_correct = tf.reduce_sum(user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)\n        wrong_user_choice_rate_rate_correct = tf.reduce_sum(wrong_user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)\n        \n        rate_feat = tf.concat([user_choice_rate_rate, wrong_user_choice_rate_rate, user_choice_rate_rate_correct, wrong_user_choice_rate_rate_correct], axis=1)\n        \n        frequency = tf.math.log1p(tf.cast(timestamp, \"float32\") / (tf.cast(order, \"float32\")+1e-9))\n        frequency = tf.expand_dims(tf.where(order > 0, frequency, 15), axis=-1)\n        content_seen_sum = tf.math.log1p(tf.reduce_sum(tf.where(tf.logical_and(tf.equal(choice_history//4, tf.expand_dims(content_id, axis=1)), choice_history > 0), 1., 0.), axis=1, keepdims=True))\n        elp_mean = tf.where(question_sum > 0, (elapsed_acc_sum) / (question_sum + 1e-9), 0.657)\n        acc_mean = tf.where(question_sum > 0, (correct_answer_sum) / (question_sum + 1e-9), 0.657)\n        acc_mean = (acc_mean * question_sum + elp_mean * 10) / (question_sum + 10) # new\n        exp_mean = tf.where(question_sum > 0, (explanation_sum + 5e-10) / (question_sum + 1e-9), .5)\n        acc_per_elapse = tf.math.log1p((acc_mean + 1e-9) / tf.math.maximum(elp_mean, 0.01))\n        scaled_elapsed_time_mean = tf.where(question_sum>0, (scaled_elapsed_time_sum) / (question_sum + 1e-9), 0)\n\n        choice_sum = tf.reduce_sum(tf.where(choice_history > 0, 1., 0.), axis=1)\n\n        day_msecs = (1000 * 60 * 60 * 24)\n        day_angle = (tf.cast(timestamp_history, \"float32\") % day_msecs) / day_msecs * 2 * np.pi\n\n        day_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(day_angle), 0), axis=1) / tf.math.maximum(choice_sum, 1e-9)\n        day_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(day_angle), 0), axis=1) / tf.math.maximum(choice_sum, 1e-9)\n        day_angle_norm = tf.math.sqrt(tf.math.square(day_angle_cos_mean) + tf.math.square(day_angle_sin_mean))\n        day_angle_normed_cos = day_angle_cos_mean / tf.math.maximum(day_angle_norm, 1e-9)\n        day_angle_normed_sin = day_angle_sin_mean / tf.math.maximum(day_angle_norm, 1e-9)\n\n        week_msecs = (1000 * 60 * 60 * 24 * 7)\n        week_angle = (tf.cast(timestamp_history, \"float32\") % week_msecs) / week_msecs * 2 * np.pi\n\n        week_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(week_angle), 0), axis=1) / tf.math.maximum(choice_sum, 1e-9)\n        week_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(week_angle), 0), axis=1) / tf.math.maximum(choice_sum, 1e-9)\n        week_angle_norm = tf.math.sqrt(tf.math.square(week_angle_cos_mean) + tf.math.square(week_angle_sin_mean))\n        week_angle_normed_cos = week_angle_cos_mean / tf.math.maximum(week_angle_norm, 1e-9)\n        week_angle_normed_sin = week_angle_sin_mean / tf.math.maximum(week_angle_norm, 1e-9)        \n        \n        last3_timestamp = tf.where(tf.slice(choice_history, [0, 497], [-1, 3]) > 0, tf.slice(timestamp_history, [0, 497], [-1, 3]), [[132324, 75973, 32387]])\n        delta_last3 = tf.math.log1p(tf.cast(tf.reshape(timestamp, (-1, 1)) - last3_timestamp, \"float32\"))\n        \n        last_correct_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_correct, choice_history), timestamp_history, 0), axis=1), \"float32\"))\n        last_wrong_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_wrong, choice_history), timestamp_history, 0), axis=1), \"float32\"))\n\n        last_correct_ts_rate = tf.where(timestamp > 0, last_correct_ts / tf.math.log1p(tf.cast(timestamp, \"float32\")), 1)\n        last_wrong_ts_rate = tf.where(timestamp > 0, last_wrong_ts / tf.math.log1p(tf.cast(timestamp, \"float32\")), 1)\n\n        new_feat = [day_angle_cos_mean, day_angle_sin_mean, day_angle_norm, day_angle_normed_cos, day_angle_normed_sin,\n                    week_angle_cos_mean, week_angle_sin_mean, week_angle_norm, week_angle_normed_cos, week_angle_normed_sin,\n                    last_correct_ts, last_wrong_ts, last_correct_ts_rate, last_wrong_ts_rate]\n        new_feat = tf.concat([tf.expand_dims(f, axis=-1) for f in new_feat] + [delta_last3], axis=1) # 17\n        \n        questions_one_hour = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-3600*1000, axis=1), 0)), \"float32\"))\n        questions_one_day = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-24*3600*1000, axis=1), 0)), \"float32\"))\n        questions_one_week = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-7*24*3600*1000, axis=1), 0)), \"float32\"))  \n        \n        num_col = tf.constant([0, 2, 4, 6, 8, 10, 12, 14])\n        div_col = tf.constant([1, 3, 5, 7, 9, 11, 13, 15])\n        row_ixs = tf.repeat(tf.range(N), len(num_col))\n        num_col = tf.tile(num_col, [N])\n        div_col = tf.tile(div_col, [N])\n        \n        num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)), (N, -1))\n        div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)), (N, -1))\n        mr_mean_feat = num_feat / (div_feat + 1e-9)\n        mr_mean_feat = (mr_mean_feat * div_feat + self.part_mean * 10) / (div_feat + 10)\n\n        num_col = tf.constant([1, 3, 5, 7, 9, 11, 13])\n        div_col = tf.constant([15, 15, 15, 15, 15, 15, 15])\n        row_ixs = tf.repeat(tf.range(N), len(num_col))\n        num_col = tf.tile(num_col, [N])\n        div_col = tf.tile(div_col, [N])\n\n        num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)), (N, -1))\n        div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)), (N, -1))\n        mr_rate_feat = num_feat / (div_feat + 1e-9)\n        mr_rate_feat = (mr_rate_feat * tf.expand_dims(tf.cast(order, \"float32\"), axis=1) + self.part_rate * 10) / (tf.expand_dims(tf.cast(order, \"float32\"), axis=1) + 10)\n\n        mr_feat = tf.concat([mr_mean_feat, mr_rate_feat], axis=1)\n        count_feat = tf.math.log1p(raw_feat)\n\n        content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4), (-1, 1))\n        harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), tf.ones(N, dtype=\"int32\")*7], axis=1)), (-1, 1))) + 1/tf.math.maximum(1e-9, content_accuracy))\n        part_harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), part], axis=1)), (-1, 1))) + 1/tf.math.maximum(1e-9, content_accuracy))\n    \n        new_feats = tf.concat([correct_streak, wrong_streak, rate_feat, harmonic_mean, part_harmonic_mean, frequency, content_seen_sum, acc_mean, acc_per_elapse, elp_mean, scaled_elapsed_time_mean, exp_mean, count_feat, mr_feat, questions_one_day, questions_one_hour, questions_one_week, new_feat], axis=1)\n        feat = tf.concat([feat, new_feats], axis=1)\n\n        return (user_ix, content_id, timestamp_cat, lag_time, order, choice_history, timestamp_history, delta_timestamp_history,\n                 raw_elapsed_time_history, elapsed_time_history, explanation_history, cooltime_history, part, feat)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:50.618893Z","iopub.status.busy":"2021-01-07T16:52:50.618249Z","iopub.status.idle":"2021-01-07T16:52:50.62285Z","shell.execute_reply":"2021-01-07T16:52:50.622324Z"},"papermill":{"duration":0.030481,"end_time":"2021-01-07T16:52:50.622942","exception":false,"start_time":"2021-01-07T16:52:50.592461","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"predictor = Predictor(timestamp_bins,\n                      delta_bins,\n                      elapsedtime_bins, \n                      choice_is_correct,\n                      choice_is_wrong,\n                      choice_is_question,\n                      answer_accuracy_mean,\n                      answer_elapsed_time_mean,\n                      answer_elapsed_time_std,\n                      correct_choice,\n                      choice_rate,\n                      part_mean,\n                      part_rate)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:50.667408Z","iopub.status.busy":"2021-01-07T16:52:50.665996Z","iopub.status.idle":"2021-01-07T16:52:50.670707Z","shell.execute_reply":"2021-01-07T16:52:50.670206Z"},"papermill":{"duration":0.027924,"end_time":"2021-01-07T16:52:50.670797","exception":false,"start_time":"2021-01-07T16:52:50.642873","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions_df = questions_df.set_index(\"content_id\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-07T16:52:50.738281Z","iopub.status.busy":"2021-01-07T16:52:50.732702Z","iopub.status.idle":"2021-01-07T16:53:36.824244Z","shell.execute_reply":"2021-01-07T16:53:36.824728Z"},"papermill":{"duration":46.131988,"end_time":"2021-01-07T16:53:36.824888","exception":false,"start_time":"2021-01-07T16:52:50.6929","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\nfrom time import time as timer\nfrom gc import collect\nimport warnings\nwarnings.simplefilter('ignore')\n\nfeat_col = ['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', \"idf\"]\nraw_feat_col = ['part0_sum', 'part0_count', 'part1_sum',\n                'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count',\n                'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum',\n                'part6_count', 'question_sum', 'question_count']\nuser_ix_last = max(user_id_map.values()) + 1\nuser_ix_max = tf.constant(max(user_id_map.values()), dtype=\"int32\")\npast_df = None\ni = 0\ncollect()\n\nmapping_question = np.vectorize(lambda x: question_id_map[x], otypes=[\"int64\"])\nmapping_lecture = np.vectorize(lambda x: lecture_id_map[x], otypes=[\"int64\"])\n\nfor (test_raw_df, sample_prediction_df) in tqdm(iter_test):\n    test_raw_df.loc[test_raw_df[\"content_type_id\"] == False, \"content_id\"] = mapping_question(test_raw_df.loc[test_raw_df[\"content_type_id\"] == False, \"content_id\"].values)\n    test_raw_df.loc[test_raw_df[\"content_type_id\"] == True, \"content_id\"] = mapping_lecture(test_raw_df.loc[test_raw_df[\"content_type_id\"] == True, \"content_id\"].values)\n    if past_df is not None:\n        past_df[\"user_answer\"] = eval(test_raw_df[\"prior_group_responses\"].values[0])\n        past_df[\"choice_id\"] = past_df[\"content_id\"]*4 + past_df[\"user_answer\"] * (past_df[\"user_answer\"] >= 0)\n        for user_ix, choice_id, timestamp, cid, part in zip(past_df[\"user_ix\"], past_df[\"choice_id\"], past_df[\"timestamp\"], past_df[\"content_id\"], past_df[\"part\"]):\n            choices[user_ix].append(choice_id)\n            timestamps[user_ix].append(timestamp)\n            elapsed_times[user_ix].append(0 if cid > 13523 else -1)\n            explanations[user_ix].append(0 if cid > 13523 else -1)\n            orders[user_ix] += 1\n            if cid <= 13523:\n                part = int(part)\n                qp_agg_df.at[user_ix, f\"part{part}_count\"] += 1\n                qp_agg_df.at[user_ix, f\"part{part}_sum\"] += int(choice_is_correct[choice_id])\n                qp_agg_df.at[user_ix, f\"question_count\"] += 1\n                qp_agg_df.at[user_ix, f\"question_sum\"] += int(choice_is_correct[choice_id])\n    for user_id in test_raw_df[\"user_id\"].values:\n        if not user_id in user_id_map.keys():\n            user_id_map[user_id] = user_ix_last\n            choices[user_ix_last] = []\n            timestamps[user_ix_last] = []\n            elapsed_times[user_ix_last] = []\n            explanations[user_ix_last] = []\n            orders[user_ix_last] = 0\n            qp_agg_df.loc[user_ix_last, :] = 0\n            user_ix_last += 1\n            \n    get = np.vectorize(lambda x: user_id_map[x])\n    test_raw_df[\"user_ix\"] = get(test_raw_df[\"user_id\"])\n   \n    test_users = test_raw_df[\"user_ix\"].unique()\n    test_questions = test_raw_df[test_raw_df[\"content_type_id\"] == False][\"content_id\"].unique()\n    test_raw_df = test_raw_df.merge(questions_df.loc[test_questions, [\"part\", 'content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', \"idf\"]],\n                                    on=\"content_id\",\n                                    how=\"left\")\n    \n    test_df = test_raw_df[test_raw_df[\"content_type_id\"] == 0]\n    test_df[\"part\"] = test_df[\"part\"].astype(np.int32)\n    test_df = test_df.merge(qp_agg_df.loc[test_users, :],\n                            on=\"user_ix\",\n                            how=\"left\")\n    for user_ix, time, expl in zip(test_df[\"user_ix\"], test_df[\"prior_question_elapsed_time\"], test_df[\"prior_question_had_explanation\"]):\n        if pd.isna(time):\n            continue\n        elapsed_times[user_ix] = [time if (t == -1 and c <= 54095) else t for t, c in zip(elapsed_times.get(user_ix, []), choices.get(user_ix, []))]\n        explanations[user_ix] = [expl if (t == -1 and c <= 54095) else t for t, c in zip(explanations.get(user_ix, []), choices.get(user_ix, []))]\n    \n    order = []\n    choice_history = []\n    delta_timestamp_history = []\n    elapsed_time_history = []\n    explanation_history = []\n    for user_ix in test_df[\"user_ix\"]:\n        order.append(orders.get(user_ix, 0))\n        choice_history.append(choices.get(user_ix, []))\n        delta_timestamp_history.append(timestamps.get(user_ix, []))\n        elapsed_time_history.append(elapsed_times.get(user_ix, []))\n        explanation_history.append(explanations.get(user_ix, []))\n    choice_history = tf.keras.preprocessing.sequence.pad_sequences(choice_history, maxlen=500, dtype=\"int32\")\n    raw_timestamp_history= tf.keras.preprocessing.sequence.pad_sequences(delta_timestamp_history, maxlen=500, value=-1, dtype=\"int64\")\n    raw_elapsed_time_history = tf.keras.preprocessing.sequence.pad_sequences(elapsed_time_history, maxlen=500, value=-1, dtype=\"float32\")\n    explanation_history = tf.keras.preprocessing.sequence.pad_sequences(explanation_history, maxlen=500, value=-1, dtype=\"float32\")\n\n    choice_history = tf.constant(choice_history, dtype=\"int32\")\n    raw_timestamp_history = tf.constant(raw_timestamp_history, dtype=\"int64\")\n    raw_elapsed_time_history = tf.constant(raw_elapsed_time_history, dtype=\"float32\")\n    explanation_history = tf.constant(explanation_history, dtype=\"float32\")\n    \n    user_ix = tf.constant(test_df[\"user_ix\"], dtype=\"int32\")\n    #user_ix = tf.where(user_ix <= user_ix_max, user_ix, tf.constant(0, dtype=\"int32\"))\n    content_id = tf.constant(test_df[\"content_id\"], dtype=\"int32\")\n    timestamp = tf.constant(test_df[\"timestamp\"], dtype=\"int64\")\n    part = tf.constant(test_df[\"part\"], dtype=\"int32\")\n    order = tf.constant(order, dtype=\"int32\")\n    feat = tf.constant(test_df[feat_col].values, dtype=\"float32\")\n    raw_feat = tf.constant(test_df[raw_feat_col].values, dtype=\"float32\")\n    start = timer()\n    input_ = predictor(user_ix, content_id, timestamp, order, choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history, part, feat, raw_feat)\n    print(timer() - start)\n    start = timer()\n    nn_pred = model(*input_, training=False)\n    print(timer() - start)\n    test_df[\"answered_correctly\"] = (np.squeeze(nn_pred.numpy()))\n    past_df = test_raw_df\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    #env.predict(sample_prediction_df)\n    #collect()+\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}