{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nimport time\nfrom tqdm import tqdm\nimport pickle\nfrom pickle import dump, load\nimport gc\nimport matplotlib.pyplot as plt\nimport random\nfrom trueskill import Rating, quality_1vs1, rate_1vs1\nimport math\nimport trueskill","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.models import Model\nfrom sklearn.model_selection import train_test_split\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WIN_SIZE = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_feather(file_name = \"train.feather\"):\n    data = pd.read_feather(file_name)\n    return data\n\ndef read_csv(file_name = \"train.csv\", dtype = None, skiprows = None, nrows = None, usecols = None):\n    data = pd.read_csv(file_name, dtype=dtype, skiprows = skiprows, nrows = nrows, low_memory = True, header = 0, usecols = usecols)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf = read_feather(\"../input/processing-riiid-train-data/all_train_dat_plus.feather\")\ntdf = tdf[['user_id', 'content_id']]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_pickle = open('../input/trueskill-mean-hung/global_mean_trueskill.pkl', 'rb')\nuser_dict = pickle.load(user_pickle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# user_dict[1688853638]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# user_dict[1893691064]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#935388: trueskill.Rating(mu=0.249, sigma=0.041\n#142896: trueskill.Rating(mu=0.238, sigma=0.065),\n#107002: trueskill.Rating(mu=0.618, sigma=0.022),\n#163243: trueskill.Rating(mu=0.197, sigma=0.066),\n#4872589: trueskill.Rating(mu=0.248, sigma=0.063),","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def win_probability(team1, team2):\n    delta_mu = team1.mu - team2.mu\n    sum_sigma = sum([team1.sigma ** 2, team2.sigma ** 2])\n    size = 2\n    denom = math.sqrt(size * (0.05 * 0.05) + sum_sigma)\n    ts = trueskill.global_env()\n    return ts.cdf(delta_mu / denom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf[\"attempt_no\"] = 1\ntdf.attempt_no = tdf.attempt_no.astype('int8')\ntdf[\"attempt_no\"] = tdf[[\"user_id\",\"content_id\",'attempt_no']].groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].cumsum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attempt_no_agg=tdf.groupby([\"user_id\",\"content_id\"])[\"attempt_no\"].agg(['max'])\ndel tdf\ngc.collect()\nattempt_no_agg=attempt_no_agg.astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nattempt_no_agg=attempt_no_agg[attempt_no_agg['max'] > 1]\nu_attempt_c_dict = attempt_no_agg['max'].to_dict(defaultdict(int))\ndel attempt_no_agg\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadAttention(layers.Layer):\n    def __init__(self, embed_dim, num_heads=8, **kwargs):\n        super(MultiHeadAttention, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if self.embed_dim % self.num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {self.embed_dim} should be divisible by number of heads = {self.num_heads}\"\n            )\n        self.projection_dim = self.embed_dim // self.num_heads\n        self.query_dense = layers.Dense(embed_dim)\n        self.key_dense = layers.Dense(embed_dim)\n        self.value_dense = layers.Dense(embed_dim)\n        self.combine_heads = layers.Dense(embed_dim)\n    \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n        })\n        return cfg\n\n    def attention(self, query, key, value, mask):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        if mask is not None:\n            scaled_score += (mask * -1e9)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, q , k ,v, mask):\n        batch_size = tf.shape(q)[0]\n        query = self.query_dense(q)  # (batch_size, seq_len, embed_dim)\n        key = self.key_dense(k)  # (batch_size, seq_len, embed_dim)\n        value = self.value_dense(v)  # (batch_size, seq_len, embed_dim)\n        query = self.separate_heads(query, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        key = self.separate_heads(key, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        value = self.separate_heads(value, batch_size)  # (batch_size, num_heads, seq_len, projection_dim)\n        attention, weights = self.attention(query, key, value, mask)\n        attention = tf.transpose(attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len, num_heads, projection_dim)\n        concat_attention = tf.reshape(attention, (batch_size, -1, self.embed_dim))  # (batch_size, seq_len, embed_dim)\n        output = self.combine_heads(concat_attention)  # (batch_size, seq_len, embed_dim)\n        return output # can return weights\n\n\"\"\"\nEncoder block as a layer\n\"\"\"\n\nclass EncoderBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim = None, rate=0.1, **kwargs):\n        super(EncoderBlock, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        self.ff_dim = ff_dim\n        self.rate = rate\n        self.att = MultiHeadAttention(self.embed_dim, self.num_heads)\n        if self.ff_dim is None: self.ff_dim = 2*self.embed_dim\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(self.ff_dim, activation=\"relu\"), layers.Dense(self.embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(self.rate)\n        self.dropout2 = layers.Dropout(self.rate)\n        \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n            'ff_dim': self.ff_dim,\n            'rate': self.rate\n        })\n        return cfg\n\n    def call(self, x, y, padding_mask, training):\n        attn_output = self.att(x, y, y, padding_mask)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n    \n\"\"\"\nDecoder block as a layer\n\"\"\"\n\nclass DecoderBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim = None, rate = 0.1, **kwargs):\n        super(DecoderBlock, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        self.ff_dim = ff_dim\n        self.rate = rate\n        self.att1 = MultiHeadAttention(self.embed_dim, self.num_heads)\n        self.att2 = MultiHeadAttention(self.embed_dim, self.num_heads)\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(self.ff_dim, activation=\"relu\"), layers.Dense(self.embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm3 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(self.rate)\n        self.dropout2 = layers.Dropout(self.rate)\n        self.dropout3 = layers.Dropout(self.rate)\n        \n    def get_config(self):\n        cfg = super().get_config()\n        cfg.update({\n            'embed_dim': self.embed_dim,\n            'num_heads': self.num_heads,\n            'ff_dim': self.ff_dim,\n            'rate': self.rate\n        })\n        return cfg\n    \n    def call(self, x, enc_output, look_ahead_mask, padding_mask, training):\n        attn1 = self.att1(x, x, x, look_ahead_mask)\n        attn1 = self.dropout1(attn1, training = training)\n        out1 = self.layernorm1(attn1 + x)\n        \n        attn2 = self.att2(out1, enc_output, enc_output, padding_mask)\n        attn2 = self.dropout2(attn2, training = training)\n        out2 = self.layernorm2(attn2 + out1)\n        \n        ffn_output = self.ffn(out2)\n        ffn_output = self.dropout3(ffn_output, training = training)\n        return self.layernorm3(ffn_output + out2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_padding_mask(seqs):\n    mask = tf.cast(tf.reduce_all(tf.math.equal(seqs, 0), axis=-1), tf.float32)\n    return mask[:, tf.newaxis, tf.newaxis, :]\n\ndef create_look_ahead_mask(size):\n    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n    return mask  # (seq_len, seq_len)\n\ndef get_angles(pos, i, embed_dim):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(embed_dim))\n    return pos * angle_rates\n\ndef positional_encoding(position, embed_dim):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                            np.arange(embed_dim)[np.newaxis, :],\n                            embed_dim)\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n    pos_encoding = angle_rads[np.newaxis, ...]\n    return tf.cast(pos_encoding, dtype=tf.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_transformer_model(feature_dim, window_size, q_size=13524, embed_dim = 256, num_heads = 16, dense_dim = 1024):\n    inputs = layers.Input(shape=(window_size, feature_dim), name = \"enc_input\")\n    min_delta = inputs[...,0]\n    day_delta = inputs[...,1]\n    month_delta = inputs[...,2]\n    cid = inputs[...,3]\n    tid = inputs[...,4]\n    prior_elapsed = inputs[...,5]\n    prior_explained = inputs[...,6]\n    is_with = inputs[...,7]\n    num_lect = inputs[...,-1,8]\n    lec_type = inputs[...,-1,9:13]\n    lec_h_past = inputs[...,-1,13]\n    c_part = inputs[...,14:23]\n    tag1 = inputs[...,23]\n    tag2 = inputs[...,24]\n    tag3 = inputs[...,25]\n    tag4 = inputs[...,26]\n    tag5 = inputs[...,27]\n    tag6 = inputs[...,28]\n    prev_answered_correct = inputs[...,29]\n    \n    #====Excercise====\n    min_delta = layers.Embedding(input_dim=1443, output_dim=embed_dim//8, input_length=window_size,\n                                 embeddings_initializer = 'glorot_uniform')(min_delta)\n    day_delta = layers.Embedding(input_dim=33, output_dim=embed_dim//16, input_length=window_size,\n                                 embeddings_initializer = 'glorot_uniform')(day_delta)\n    month_delta = layers.Embedding(input_dim=9, output_dim=embed_dim//16, input_length=window_size,\n                                   embeddings_initializer = 'glorot_uniform')(month_delta)\n    cid = layers.Embedding(input_dim=q_size, output_dim=embed_dim, input_length=window_size,\n                           embeddings_initializer = 'glorot_uniform')(cid)\n    tid = layers.Embedding(input_dim=2001, output_dim=embed_dim//16, input_length=window_size,\n                           embeddings_initializer = 'glorot_uniform')(tid)\n    is_with = layers.Embedding(input_dim=3, output_dim=2, input_length=window_size,\n                               embeddings_initializer = 'glorot_uniform')(is_with)\n    c_part = layers.Dense(embed_dim//4, activation = 'relu', use_bias=False)(c_part)\n#     tag_emb = layers.Embedding(input_dim=189, output_dim=embed_dim//4)\n    tag1 = layers.Embedding(input_dim=189, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag1)\n    tag2 = layers.Embedding(input_dim=179, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag2)\n    tag3 = layers.Embedding(input_dim=162, output_dim=embed_dim//8, input_length=window_size,\n                            embeddings_initializer = 'glorot_uniform')(tag3)\n#     tag4 = tag_emb(tag4)\n#     tag5 = tag_emb(tag5)\n#     tag6 = tag_emb(tag6)\n    enc_ex = layers.Concatenate()([min_delta, day_delta, month_delta, tid, c_part,\n                                 tag1, tag2, tag3, is_with]) #tag4, tag5, tag6\n    enc_ex = layers.Dense(embed_dim, activation = 'relu')(enc_ex)\n    \n    #====Lecture====\n    num_lect = layers.Embedding(input_dim=160, output_dim=embed_dim//16,\n                                embeddings_initializer = 'glorot_uniform')(num_lect)\n    lec_type = layers.Dense(embed_dim//8, activation = 'relu', use_bias=False)(lec_type)\n    lec_h_past = layers.Embedding(input_dim=724, output_dim=embed_dim//8,\n                                  embeddings_initializer = 'glorot_uniform')(lec_h_past)\n    enc_lec = layers.Concatenate()([num_lect, lec_type, lec_h_past])\n    enc_lec = layers.Dense(embed_dim//2, activation = 'relu')(enc_lec)\n    enc_lec = layers.Dropout(0.1)(enc_lec)\n\n    #====Response====\n    prev_answered_correct = layers.Embedding(input_dim=4, output_dim=embed_dim, input_length=window_size,\n                                             embeddings_initializer = 'glorot_uniform')(prev_answered_correct)\n    prior_elapsed = layers.Embedding(input_dim=302, output_dim=embed_dim//4, input_length=window_size,\n                                     embeddings_initializer = 'glorot_uniform')(prior_elapsed)\n    prior_explained = layers.Embedding(input_dim=3, output_dim=embed_dim//4, input_length=window_size,\n                                       embeddings_initializer = 'glorot_uniform')(prior_explained)\n    prior_inter = layers.Concatenate()([prior_elapsed, prior_explained])\n    prior_inter = layers.Dense(embed_dim, activation = 'relu')(prior_inter)\n    \n    #====Mask====\n    padding_mask = create_padding_mask(inputs)\n    look_ahead_mask = create_look_ahead_mask(window_size)\n    dec_combined_mask = tf.maximum(padding_mask, look_ahead_mask)\n    pos_enc = positional_encoding(window_size, embed_dim)\n    \n    #++++Model++++\n    e_enc_input = layers.Add()([cid, pos_enc, enc_ex])\n    dec_input = layers.Add()([prev_answered_correct, pos_enc, prior_inter])\n    \n    x1 = EncoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(e_enc_input, e_enc_input, padding_mask)\n    x1 = EncoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(x1, x1, padding_mask)\n    x1 = layers.Add()([e_enc_input, x1])\n    x3 = DecoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(dec_input, x1,\n                                                                dec_combined_mask, padding_mask)\n    x3 = DecoderBlock(embed_dim, num_heads, ff_dim = dense_dim)(x3, x1,\n                                                                dec_combined_mask, padding_mask)\n    x = x3[:, -1, :]\n    x = layers.Concatenate()([x, enc_lec])\n    x = layers.Dense(embed_dim, activation=\"relu\")(x)\n#     x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.2)(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\",\n                           kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),\n                           name = \"output\")(x)\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = custom_transformer_model(30, WIN_SIZE, embed_dim = 128, dense_dim = 512, num_heads=8)\nmodel.load_weights(\"../input/model-sak/transformer_2seq_8head.03.h5\")\nprint('Load neural network done.')\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf = read_feather(\"../input/processing-riiid-train-data/all_train_dat_plus.feather\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf.numlect = tdf.numlect.astype(np.uint8)\ntdf.task_container_id[tdf.task_container_id > 2000] = 2000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf = tdf.groupby('user_id').tail(WIN_SIZE)\ngc.collect()\ntdf.reset_index(drop = True, inplace = True)\nprint(tdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adf = tdf.groupby('user_id')\nusers_dict = adf.groups\ndel adf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tdf.drop(columns = ['user_id'], inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_df = {uid: tdf[users_dict[uid][0]:users_dict[uid][-1]+1].copy().to_numpy(copy = True) for uid in tqdm(user_dict)}\ndel tdf\ndel users_dict\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##TODO 수정: lgbm_model_new_mu_sigma_hung_trueskill_mean.txt\nlgbm_model = lgb.Booster(model_file=\"../input/new-mu-3rd-train/lgbm_model_3rd_train_new_mu_sigma_hung_trueskill_mean.txt\")\nprint('Load lgbm done.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_time_series(x, windows_size):\n    pad_size = x.shape[0]\n    x = np.pad(x, [[ windows_size-pad_size, 0], [0, 0]], constant_values=0)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION_DTYPES = {\n    'question_id': np.uint16,\n    'bundle_id': np.uint16,\n    'correct_answer': np.int8,\n    'part': np.int8,\n    'tags': str\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainlec_dict = load(open('../input/feather-data/trainlec_dict.pkl', 'rb'))\ndf_q = read_feather(\"../input/feather-data/questions_processed.feather\")\ndf_lec = read_feather(\"../input/feather-data/lectures_processed.feather\")\nlec_pos = {}\npart_df = read_csv(file_name = '../input/riiid-test-answer-prediction/questions.csv', dtype = QUESTION_DTYPES)\npart_df.drop(columns = ['question_id', 'bundle_id', 'correct_answer', 'tags'], inplace=True)\nbundle_df = read_feather('../input/riiid-data-processing4/questions_processed.feather')\nbundle_df = bundle_df[['bundle_id']]\ngc.collect()\nfor i in range(df_lec.shape[0]):\n    lec_pos[df_lec['lecture_id'][i]] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c_percent_dict = load(open('../input/riiid-data-processing-lgbm2/c_percent_dict.pkl', 'rb'))\nu_percent_dict = load(open('../input/riiid-data-processing-lgbm2/u_percent_dict.pkl', 'rb'))\nt_percent_dict = load(open('../input/riiid-data-processing-lgbm2/t_percent_dict.pkl', 'rb'))\nb_means_dict = load(open('../input/feather-data/b_mean_elapsed_dict.pkl', 'rb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# content_dict = {}\n# for key, value in c_percent_dict.items():\n#     #content_dict[key] = value[0]/value[1]\n#     content_dict[key][0] = value[0]\n#     content_dict[key][1] = value[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# content_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_tag_df = pd.DataFrame(df_q[df_q[['t1','t2','t3','t4','t5','t6']] > 1].T.count(), columns = ['num_tag'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nfrom bisect import bisect\nimport json\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (tdf, sample_prediction_df) in iter_test:\n#     start = time.time()\n    prev_correct = json.loads(tdf[\"prior_group_answers_correct\"].iloc[0])\n    if len(prev_correct) != 0:\n        i = -1\n        for cor in prev_correct:\n            if cor == -1: continue\n            i += 1\n            prv_udat = np.array([prev_dat[i].copy()+[cor]], dtype = np.int64)\n            prv_uid = prev_uid[i]\n            prv_cid = prev_cid[i][0]\n            prv_part = prev_cid[i][1]\n            prv_t1 = prev_cid[i][2]\n            prv_skill = 'lis'\n            if prv_part > 4: prv_skill = 'read'\n            if not prv_uid in u_percent_dict:\n                u_percent_dict[prv_uid] = {'cor':0, 'tot':0, 1:[0,0], 2:[0,0], 3:[0,0], 4:[0,0], 5:[0,0], 6:[0,0], 7:[0,0],\n                                           'lis':[0,0], 'read':[0,0], 'mean_gap':0, 'total_explained':0}\n            c_percent_dict[prv_cid][1] += 1\n            t_percent_dict[prv_t1][1] += 1\n            u_percent_dict[prv_uid]['tot'] += 1\n            u_percent_dict[prv_uid][prv_part][1] += 1\n            u_percent_dict[prv_uid][prv_skill][1] += 1\n            if cor == 1:\n                c_percent_dict[prv_cid][0] += 1\n                new_user_rating, new_question_rating = rate_1vs1(user_dict[prv_uid], trueskill.setup(mu=1 - (c_percent_dict[prv_cid][0] / c_percent_dict[prv_cid][1]),\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,\n                                                                      draw_probability=0).Rating())\n                u_percent_dict[prv_uid]['cor'] += 1\n                t_percent_dict[prv_t1][0] += 1\n                u_percent_dict[prv_uid][prv_part][0] += 1\n                u_percent_dict[prv_uid][prv_skill][0] += 1\n            if cor == 0:\n                new_question_rating, new_user_rating = rate_1vs1(trueskill.setup(mu=1 - (c_percent_dict[prv_cid][0] / c_percent_dict[prv_cid][1]),\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,\n                                                                      draw_probability=0).Rating(), user_dict[prv_uid])\n            if prv_uid in user_df:\n                prv_udat = np.concatenate([user_df[prv_uid], prv_udat], axis = 0)[-100:]\n            user_df[prv_uid] = prv_udat\n            user_dict[prv_uid] = new_user_rating\n    tdf['prior_question_had_explanation'].fillna(0, inplace = True)\n    tdf['prior_question_had_explanation'] = tdf['prior_question_had_explanation'].astype(np.int8)\n    tdf['prior_question_had_explanation'] = tdf['prior_question_had_explanation'] + 1\n    tdf['prior_question_elapsed_time'].fillna(23000, inplace = True) #mean: 25300 med: 21000\n    tdf['prior_question_elapsed_time'] = (tdf['prior_question_elapsed_time'] // 1000 + 1)\n    tdf['content_id'] = tdf['content_id'] + 1\n    tdf['task_container_id'] = tdf['task_container_id'] + 1\n#     print(time.time() - start)\n    \n    prev_dat = []\n    prev_uid = []\n    prev_cid = []\n    total_dat = []\n    temp_udat = []\n    row_id = []\n    total_lgbm_dat = []\n    for i in range(tdf.shape[0]):\n        ##For questions\n        can_if = False\n        if tdf['content_type_id'].iloc[i] == 0:\n            user_id = tdf['user_id'].iloc[i]\n            question_id = tdf['content_id'].iloc[i] - 1\n            if(user_id not in user_dict.keys()):\n                user_dict[user_id] = trueskill.setup(mu=0.3,\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,\n                                                                      draw_probability=0).Rating()\n            prob = win_probability(user_dict[user_id], trueskill.setup(mu=1 - (c_percent_dict[question_id][0] / c_percent_dict[question_id][1]),\n                                                                      sigma=0.164486,\n                                                                      beta=0.05, tau=0.00164,\n                                                                      draw_probability=0).Rating())\n            mu = user_dict[user_id].mu\n            sigma = user_dict[user_id].sigma\n            temp_udat = tdf.iloc[i].to_list()[:-2]\n            row_id.append(temp_udat.pop(0))\n            ts_cur = temp_udat[0]\n            uid = temp_udat.pop(1)\n            cid = temp_udat[1]\n            temp_udat.pop(2)\n            lgbm_udat = temp_udat[2:].copy()\n            lgbm_udat[0] -= 1\n            lgbm_udat[1] -= 1\n            lgbm_udat[2] -= 1\n            temp_udat[2] = min(2000, temp_udat[2])\n            assert df_q['question_id'][cid-1] == cid, 'hmm somethings wrong ' + str(df_q['question_id'][cid]) + \" \" + str(cid)\n            if (i-1 >= 0 and tdf['task_container_id'].iloc[i] == tdf['task_container_id'].iloc[i-1] \n                and tdf['user_id'].iloc[i] == tdf['user_id'].iloc[i-1]):\n                temp_udat.append(2)\n            elif (i+1 < tdf.shape[0] and tdf['task_container_id'].iloc[i] == tdf['task_container_id'].iloc[i+1] \n                  and tdf['user_id'].iloc[i] == tdf['user_id'].iloc[i+1]):\n                temp_udat.append(2)\n            else: temp_udat.append(1)\n            part = part_df['part'][cid-1]\n            bundle = bundle_df['bundle_id'][cid-1]\n            t1 = df_q['t1'][cid-1]\n            num_tag = num_tag_df['num_tag'][cid - 1]\n            lgbm_udat.append(part)\n            lgbm_udat.append(t1)\n            lgbm_udat.append(num_tag)\n            skill = 'lis'\n            if part > 4: skill = 'read'\n            if not uid in trainlec_dict: \n                temp_udat += [1,0,0,0,0,1]\n            else:\n                if not part in trainlec_dict[uid]:\n                    temp_udat += [1,0,0,0,0,1]\n                else:\n                    idx_cur = bisect(trainlec_dict[uid][part]['ts'], ts_cur) - 1\n                    if idx_cur == -1:\n                        temp_udat += [1,0,0,0,0,1]\n                    else:\n                        num_lect = min(159, len(trainlec_dict[uid][part]['lt']) + 1)\n                        tlast = [0]*4\n                        tlast[trainlec_dict[uid][part]['lt'][idx_cur]] = 1\n                        tsago = min((ts_cur - trainlec_dict[uid][part]['ts'][idx_cur])//3600000 + 2, 723)\n                        temp_udat.append(num_lect)\n                        temp_udat += tlast\n                        temp_udat.append(tsago)\n            q_fet = df_q.iloc[cid-1].to_list()\n            q_fet.pop(0)\n            temp_udat += q_fet\n            if not uid in user_df:\n                ts_fet = [1, 1, 1]\n                #===========u_attemp_c===============\n                lgbm_udat.append(0)\n                u_attempt_c_dict[(uid, cid)] = 1\n                #====================================\n                temp_udat = ts_fet + temp_udat\n                infer_dat = np.array([temp_udat+ [3]])\n                infer_dat = make_time_series(infer_dat, WIN_SIZE)\n                lgbm_udat[1] = -1\n                lgbm_udat[2] = -1\n                lgbm_udat = [-1] + lgbm_udat\n                lgbm_udat = lgbm_udat + [0, 0, 0, 0, 0, b_means_dict[bundle][0], -1, 0]\n                lgbm_udat = lgbm_udat + [c_percent_dict[cid-1][0] / c_percent_dict[cid-1][1], c_percent_dict[cid-1][1]]\n                lgbm_udat = lgbm_udat + [-1, 0, -1, 0]\n                lgbm_udat = lgbm_udat + [t_percent_dict[t1][0] / t_percent_dict[t1][1], c_percent_dict[t1][1]]\n                lgbm_udat = lgbm_udat + [-1, 0]\n            else:\n                #===========u_attemp_c===============\n                if (uid, cid) in u_attempt_c_dict:\n                    lgbm_udat.append(u_attempt_c_dict[(uid, cid)])\n                    u_attempt_c_dict[(uid, cid)] += 1\n                else:\n                    attempt_recent = user_df[uid][user_df[uid][:,4] == cid].shape[0]\n                    lgbm_udat.append(attempt_recent)\n                    u_attempt_c_dict[(uid, cid)] = attempt_recent + 1\n                #====================================\n                ts_prev = user_df[uid][-1,3]\n                ts_recent_prev10 = user_df[uid][0,3]\n                if user_df[uid].shape[0] > 9:\n                    ts_recent_prev10 = user_df[uid][-9,3]\n                ts_diff = ts_cur - ts_prev\n                min_del = min(1440, ts_diff//60000) + 2\n                day_del = min(30, ts_diff//86400000) + 2\n                mon_del = min(6, ts_diff//2592000000) + 2\n                ts_fet = [min_del, day_del, mon_del]\n                temp_udat = ts_fet + temp_udat\n                infer_dat = np.concatenate([user_df[uid], np.array([temp_udat + [2]])])\n                infer_dat[:,-1] = infer_dat[:,-1] + 1\n                infer_dat[:,-1] = np.roll(infer_dat[:, -1], 1)\n                if infer_dat.shape[0] > 100: infer_dat = infer_dat[-100:]\n                if infer_dat.shape[0] < 100: infer_dat = make_time_series(infer_dat, WIN_SIZE)\n                if lgbm_udat[2] == 1: u_percent_dict[uid]['total_explained'] += 1\n                if lgbm_udat[0] != 0: u_percent_dict[uid]['mean_gap'] = ts_cur / lgbm_udat[0]\n                lgbm_udat = [ts_diff//1000] + lgbm_udat\n                recent_10_correctness = user_df[uid][-10:, -1].sum()/(min(10, user_df[uid].shape[0]))\n                recent_10_mean_gap = (ts_cur - ts_recent_prev10)/((min(9, user_df[uid].shape[0])+1)*1000)\n                mean_elapsed = (lgbm_udat[3] + user_df[uid][:, 6].sum() - 23) / (user_df[uid].shape[0])\n                lgbm_udat = lgbm_udat + [u_percent_dict[uid]['total_explained'], recent_10_correctness, recent_10_mean_gap,\n                                         mean_elapsed, u_percent_dict[uid]['mean_gap']//1000, b_means_dict[bundle][0]]\n                lgbm_udat = lgbm_udat + [u_percent_dict[uid]['cor'] / u_percent_dict[uid]['tot'], u_percent_dict[uid]['tot']]\n                lgbm_udat = lgbm_udat + [c_percent_dict[cid-1][0] / c_percent_dict[cid-1][1], c_percent_dict[cid-1][1]]\n                try:\n                    u_part_chance = u_percent_dict[uid][part][0] / u_percent_dict[uid][part][1]\n                    if u_percent_dict[uid][part][1] > 15: can_if = True\n                except:\n                    u_part_chance = -1\n                try:\n                    u_skill_chance = u_percent_dict[uid][skill][0] / u_percent_dict[uid][skill][1]\n                except:\n                    u_skill_chance = -1\n                lgbm_udat = lgbm_udat + [u_part_chance, u_percent_dict[uid][part][1], u_skill_chance, u_percent_dict[uid][skill][1]]\n                lgbm_udat = lgbm_udat + [t_percent_dict[t1][0] / t_percent_dict[t1][1], c_percent_dict[t1][1]]\n                lgbm_udat = lgbm_udat + [user_df[uid][-1,-1], user_df[uid][-1,24]]\n            total_dat.append(infer_dat)\n            lgbm_udat.append(prob)\n            lgbm_udat.append(mu)\n            lgbm_udat.append(sigma)\n            total_lgbm_dat.append(lgbm_udat)\n            prev_dat.append(temp_udat)\n            prev_uid.append(uid)\n            prev_cid.append([cid-1,part,t1])\n        ##For lectures\n        else:\n            uid = tdf['user_id'].iloc[i]\n            if not uid in trainlec_dict: trainlec_dict[uid] = {}\n            lid = tdf['content_id'].iloc[i] - 1\n            pos = lec_pos[lid]\n            part = df_lec['part'][pos]\n            tag = df_lec['tag'][pos]\n            ltype = np.argmax(df_lec[['ty1', 'ty2', 'ty3', 'ty4']][pos : pos+1].to_numpy())\n            if not part in trainlec_dict[uid]: trainlec_dict[uid][part] = {'tt':[], 'lt':[], 'ts':[]}\n            trainlec_dict[uid][part]['tt'].append(tag)\n            trainlec_dict[uid][part]['lt'].append(ltype)\n            trainlec_dict[uid][part]['ts'].append(tdf['timestamp'].iloc[i])\n    total_dat = np.delete(np.array(total_dat), 3, axis = 2)\n    pred0 = model.predict(total_dat)\n    pred1 = lgbm_model.predict(total_lgbm_dat)\n    pred0 = np.squeeze(pred0, axis = -1)\n    pred0 = list(pred1)\n    #pred0 = list(pred1)\n    pred_df = pd.DataFrame({\"row_id\": row_id, \"answered_correctly\": pred0}, columns = [\"row_id\", \"answered_correctly\"])\n    env.predict(pred_df)\n#     print(time.time() - start)\n#     print(\"=======\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}