{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:06:17.877069Z","start_time":"2021-03-03T13:06:16.378071Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:02.298438Z","iopub.status.busy":"2021-03-03T18:56:02.297756Z","iopub.status.idle":"2021-03-03T18:56:02.303842Z","shell.execute_reply":"2021-03-03T18:56:02.303025Z"},"papermill":{"duration":0.027971,"end_time":"2021-03-03T18:56:02.304042","exception":false,"start_time":"2021-03-03T18:56:02.276071","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport gc\n\nfrom tqdm.notebook import tqdm\n\n\nrandom.seed(1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-03T18:56:02.3374Z","iopub.status.busy":"2021-03-03T18:56:02.336675Z","iopub.status.idle":"2021-03-03T18:56:11.488395Z","shell.execute_reply":"2021-03-03T18:56:11.487847Z"},"papermill":{"duration":9.17006,"end_time":"2021-03-03T18:56:11.488555","exception":false,"start_time":"2021-03-03T18:56:02.318495","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install adabelief-tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\",\n                       usecols=[2, 3, 4, 6],\n                       dtype={\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'user_answer': 'int8',\n                              }\n                      )\nlectures_df = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\nquestions_df = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_id_map = {id_: i+1 for i, id_ in enumerate(questions_df[\"question_id\"])}\nlecture_id_map = {id_: i+questions_df.shape[0]+1 for i, id_ in enumerate(lectures_df[\"lecture_id\"])}\nquestions_df[\"content_id\"] = questions_df[\"question_id\"].map(question_id_map)\nlectures_df[\"content_id\"] = lectures_df[\"lecture_id\"].map(lecture_id_map)\n\ntrain_df.loc[train_df[\"user_answer\"] != -1, \"content_id\"] = train_df.loc[train_df[\"user_answer\"] != -1, \"content_id\"].map(question_id_map)\ntrain_df.loc[train_df[\"user_answer\"] == -1, \"content_id\"] = train_df.loc[train_df[\"user_answer\"] == -1, \"content_id\"].map(lecture_id_map)\n\ntrain_df[\"choice_id\"] = train_df[\"content_id\"].astype(np.int32)*4 + train_df[\"user_answer\"].astype(np.int32) * (train_df[\"user_answer\"] >= 0)\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:06:48.354648Z","start_time":"2021-03-03T13:06:36.614751Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:11.943911Z","iopub.status.busy":"2021-03-03T18:56:11.943208Z","iopub.status.idle":"2021-03-03T18:56:20.359321Z","shell.execute_reply":"2021-03-03T18:56:20.357984Z"},"papermill":{"duration":8.443886,"end_time":"2021-03-03T18:56:20.35949","exception":false,"start_time":"2021-03-03T18:56:11.915604","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nchoice_parts = [0] * ((lectures_df[\"content_id\"].max()+1) * 4)\nchoice_tags = [[] for _ in range((lectures_df[\"content_id\"].max()+1) * 4)]\ncorrecr_answers = [-1] * ((lectures_df[\"content_id\"].max()+1) * 4)\n\nfor i, row in questions_df.iterrows():\n    tags = [] if pd.isna(row[\"tags\"]) else list(map(int, row[\"tags\"].split()))\n    for i in range(4):\n        choice_tags[i + row[\"content_id\"]*4] = [t for t in tags]\n        choice_parts[i + row[\"content_id\"]*4] = row[\"part\"]\n        correcr_answers[i + row[\"content_id\"]*4] = row[\"correct_answer\"]\n        \nfor i, row in lectures_df.iterrows():\n    tags = [row[\"tag\"]]\n    for i in range(4):\n        choice_tags[i + row[\"content_id\"]*4] = [t for t in tags]\n        choice_parts[i + row[\"content_id\"]*4] = row[\"part\"]\n        \nchoice_parts = tf.constant(choice_parts)\ncorrecr_answers = tf.constant(correcr_answers)\nchoice_tags = tf.keras.preprocessing.sequence.pad_sequences(choice_tags, dtype=\"int16\", value=-1, padding=\"post\") + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_of_records = (lectures_df[\"content_id\"].max() + 1) * 4\nrecords = []\nrecords_ixs = {}\nfor i, (user_id, df) in tqdm(enumerate(train_df.groupby(\"user_id\")), total=train_df[\"user_id\"].nunique()):\n    records.append(np.int32(np.concatenate([[start_of_records], df[\"choice_id\"].values])))\n    records_ixs[user_id] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gc import collect\ndel train_df\ncollect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_user, test_user = train_test_split(list(records_ixs.keys()), test_size=.1, random_state=2021, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model definition"},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:41:39.812816Z","start_time":"2021-03-03T13:41:39.79282Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:20.405613Z","iopub.status.busy":"2021-03-03T18:56:20.404655Z","iopub.status.idle":"2021-03-03T18:56:20.40707Z","shell.execute_reply":"2021-03-03T18:56:20.407529Z"},"papermill":{"duration":0.030184,"end_time":"2021-03-03T18:56:20.407704","exception":false,"start_time":"2021-03-03T18:56:20.37752","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from random import randint\nclass Sampler():\n    def __init__(self, records, records_ixs):\n        self.records = records\n        self.records_ixs = records_ixs\n    \n    def stream(self, raw_users, window=32, batch_size=64):\n        users = np.copy(raw_users)\n        np.random.shuffle(users)\n        X = []\n        y = []\n        for user in users:\n            vec = self.records[self.records_ixs[user]]\n            ix = randint(1, len(vec)-1)\n            \n            X.append(vec[max(ix-window, 0):ix])\n            y.append(vec[ix])\n            if len(X) == batch_size:\n                yield X, y\n                X = []\n                y = []\n        return\n                ","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:41:40.248854Z","start_time":"2021-03-03T13:41:40.238852Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:20.451945Z","iopub.status.busy":"2021-03-03T18:56:20.451049Z","iopub.status.idle":"2021-03-03T18:56:20.453754Z","shell.execute_reply":"2021-03-03T18:56:20.454219Z"},"papermill":{"duration":0.029074,"end_time":"2021-03-03T18:56:20.454386","exception":false,"start_time":"2021-03-03T18:56:20.425312","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class Tokenizer(tf.keras.Model):\n    def __init__(self, emb_dim):\n        super(Tokenizer, self).__init__()\n        self.content_embedding = tf.keras.layers.Embedding(13943, emb_dim, mask_zero=True)\n        \n    def call(self, choice_ids):\n        return self.content_embedding(choice_ids//4)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:41:40.936188Z","start_time":"2021-03-03T13:41:40.907147Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:20.506691Z","iopub.status.busy":"2021-03-03T18:56:20.505969Z","iopub.status.idle":"2021-03-03T18:56:20.508277Z","shell.execute_reply":"2021-03-03T18:56:20.508778Z"},"papermill":{"duration":0.036542,"end_time":"2021-03-03T18:56:20.508948","exception":false,"start_time":"2021-03-03T18:56:20.472406","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class Encoder(tf.keras.Model):\n    def __init__(self, num_layer, emb_dim):\n        super(Encoder, self).__init__()\n        self.num_layer = num_layer\n        \n        self.dense_layers = []\n        self.lstm_layers = []\n        \n        for i in range(num_layer):\n            self.dense_layers.append(tf.keras.layers.Dense(emb_dim))\n            self.lstm_layers.append(tf.keras.layers.LSTM(emb_dim, return_sequences=True))\n\n\n    def create_mask(self, window, batch_size):\n        \"\"\"\n        mask: 自身と自身より先のレコードを参照しないためのマスク\n        start_mask: 一番最初のレコードはどこも参照できないので、全てのweightを0にするためのmask\n        \"\"\"\n        \n        row = tf.reshape(tf.repeat(tf.range(window), window), (window, window))\n        col = tf.transpose(row)\n        raw_mask = tf.expand_dims(tf.where(row <= col, tf.float32.min / 100, 0.) + tf.eye(window, dtype=\"float64\") * tf.float32.max / 1000, axis=0)\n        mask = tf.concat([raw_mask for _ in range(batch_size)], axis=0)\n        mask = tf.cast(mask, \"float32\")\n        \n        start_mask = tf.where(tf.reshape(tf.range(window**2), (window, window)) > 0, 1., 0.)\n        start_mask = tf.concat([tf.expand_dims(start_mask, axis=0) for _ in range(batch_size)], axis=0)\n        \n        return mask, start_mask\n\n    @tf.function\n    def call(self, X, batch_size, window):\n        mask, start_mask = self.create_mask(window, batch_size)\n        for i in range(self.num_layer):\n            X = X + self.lstm_layers[i](X)\n            \n            query = X / tf.stop_gradient(tf.norm(X, axis=2, keepdims=True))\n            key = X\n            value = X\n            \n            logit = tf.matmul(query, key, transpose_b=True) + mask\n            weight = tf.math.softmax(logit) * start_mask\n            attention = tf.matmul(weight, value)\n            X = X + self.dense_layers[i](attention)\n        return X","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:41:41.696803Z","start_time":"2021-03-03T13:41:41.678088Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:20.548374Z","iopub.status.busy":"2021-03-03T18:56:20.547768Z","iopub.status.idle":"2021-03-03T18:56:20.553768Z","shell.execute_reply":"2021-03-03T18:56:20.55436Z"},"papermill":{"duration":0.027326,"end_time":"2021-03-03T18:56:20.554544","exception":false,"start_time":"2021-03-03T18:56:20.527218","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class TopModel(tf.keras.Model):\n    def __init__(self, emb_dim):\n        super(TopModel, self).__init__()\n        self.lstm_layer = tf.keras.layers.LSTM(emb_dim)\n        self.cross_layer = tf.keras.layers.Dense(emb_dim)\n        self.conv_layer = tf.keras.layers.Conv1D(emb_dim, 1, activation=\"relu\")\n        self.gmp_layer = tf.keras.layers.GlobalMaxPooling1D()\n        \n        self.gap = tf.keras.layers.GlobalAveragePooling1D()\n    \n    @tf.function\n    def call(self, X, exist_mask):\n        X_mean = self.gap(X, exist_mask)\n        X_lstm = self.lstm_layer(X)\n        X_conv = self.gmp_layer(self.conv_layer(X))\n        \n        return X_mean * self.cross_layer(tf.concat([X_lstm, X_conv], axis=-1)) + X_mean","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:48:05.829931Z","start_time":"2021-03-03T13:48:05.812227Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:20.599348Z","iopub.status.busy":"2021-03-03T18:56:20.598629Z","iopub.status.idle":"2021-03-03T18:56:20.601669Z","shell.execute_reply":"2021-03-03T18:56:20.601161Z"},"papermill":{"duration":0.028976,"end_time":"2021-03-03T18:56:20.601811","exception":false,"start_time":"2021-03-03T18:56:20.572835","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class Model(tf.keras.Model):\n    def __init__(self, tokenizer, encoder, top_model):\n        super(Model, self).__init__()\n        self.tokenizer = tokenizer\n        self.encoder = encoder\n        self.top_model = top_model\n    \n    @tf.function\n    def call(self, X, y, exist_mask, target_mask, window, batch_size, num_split):\n        X = self.tokenizer(X)\n        X = self.encoder(X, window=window, batch_size=batch_size)\n        X = self.top_model(X, exist_mask)\n\n        y = self.tokenizer(y)\n\n        logits = []\n        for X_split, y_split in zip(tf.split(X, num_or_size_splits=num_split, axis=0), tf.split(y, num_or_size_splits=num_split, axis=0)):\n            logit = tf.matmul(X_split, y_split, transpose_b=True)\n            logits.append(logit)\n        \n        return tf.keras.activations.linear(tf.concat(logits, axis=0) + target_mask)\n        #pred = tf.math.softmax(tf.concat(logits, axis=0) + target_mask)\n\n        #return pred        \n        ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-03T18:56:20.693815Z","iopub.status.busy":"2021-03-03T18:56:20.693161Z","iopub.status.idle":"2021-03-03T18:56:21.704654Z","shell.execute_reply":"2021-03-03T18:56:21.704082Z"},"papermill":{"duration":1.034854,"end_time":"2021-03-03T18:56:21.7048","exception":false,"start_time":"2021-03-03T18:56:20.669946","status":"completed"},"tags":[],"trusted":true},"cell_type":"markdown","source":"# training"},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:54:35.924011Z","start_time":"2021-03-03T13:54:35.832247Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:21.858722Z","iopub.status.busy":"2021-03-03T18:56:21.857799Z","iopub.status.idle":"2021-03-03T18:56:21.935322Z","shell.execute_reply":"2021-03-03T18:56:21.934701Z"},"papermill":{"duration":0.10227,"end_time":"2021-03-03T18:56:21.935478","exception":false,"start_time":"2021-03-03T18:56:21.833208","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"emb_dim = 64\n\nsampler = Sampler(records, records_ixs)\ntokenizer = Tokenizer(emb_dim=emb_dim)\nencoder = Encoder(num_layer=8, emb_dim=emb_dim)\ntop_model = TopModel(emb_dim=emb_dim)\nmodel = Model(tokenizer, encoder, top_model)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2021-03-03T13:54:36.271353Z","start_time":"2021-03-03T13:54:36.245353Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:21.980373Z","iopub.status.busy":"2021-03-03T18:56:21.979688Z","iopub.status.idle":"2021-03-03T18:56:22.008218Z","shell.execute_reply":"2021-03-03T18:56:22.008777Z"},"papermill":{"duration":0.05413,"end_time":"2021-03-03T18:56:22.008955","exception":false,"start_time":"2021-03-03T18:56:21.954825","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from adabelief_tf import AdaBeliefOptimizer\nloss_object = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\noptimizer = AdaBeliefOptimizer(learning_rate=1e-3, weight_decay=1e-4) \n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')\ntest_loss = tf.keras.metrics.Mean(name='test_loss')\ntest_rank = tf.keras.metrics.Mean(name=\"test_rank\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_user)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_user)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"start_time":"2021-03-03T13:55:34.606Z"},"execution":{"iopub.execute_input":"2021-03-03T18:56:22.066932Z","iopub.status.busy":"2021-03-03T18:56:22.06587Z","iopub.status.idle":"2021-03-04T02:12:30.857696Z","shell.execute_reply":"2021-03-04T02:12:30.858314Z"},"papermill":{"duration":26168.830131,"end_time":"2021-03-04T02:12:30.859295","exception":false,"start_time":"2021-03-03T18:56:22.029164","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from time import time\n\nwindow = 32\nbatch_size = 2**13\nnum_split = 2**3\nstart_time = time()\ntrain_losses = []\ntest_losses = []\ntest_ranks = []\n\ndef padding_sequence(X, y):\n    X = tf.keras.preprocessing.sequence.pad_sequences(X, dtype=\"int32\", value=0, padding=\"pre\")\n    y = tf.constant(y, dtype=\"int32\")\n    return X, y\n\n@tf.function\ndef preprocess(X, y, num_split):   \n    exist_mask = X > 0\n\n    target_masks = []\n    targets = []\n    for y_split in tf.split(y, num_or_size_splits=num_split, axis=0):\n        target = tf.eye(len(y_split))\n        row = tf.reshape(tf.repeat(y_split, len(y_split)), (len(y_split), len(y_split)))\n        col = tf.transpose(row)\n        target_mask = tf.cast(tf.where(tf.logical_and(target != 1., row == col), tf.float32.min / 100, 0), \"float32\")\n        \n        targets.append(target)\n        target_masks.append(target_mask)\n    \n    target = tf.concat(targets, axis=0)\n    target_mask = tf.concat(target_masks, axis=0)\n    \n    return X, y, target, exist_mask, target_mask\n\n@tf.function\ndef train_step(X, y, exist_mask, target_mask, window, batch_size, num_split):\n    with tf.GradientTape() as tape:\n        pred = model(X, y, exist_mask, target_mask, window, batch_size, num_split)\n        loss = loss_object(target, pred)\n    gradients = tape.gradient(loss, model.trainable_variables)\n    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n    return loss\n\nlast_train_loss = np.inf\nlast_test_loss = np.inf\nlast_test_rank = np.inf\n\nwith tqdm(total=128) as pbar:\n    for epoch in range(128):\n        for i, (X, y) in enumerate(sampler.stream(train_user, window=window, batch_size=batch_size)):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, num_split)\n            \n            loss = train_step(X, y, exist_mask, target_mask, window, batch_size, num_split)\n\n            # metric表示部分\n            train_loss(loss)\n            learning_text = \"[{}/{}] \".format(str(i).zfill(3), len(train_user)//batch_size)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} rank{: .5f}\".format(train_loss.result(), last_test_loss, last_test_rank)\n            pbar.set_postfix_str(learning_text + progress_text)\n            \n        last_train_loss = train_loss.result()\n        \n        content_embeddings = model.tokenizer(np.arange(13943, dtype=np.uint16)*4)\n        for X, y in sampler.stream(test_user, window=window, batch_size=batch_size//num_split):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, 1)\n            \n            pred = model(X, y, exist_mask, target_mask, window, batch_size//num_split, 1)\n            loss = loss_object(target, pred)    \n\n            X_emb = tokenizer(X)\n            X_emb = encoder(X_emb, window=window, batch_size=batch_size//num_split)\n            X_emb = top_model(X_emb, exist_mask)\n            logit = tf.matmul(X_emb, content_embeddings, transpose_b=True) \n            arg = tf.argsort(logit, axis=1, direction=\"DESCENDING\")\n            mean_rank = tf.reduce_mean(tf.cast(tf.where(arg == tf.reshape(tf.cast(y, \"int32\")//4, (-1, 1)))[:, 1], \"float32\"))\n            test_rank(mean_rank)\n            \n            # metric表示部分\n            test_loss(loss)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} rank{: .5f}\".format(last_train_loss, test_loss.result(), test_rank.result())\n            pbar.set_postfix_str(progress_text)\n\n        last_test_loss = test_loss.result()\n        last_test_rank = test_rank.result()\n        pbar.update(1)\n\n        train_losses.append(train_loss.result())\n        test_losses.append(test_loss.result())\n        test_ranks.append(test_rank.result())\n\n        train_loss.reset_states()\n        test_loss.reset_states()\n        test_rank.reset_states()\n\nmodel.save_weights(\"./embedding_model.model\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## plot loss"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-04T02:12:30.978493Z","iopub.status.busy":"2021-03-04T02:12:30.977905Z","iopub.status.idle":"2021-03-04T02:12:31.359462Z","shell.execute_reply":"2021-03-04T02:12:31.358703Z"},"papermill":{"duration":0.460773,"end_time":"2021-03-04T02:12:31.359623","exception":false,"start_time":"2021-03-04T02:12:30.89885","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(train_losses)\nplt.plot(test_losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(test_ranks)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-04T02:12:31.445299Z","iopub.status.busy":"2021-03-04T02:12:31.444728Z","iopub.status.idle":"2021-03-04T02:12:31.492828Z","shell.execute_reply":"2021-03-04T02:12:31.492112Z"},"papermill":{"duration":0.093472,"end_time":"2021-03-04T02:12:31.493004","exception":false,"start_time":"2021-03-04T02:12:31.399532","status":"completed"},"tags":[],"trusted":true},"cell_type":"markdown","source":"# Visualization of embedded representation"},{"metadata":{},"cell_type":"markdown","source":"## item embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"import umap\nembs = model.tokenizer(np.arange(13943, dtype=np.uint16)*4).numpy()\n\nixs = np.arange(13943)\nnp.random.shuffle(ixs)\n\nmapper = umap.UMAP(n_neighbors=15, n_components=2, metric=\"cosine\", verbose=True).fit(embs[ixs])\n\nimport matplotlib.pyplot as plt\npart = choice_parts.numpy()[np.arange(13942, dtype=np.uint16)*4]\numap_emb = mapper.transform(embs)\nfor part in range(1, 8):\n    ix = np.where((choice_parts.numpy() == part)[::4])\n    plt.scatter(umap_emb[ix, 0], umap_emb[ix, 1], s=5, label=part)\n\n\nplt.legend()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization of user embedding by problem"},{"metadata":{"trusted":true},"cell_type":"code","source":"def gather_question(question_id, window):\n    X = []\n    y = []\n    for user in test_user:\n        vec = records[records_ixs[user]]\n        ixs = np.where(vec//4 == question_id)[0]\n        if len(ixs):\n            X.append(vec[max(0, ixs[0]-window):ixs[0]])\n            y.append(vec[ixs[0]])\n    return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_id = 3365 # count: 172574 mean_accuracy: 0.547162\nprint(question_id)\nX, y = gather_question(question_id, window)\n\nbatch_size = 1024\nuser_emb = []\nixs = np.arange(len(y))\nfor batch in range(len(y)//batch_size + 1):\n    ix = ixs[batch*batch_size:(batch+1)*batch_size]\n    X, y = padding_sequence(X, y)\n    X_emb, y_emb, target, exist_mask, target_mask = preprocess(np.array(X)[ix], np.array(y)[ix], 1)\n    X_emb = tokenizer(X_emb)\n    X_emb = encoder(X_emb, batch_size=len(y_emb), window=window)\n    X_emb = tf.keras.layers.GlobalAveragePooling1D()(X_emb, exist_mask)\n    user_emb.append(X_emb.numpy())\n\nuser_emb = np.vstack(user_emb)\nis_correct = (np.array(y)%4 == correcr_answers.numpy()[np.array(y)])\n\nixs = np.arange(len(y))\nnp.random.shuffle(ixs)\n\nmapper = umap.UMAP(n_neighbors=30, n_components=2, metric=\"cosine\", verbose=True).fit(user_emb[ixs])\numap_emb = mapper.transform(user_emb)\n\nplt.scatter(umap_emb[:, 0], umap_emb[:, 1], c=np.array(y)%4 == correcr_answers.numpy()[y], s=10, alpha=0.3)\nplt.legend()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_id = 7218 # count: 160300, mean_accuracy: 0.501142\nprint(question_id)\nX, y = gather_question(question_id, window)\n\nbatch_size = 1024\nuser_emb = []\nixs = np.arange(len(y))\nfor batch in range(len(y)//batch_size + 1):\n    ix = ixs[batch*batch_size:(batch+1)*batch_size]\n    X, y = padding_sequence(X, y)\n    X_emb, y_emb, target, exist_mask, target_mask = preprocess(np.array(X)[ix], np.array(y)[ix], 1)\n    X_emb = tokenizer(X_emb)\n    X_emb = encoder(X_emb, batch_size=len(y_emb), window=window)\n    X_emb = tf.keras.layers.GlobalAveragePooling1D()(X_emb, exist_mask)\n    user_emb.append(X_emb.numpy())\n\nuser_emb = np.vstack(user_emb)\nis_correct = (np.array(y)%4 == correcr_answers.numpy()[np.array(y)])\n\nixs = np.arange(len(y))\nnp.random.shuffle(ixs)\n\nmapper = umap.UMAP(n_neighbors=30, n_components=2, metric=\"cosine\", verbose=True).fit(user_emb[ixs])\numap_emb = mapper.transform(user_emb)\n\nplt.scatter(umap_emb[:, 0], umap_emb[:, 1], c=np.array(y)%4 == correcr_answers.numpy()[y], s=10, alpha=0.3)\nplt.legend()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_id = randint(1, questions_df.shape[0]-1)\nprint(question_id)\nX, y = gather_question(question_id, window)\n\nbatch_size = 1024\nuser_emb = []\nixs = np.arange(len(y))\nfor batch in range(len(y)//batch_size + 1):\n    ix = ixs[batch*batch_size:(batch+1)*batch_size]\n    X, y = padding_sequence(X, y)\n    X_emb, y_emb, target, exist_mask, target_mask = preprocess(np.array(X)[ix], np.array(y)[ix], 1)\n    X_emb = tokenizer(X_emb)\n    X_emb = encoder(X_emb, batch_size=len(y_emb), window=window)\n    X_emb = tf.keras.layers.GlobalAveragePooling1D()(X_emb, exist_mask)\n    user_emb.append(X_emb.numpy())\n\nuser_emb = np.vstack(user_emb)\nis_correct = (np.array(y)%4 == correcr_answers.numpy()[np.array(y)])\n\nixs = np.arange(len(y))\nnp.random.shuffle(ixs)\n\nmapper = umap.UMAP(n_neighbors=30, n_components=2, metric=\"cosine\", verbose=True).fit(user_emb[ixs])\numap_emb = mapper.transform(user_emb)\n\nplt.scatter(umap_emb[:, 0], umap_emb[:, 1], c=np.array(y)%4 == correcr_answers.numpy()[y], s=10, alpha=0.3)\nplt.legend()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# fine tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TransformerModel(tf.keras.Model):\n    def __init__(self, tokenizer, encoder, window):\n        super(TransformerModel, self).__init__()\n        self.tokenizer = tokenizer\n        self.encoder = encoder\n        self.gap = tf.keras.layers.GlobalAvgPool1D()\n        self.window = window\n    \n    @tf.function\n    def __call__(self, X, y, exist_mask):\n        X_emb = self.tokenizer(X)\n        X_emb = self.encoder(X_emb, batch_size=len(y), window=self.window)\n        X_emb = self.gap(X_emb, exist_mask)\n        y_emb = self.tokenizer(y)\n        return tf.concat([X_emb, y_emb], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RiiidTop(tf.keras.Model):\n    def __init__(self, layer_dims=[128, 128]):\n        super(RiiidTop, self).__init__()\n        self.denses = [tf.keras.layers.Dense(dim, activation=\"relu\") for dim in layer_dims]\n        self.out = tf.keras.layers.Dense(1, activation=\"sigmoid\")\n    \n    def call(self, emb):\n        for i in range(len(self.denses)):\n            emb = self.denses[i](emb)\n        return self.out(emb)     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RiiidModel(tf.keras.Model):\n    def __init__(self, input_layer, top_model):\n        super(RiiidModel, self).__init__()\n        self.input_layer = input_layer\n        self.top_model = top_model\n    \n    def call(self, X, y, exist_mask):\n        emb = self.input_layer(X, y, exist_mask)\n        return self.top_model(emb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"window = 32\ntransformer_layer = TransformerModel(tokenizer, encoder, window=window)\ntransformer_layer.trainable = False\ntransformer_riiid_model = RiiidModel(transformer_layer, RiiidTop([128, 128]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from adabelief_tf import AdaBeliefOptimizer\nloss_object = tf.keras.losses.BinaryCrossentropy()\noptimizer = AdaBeliefOptimizer(learning_rate=1e-3, weight_decay=1e-4) \n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')\ntest_loss = tf.keras.metrics.Mean(name='test_loss')\ntest_auc = tf.keras.metrics.AUC(name=\"test_auc\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time\n\nwindow = 32\nbatch_size = 2**13\nstart_time = time()\ntrain_losses = []\ntest_losses = []\ntest_aucs = []\n\ndef preprocess(X, y):\n    X = tf.keras.preprocessing.sequence.pad_sequences(X, dtype=\"int32\", value=0, padding=\"pre\")\n    y = tf.constant(y, dtype=\"int32\")    \n    target = tf.eye(batch_size)\n    exist_mask = X > 0\n\n    row = tf.reshape(tf.repeat(tf.cast(y, \"int32\"), len(y)), (len(y), len(y)))\n    col = tf.transpose(row)\n    target_mask = tf.cast(tf.where(tf.logical_and(target != 1., row == col), tf.float32.min / 100, 0), \"float32\")\n    \n    return X, y, target, exist_mask, target_mask\n\ndef padding_sequence(X, y):\n    X = tf.keras.preprocessing.sequence.pad_sequences(X, dtype=\"int32\", value=0, padding=\"pre\")\n    y = tf.constant(y, dtype=\"int32\")\n    return X, y\n\n@tf.function\ndef preprocess(X, y, num_split):   \n    exist_mask = X > 0\n\n    target_masks = []\n    targets = []\n    for y_split in tf.split(y, num_or_size_splits=num_split, axis=0):\n        target = tf.eye(len(y_split))\n        row = tf.reshape(tf.repeat(y_split, len(y_split)), (len(y_split), len(y_split)))\n        col = tf.transpose(row)\n        target_mask = tf.cast(tf.where(tf.logical_and(target != 1., row == col), tf.float32.min / 100, 0), \"float32\")\n        \n        targets.append(target)\n        target_masks.append(target_mask)\n    \n    target = tf.concat(targets, axis=0)\n    target_mask = tf.concat(target_masks, axis=0)\n    \n    return X, y, target, exist_mask, target_mask\n\n@tf.function\ndef train_step(X, y, exist_mask, target_mask, window, batch_size, num_split):\n    label = tf.reshape(tf.cast(y%4 == tf.gather(correcr_answers, y), \"float32\"), (-1, 1))\n    with tf.GradientTape() as tape:\n        pred = transformer_riiid_model(X, y, exist_mask)\n        loss = loss_object(label, pred)\n    grad = tape.gradient(loss, transformer_riiid_model.trainable_variables)\n    optimizer.apply_gradients(zip(grad, transformer_riiid_model.trainable_variables))\n    return loss\n\nlast_train_loss = np.inf\nlast_test_loss = np.inf\nlast_test_auc = np.inf\n\nwith tqdm(total=64) as pbar:\n    for epoch in range(64):\n        for i, (X, y) in enumerate(sampler.stream(train_user, window=window, batch_size=batch_size)):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, num_split)\n            loss = train_step(X, y, exist_mask, target_mask, window, batch_size, num_split)\n            \n            train_loss(loss)\n            learning_text = \"[{}/{}] \".format(str(i).zfill(3), len(train_user)//batch_size)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} auc{: .5f}\".format(train_loss.result(), last_test_loss, last_test_auc)\n            pbar.set_postfix_str(learning_text + progress_text)\n            \n        last_train_loss = train_loss.result()\n        \n        content_embeddings = model.tokenizer(np.arange(13943, dtype=np.uint16)*4)\n        for i, (X, y) in enumerate(sampler.stream(test_user, window=window, batch_size=batch_size)):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, num_split)\n            label = tf.reshape(tf.cast(y%4 == tf.gather(correcr_answers, y), \"float32\"), (-1, 1))\n    \n            pred = transformer_riiid_model(X, y, exist_mask)\n            loss = loss_object(label, pred)\n            \n            # metric表示部分\n            test_loss(loss)\n            test_auc(label, pred)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} auc{: .5f}\".format(last_train_loss, test_loss.result(), test_auc.result())\n            pbar.set_postfix_str(progress_text)\n\n        last_test_loss = test_loss.result()\n        last_test_auc = test_auc.result()\n        pbar.update(1)\n\n        train_losses.append(train_loss.result())\n        test_losses.append(test_loss.result())\n        test_aucs.append(test_auc.result())\n\n        train_loss.reset_states()\n        test_loss.reset_states()\n        test_auc.reset_states()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(train_losses)\nplt.plot(test_losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(test_aucs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# appendix: fine tuning with word2vec"},{"metadata":{"trusted":true},"cell_type":"code","source":"w2v_vec = pd.read_pickle(\"../input/gensim-word2vec/word2vec_weight.npy\")\nw2v_tokenizer = Tokenizer(emb_dim=64)\nw2v_tokenizer.compile()\nw2v_tokenizer(0)\nw2v_tokenizer.content_embedding.set_weights([w2v_vec])\nw2v_tokenizer.trainable = False\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class W2VModel(tf.keras.Model):\n    def __init__(self, w2v_tokenizer):\n        super(W2VModel, self).__init__()\n        self.w2v_tokenizer = w2v_tokenizer\n        self.gap = tf.keras.layers.GlobalAvgPool1D()\n    \n    @tf.function\n    def __call__(self, X, y, exist_mask):\n        X_emb = self.w2v_tokenizer(X)\n        X_emb = self.gap(X_emb, exist_mask)\n        y_emb = self.w2v_tokenizer(y)\n        return tf.concat([X_emb, y_emb], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w2v_layer = W2VModel(w2v_tokenizer)\nw2v_layer.trainable = False\nw2v_riiid_model = RiiidModel(w2v_layer, RiiidTop([128, 128]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from adabelief_tf import AdaBeliefOptimizer\nloss_object = tf.keras.losses.BinaryCrossentropy()\noptimizer = AdaBeliefOptimizer(learning_rate=1e-3, weight_decay=1e-4) \n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')\ntest_loss = tf.keras.metrics.Mean(name='test_loss')\ntest_auc = tf.keras.metrics.AUC(name=\"test_auc\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time\n\nwindow = 32\nbatch_size = 2**13\nstart_time = time()\ntrain_losses = []\ntest_losses = []\ntest_aucs = []\n\ndef preprocess(X, y):\n    X = tf.keras.preprocessing.sequence.pad_sequences(X, dtype=\"int32\", value=0, padding=\"pre\")\n    y = tf.constant(y, dtype=\"int32\")    \n    target = tf.eye(batch_size)\n    exist_mask = X > 0\n\n    row = tf.reshape(tf.repeat(tf.cast(y, \"int32\"), len(y)), (len(y), len(y)))\n    col = tf.transpose(row)\n    target_mask = tf.cast(tf.where(tf.logical_and(target != 1., row == col), tf.float32.min / 100, 0), \"float32\")\n    \n    return X, y, target, exist_mask, target_mask\n\ndef padding_sequence(X, y):\n    X = tf.keras.preprocessing.sequence.pad_sequences(X, dtype=\"int32\", value=0, padding=\"pre\")\n    y = tf.constant(y, dtype=\"int32\")\n    return X, y\n\n@tf.function\ndef preprocess(X, y, num_split):   \n    exist_mask = X > 0\n\n    target_masks = []\n    targets = []\n    for y_split in tf.split(y, num_or_size_splits=num_split, axis=0):\n        target = tf.eye(len(y_split))\n        row = tf.reshape(tf.repeat(y_split, len(y_split)), (len(y_split), len(y_split)))\n        col = tf.transpose(row)\n        target_mask = tf.cast(tf.where(tf.logical_and(target != 1., row == col), tf.float32.min / 100, 0), \"float32\")\n        \n        targets.append(target)\n        target_masks.append(target_mask)\n    \n    target = tf.concat(targets, axis=0)\n    target_mask = tf.concat(target_masks, axis=0)\n    \n    return X, y, target, exist_mask, target_mask\n\n@tf.function\ndef train_step(X, y, exist_mask, target_mask, window, batch_size, num_split):\n    label = tf.reshape(tf.cast(y%4 == tf.gather(correcr_answers, y), \"float32\"), (-1, 1))\n    with tf.GradientTape() as tape:\n        pred = w2v_riiid_model(X, y, exist_mask)\n        loss = loss_object(label, pred)\n    grad = tape.gradient(loss, w2v_riiid_model.trainable_variables)\n    optimizer.apply_gradients(zip(grad, w2v_riiid_model.trainable_variables))\n    return loss\n\nlast_train_loss = np.inf\nlast_test_loss = np.inf\nlast_test_auc = np.inf\n\nwith tqdm(total=64) as pbar:\n    for epoch in range(64):\n        for i, (X, y) in enumerate(sampler.stream(train_user, window=window, batch_size=batch_size)):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, num_split)\n            loss = train_step(X, y, exist_mask, target_mask, window, batch_size, num_split)\n            \n            train_loss(loss)\n            learning_text = \"[{}/{}] \".format(str(i).zfill(3), len(train_user)//batch_size)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} auc{: .5f}\".format(train_loss.result(), last_test_loss, last_test_auc)\n            pbar.set_postfix_str(learning_text + progress_text)\n            \n        last_train_loss = train_loss.result()\n        \n        content_embeddings = model.tokenizer(np.arange(13943, dtype=np.uint16)*4)\n        for i, (X, y) in enumerate(sampler.stream(test_user, window=window, batch_size=batch_size)):\n            X, y = padding_sequence(X, y)\n            X, y, target, exist_mask, target_mask = preprocess(X, y, num_split)\n            label = tf.reshape(tf.cast(y%4 == tf.gather(correcr_answers, y), \"float32\"), (-1, 1))\n    \n            pred = w2v_riiid_model(X, y, exist_mask)\n            loss = loss_object(label, pred)\n            \n            # metric表示部分\n            test_loss(loss)\n            test_auc(label, pred)\n            progress_text = \"train | Loss: {:.5f} test| Loss{: .5f} auc{: .5f}\".format(last_train_loss, test_loss.result(), test_auc.result())\n            pbar.set_postfix_str(progress_text)\n\n        last_test_loss = test_loss.result()\n        last_test_auc = test_auc.result()\n        pbar.update(1)\n\n        train_losses.append(train_loss.result())\n        test_losses.append(test_loss.result())\n        test_aucs.append(test_auc.result())\n\n        train_loss.reset_states()\n        test_loss.reset_states()\n        test_auc.reset_states()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(train_losses)\nplt.plot(test_losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(test_aucs)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}