{"metadata":{"kernelspec":{"display_name":"gpu","language":"python","name":"gpu"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[An essay for a PSPFGP comepetition.](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/425798)\n\n1. [PSPFGP_NN_dataset](https://www.kaggle.com/code/dongyk/pspfgp-nn-dataset)\n2. [PSPFGP_NN_Pretrain](https://www.kaggle.com/code/dongyk/pspfgp-nn-pretrain)\n3. [PSPFGP_NN_Train](https://www.kaggle.com/code/dongyk/pspfgp-nn-train)\n4. [PSPFGP_NN_Inference](https://www.kaggle.com/code/dongyk/pspfgp-nn-inference)","metadata":{}},{"cell_type":"code","source":"VERSION = 4\nPRETRAIN_VERSION = f'PSPFGP_NN_Pretrain_{VERSION}'\nDATASET_VERSION = f'PSPFGP_NN_dataset_{VERSION}'\nCODE_VERSION = f'PSPFGP_NN_code_{VERSION}'\nprint(PRETRAIN_VERSION)\nprint(DATASET_VERSION)\nprint(CODE_VERSION)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.python.client import device_lib\n\nprint(device_lib.list_local_devices())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nimport gc\nimport os\nimport matplotlib.pyplot as plt\n\n%load_ext memory_profiler","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nSEED = 0\n\nos.environ['PYTHONHASHSEED'] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n# This causes an error when using Adam.\n# \"deterministic gpu implementation of unsorted segment reduction op not available.\"\n#os.environ['TF_DETERMINISTIC_OPS'] = '1'\nos.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'  # TF will not use all memory","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEVEL_GROUPS = ['0-4', '5-12', '13-22']\n\nCAT_FEATURES = ['coded_event_name_name', 'coded_room_fqid', 'coded_text', 'coded_fqid']\nNUM_FEATURES = ['elapsed_time_diff']\nFEATURES = NUM_FEATURES + CAT_FEATURES\n\nLENGTHS = {'0-4':600, '5-12':1400, '13-22':2000}\nQNS = {'0-4':list(range(1, 4)), '5-12':list(range(4, 14)), '13-22':list(range(14, 19))}\n\nN_FOLDS = 5\nBATCH_SIZE = 128\nEPOCHS_PER_STEP = 2\nEPOCHS_PER_CYCLE = 2 * EPOCHS_PER_STEP\nD_MODEL = 24\nN_UNITS = [512, 128, 64]\nMAX_LR = 0.01\nEPOCHS = 25*EPOCHS_PER_CYCLE","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\ndataset = pickle.load(open(f'{DATASET_VERSION}.pkl', 'rb'))\nCODE, CODE_LEN = pickle.load(open(f'{CODE_VERSION}.pkl', 'rb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvBlock(tf.keras.layers.Layer):\n    def __init__(self, d_model, dropout_rate):\n        super().__init__()\n        self.conv1d = tf.keras.layers.Conv1D(filters=d_model, kernel_size=5, padding='same', activation='gelu', kernel_initializer='he_uniform')\n        self.layer_norm = tf.keras.layers.LayerNormalization()\n        self.dropout = tf.keras.layers.Dropout(rate=dropout_rate)\n\n    def call(self, inputs):\n        x = self.conv1d(inputs)\n        x = x + inputs\n        x = self.layer_norm(x)\n        outputs = self.dropout(x)\n        \n        return outputs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeEmbedding(tf.keras.layers.Layer):\n    def __init__(self, n_blocks, d_model, dropout_rate):\n        super().__init__()\n        self.conv_blocks = [ConvBlock(d_model, dropout_rate=dropout_rate) for _ in range(n_blocks)]\n\n    def call(self, inputs):\n        x = tf.expand_dims(inputs, axis=-1)\n        for conv_block in self.conv_blocks:\n            x = conv_block(x)\n        \n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvNet(tf.keras.Model):\n    def __init__(self, input_dims, d_model, n_blocks=4, name='convnet_'):\n        super().__init__(name=name)\n        self.input_dims = input_dims\n        self.d_model = d_model\n        self.n_blocks = n_blocks\n        self.event_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_event_name_name'], output_dim=d_model, mask_zero=True, name='event_embedding')\n        self.room_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_room_fqid'], output_dim=d_model, mask_zero=True, name='room_embedding')\n        self.text_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_text'], output_dim=d_model, mask_zero=True, name='text_embedding')\n        self.fqid_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_fqid'], output_dim=d_model, mask_zero=True, name='fqid_embedding')\n        self.time_embedding = TimeEmbedding(n_blocks, d_model, dropout_rate=0.2)\n        self.pooling = tf.keras.layers.GlobalAveragePooling1D()\n\n    def call(self, inputs):\n        event = self.event_embedding(inputs['coded_event_name_name'])\n        room = self.room_embedding(inputs['coded_room_fqid'])\n        text = self.text_embedding(inputs['coded_text'])\n        fqid = self.fqid_embedding(inputs['coded_fqid'])\n        time = self.time_embedding(inputs['elapsed_time_diff'])\n        x = time * (event + room + text + fqid)\n        outputs = self.pooling(x)\n        \n        return outputs\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'input_dims':self.input_dims,\n            'd_model':self.d_model,\n            'n_blocks':self.n_blocks,\n        })\n\n        return config\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleHead(tf.keras.Model):\n    def __init__(self, n_units, n_outputs, name=None):\n        super().__init__(name=name)\n        self.n_units = n_units\n        self.n_outputs = n_outputs\n        self.ffs = [tf.keras.layers.Dense(units, activation='leaky_relu', kernel_initializer='he_uniform') for units in n_units]\n        self.out = tf.keras.layers.Dense(n_outputs, activation='sigmoid')\n\n    def call(self, inputs):\n        x = inputs\n        for ff in self.ffs:\n            x = ff(x)\n        outputs = self.out(x)\n\n        return outputs\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'n_units':self.n_units,\n            'n_outputs':self.n_outputs,\n        })\n\n        return config\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PretrainingModel(tf.keras.Model):\n    def __init__(self, convnet, head):\n        super().__init__()\n        self.convnet = convnet\n        self.head = head\n\n    def call(self, inputs):\n        x = self.convnet(inputs)\n        outputs = self.head(x)\n\n        return outputs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_addons.optimizers import CyclicalLearningRate\nimport math\n\ndef build_pretraining_model(fold, level_group, input_len, max_lr):\n    \n    suffix = f\"f{fold}_{level_group.replace('-', '_')}\"\n    convnet = ConvNet(input_dims=CODE_LEN[level_group], d_model=D_MODEL, name=f'convnet_{suffix}')\n    head = SimpleHead(n_units=N_UNITS, n_outputs=len(QNS[level_group]), name=f'pretraining_head_{suffix}')\n    pretraining_model = PretrainingModel(convnet, head)\n\n    iterations_per_epoch = math.ceil(input_len / BATCH_SIZE)\n    clr = CyclicalLearningRate(initial_learning_rate=0.01 * max_lr,\n                               maximal_learning_rate=max_lr,\n                               scale_fn=lambda step: 1/(1.2**(step-1)),\n                               scale_mode='cycle',\n                               step_size=EPOCHS_PER_STEP * iterations_per_epoch)\n    pretraining_model.compile(optimizer=tf.keras.optimizers.Adam(clr, clipnorm=1), loss='binary_crossentropy')\n    \n    return pretraining_model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_scores(logs):\n    loss = logs['loss']\n    val_loss = logs['val_loss']\n    xticks = range(1, len(loss) + 1)\n    plt.figure(figsize=(8, 3))\n    plt.plot(xticks, loss)\n    plt.plot(xticks, val_loss)\n    plt.legend(['loss', 'val_loss'])\n    plt.xlabel('epoch')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nclass LogCallback(tf.keras.callbacks.Callback):\n    def __init__(self):\n        self.history = []\n        self.best_score = 100\n        self.t_0 = time.time()\n        \n    def on_epoch_end(self, epoch, logs=None):\n        self.history.append({\n            'time': int(time.time() - self.t_0),\n            'epoch': epoch,\n            'epoch_end_lr': self.model.optimizer._decayed_lr(tf.float32).numpy(),\n            'loss': logs['loss'],\n            'val_loss': logs['val_loss']\n        })\n\n        if epoch == 0:\n            print(f\"{'time' :^11} {'epoch' :^9} {'epoch_end_lr' :^12} {'loss' :^10} {'val_loss' :^10}\")\n\n        info = self.history[-1]\n        hour = f\"{info['time'] // 3600 :02}\"\n        minute = f\"{info['time'] // 60 % 60 :02}\"\n        second = f\"{info['time'] % 60 :02}\"\n\n        if self.best_score > info['val_loss']:\n            self.best_score = info['val_loss']\n\n        print(\n            f\"{f'{hour}:{minute}:{second}' :>11} {epoch + 1 :>9} {info['epoch_end_lr'] :>12.4f} {info['loss'] :>10.5f} {info['val_loss'] :>10.5f} {'-' if info['val_loss'] == self.best_score else ''}\"\n        )\n\n    def on_train_end(self, logs=None):\n        h = pd.DataFrame(self.history)\n        plot_scores(h)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', \n                                                  patience=4*EPOCHS_PER_CYCLE, \n                                                  verbose=0, \n                                                  restore_best_weights=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fast_f1_score(trues, preds):\n    all_positives = (trues + preds == 2).mean()\n    all_negatives = (trues + preds == 0).mean()\n    score = 1 - (1 - all_negatives - all_positives) / (1 - (all_negatives - all_positives) ** 2)\n    \n    return np.round(score, 5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize_threshold(trues, preds, step_size=0.0025):\n    best_score = 0\n    best_threshold = 0\n    for threshold in np.arange(0.55, 0.70, step_size):\n        binarized_preds = (preds > threshold).astype('int')\n        score = fast_f1_score(trues.reshape(-1), binarized_preds.reshape(-1))\n        if score > best_score:\n            best_score = score\n            best_threshold = threshold\n\n    return best_score, best_threshold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nmodels = {}\nscores = []\n\nfor f in range(N_FOLDS):\n    print('-----------------------------')\n    print(f'f{f}')\n    print()\n    models[f'f{f}'] = {}\n    oof_preds = []\n    oof_trues = []\n    \n    (X_train, y_train), (X_valid, y_valid) = dataset[f'f{f}']\n    \n    for lvgp in LEVEL_GROUPS:\n        print(f'Pretrain {lvgp} ------------')\n        print()\n\n        input_len = len(y_train[lvgp])\n        train_dataset = tf.data.Dataset.from_tensor_slices((X_train[lvgp], y_train[lvgp]))\n        train_dataset = train_dataset.batch(BATCH_SIZE).prefetch(buffer_size=tf.data.AUTOTUNE)\n        valid_dataset = tf.data.Dataset.from_tensor_slices((X_valid[lvgp], y_valid[lvgp]))\n        valid_dataset = valid_dataset.batch(BATCH_SIZE).prefetch(buffer_size=tf.data.AUTOTUNE)\n        \n        pretraining_model = build_pretraining_model(fold=f, level_group=lvgp, input_len=input_len, max_lr=MAX_LR)\n        callbacks = [LogCallback(), early_stopping]\n        history = pretraining_model.fit(x=train_dataset, validation_data=valid_dataset, epochs=EPOCHS, verbose=0, callbacks=callbacks)\n\n        models[f'f{f}'][lvgp] = {}\n        models[f'f{f}'][lvgp]['convnet'] = pretraining_model.convnet\n        models[f'f{f}'][lvgp]['head'] = pretraining_model.head\n        \n        oof_preds.append(pretraining_model.predict(valid_dataset, verbose=False))\n        oof_trues.append(y_valid[lvgp])\n\n        if f == 0:\n            display(pretraining_model.summary(expand_nested=True))\n\n    preds = np.concatenate(oof_preds, axis=1)\n    trues = np.concatenate(oof_trues, axis=1)\n    ova_score, threshold = optimize_threshold(trues, preds)\n    ss = []\n    ss.append(threshold)\n    for i in range(18):\n        s = fast_f1_score(trues[:, i], (preds[:, i] > threshold).astype('int'))\n        ss.append(s)\n    ss.append(ova_score)\n    score = pd.DataFrame(ss, columns=[f'f{f}'], index=['threshold'] + [q for q in range(1, 19)] + ['ova_score'])\n    scores.append(score)\n    scores_df = pd.concat(scores, axis=1)\n    scores_df['mean'] = scores_df.mean(axis=1)\n    display(scores_df)\n\n    tf.keras.backend.clear_session()\n    del X_train, y_train, X_valid, y_valid, train_dataset, valid_dataset, pretraining_model, callbacks, history\n    gc.collect()","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = os.path.join(os.getcwd(), 'model', PRETRAIN_VERSION)\nprint(path)\nos.makedirs(path, exist_ok=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in range(N_FOLDS):\n    for lvgp in LEVEL_GROUPS:\n        suffix = f\"f{f}_{lvgp.replace('-', '_')}\"\n        inputs = {feature: tf.keras.Input(shape=(LENGTHS[lvgp], ), name=f'input_{feature}_{suffix}') for feature in FEATURES}\n        outputs = models[f'f{f}'][lvgp]['convnet'](inputs)\n        convnet = tf.keras.Model(inputs=inputs, outputs=outputs, name=f'convnet_{suffix}')\n        f_name = os.path.join(path, f\"convnet_{suffix}.h5\")\n        convnet.save(f_name)","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]}]}