{"metadata":{"kernelspec":{"display_name":"gpu","language":"python","name":"gpu"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[An essay for a PSPFGP comepetition.](https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/425798)\n\n1. [PSPFGP_NN_dataset](https://www.kaggle.com/code/dongyk/pspfgp-nn-dataset)\n2. [PSPFGP_NN_Pretrain](https://www.kaggle.com/code/dongyk/pspfgp-nn-pretrain)\n3. [PSPFGP_NN_Train](https://www.kaggle.com/code/dongyk/pspfgp-nn-train)\n4. [PSPFGP_NN_Inference](https://www.kaggle.com/code/dongyk/pspfgp-nn-inference)","metadata":{}},{"cell_type":"code","source":"VERSION = 4\nTRAIN_VERSION = f'PSPFGP_NN_Train_{VERSION}'\nPRETRAIN_VERSION = f'PSPFGP_NN_Pretrain_{VERSION}'\nDATASET_VERSION = f'PSPFGP_NN_dataset_{VERSION}'\nCODE_VERSION = f'PSPFGP_NN_code_{VERSION}'\nTHR_VERSION = f'PSPFGP_NN_thr_{VERSION}'\nprint(TRAIN_VERSION)\nprint(PRETRAIN_VERSION)\nprint(DATASET_VERSION)\nprint(CODE_VERSION)\nprint(THR_VERSION)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.python.client import device_lib\n\nprint(device_lib.list_local_devices())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nimport gc\nimport os\nimport matplotlib.pyplot as plt\n\n%load_ext memory_profiler","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nSEED = 0\n\nos.environ['PYTHONHASHSEED'] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n# This causes an error when using Adam.\n# \"deterministic gpu implementation of unsorted segment reduction op not available.\"\n#os.environ['TF_DETERMINISTIC_OPS'] = '1'\nos.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'  # TF will not use all memory","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEVEL_GROUPS = ['0-4', '5-12', '13-22']\n\nCAT_FEATURES = ['coded_event_name_name', 'coded_room_fqid', 'coded_text', 'coded_fqid']\nNUM_FEATURES = ['elapsed_time_diff']\nFEATURES = NUM_FEATURES + CAT_FEATURES\n\nLENGTHS = {'0-4':600, '5-12':1400, '13-22':2000}\nQNS = {'0-4':list(range(1, 4)), '5-12':list(range(4, 14)), '13-22':list(range(14, 19))}\n\nN_FOLDS = 5\nBATCH_SIZE = 128\nEPOCHS_PER_STEP = 2\nEPOCHS_PER_CYCLE = 2 * EPOCHS_PER_STEP\nD_MODEL = 24\nN_UNITS = [512, 128, 64]\nMAX_LR = 0.01\nEPOCHS = 25*EPOCHS_PER_CYCLE","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\ndataset = pickle.load(open(f'{DATASET_VERSION}.pkl', 'rb'))\nCODE, CODE_LEN = pickle.load(open(f'{CODE_VERSION}.pkl', 'rb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvBlock(tf.keras.layers.Layer):\n    def __init__(self, d_model, dropout_rate):\n        super().__init__()\n        self.conv1d = tf.keras.layers.Conv1D(filters=d_model, kernel_size=5, padding='same', activation='gelu', kernel_initializer='he_uniform')\n        self.layer_norm = tf.keras.layers.LayerNormalization()\n        self.dropout = tf.keras.layers.Dropout(rate=dropout_rate)\n\n    def call(self, inputs):\n        x = self.conv1d(inputs)\n        x = x + inputs\n        x = self.layer_norm(x)\n        outputs = self.dropout(x)\n        \n        return outputs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeEmbedding(tf.keras.layers.Layer):\n    def __init__(self, n_blocks, d_model, dropout_rate):\n        super().__init__()\n        self.conv_blocks = [ConvBlock(d_model, dropout_rate=dropout_rate) for _ in range(n_blocks)]\n\n    def call(self, inputs):\n        x = tf.expand_dims(inputs, axis=-1)\n        for conv_block in self.conv_blocks:\n            x = conv_block(x)\n        \n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvNet(tf.keras.Model):\n    def __init__(self, input_dims, d_model, n_blocks=4, name=None):\n        super().__init__(name=name)\n        self.input_dims = input_dims\n        self.d_model = d_model\n        self.n_blocks = n_blocks\n        self.event_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_event_name_name'], output_dim=d_model, mask_zero=True, name='event_embedding')\n        self.room_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_room_fqid'], output_dim=d_model, mask_zero=True, name='room_embedding')\n        self.text_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_text'], output_dim=d_model, mask_zero=True, name='text_embedding')\n        self.fqid_embedding = tf.keras.layers.Embedding(input_dim=input_dims['coded_fqid'], output_dim=d_model, mask_zero=True, name='fqid_embedding')\n        self.time_embedding = TimeEmbedding(n_blocks, d_model, dropout_rate=0.2)\n        self.pooling = tf.keras.layers.GlobalAveragePooling1D()\n\n    def call(self, inputs):\n        event = self.event_embedding(inputs['coded_event_name_name'])\n        room = self.room_embedding(inputs['coded_room_fqid'])\n        text = self.text_embedding(inputs['coded_text'])\n        fqid = self.fqid_embedding(inputs['coded_fqid'])\n        time = self.time_embedding(inputs['elapsed_time_diff'])\n        x = time * (event + room + text + fqid)\n        outputs = self.pooling(x)\n        \n        return outputs\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'input_dims':self.input_dims,\n            'd_model':self.d_model,\n            'n_blocks':self.n_blocks,\n        })\n\n        return config\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleHead(tf.keras.Model):\n    def __init__(self, n_units, n_outputs, name=None):\n        super().__init__(name=name)\n        self.n_units = n_units\n        self.n_outputs = n_outputs\n        self.ffs = [tf.keras.layers.Dense(units, activation='leaky_relu', kernel_initializer='he_uniform') for units in n_units]\n        self.out = tf.keras.layers.Dense(n_outputs, activation='sigmoid')\n\n    def call(self, inputs):\n        x = inputs\n        for ff in self.ffs:\n            x = ff(x)\n        outputs = self.out(x)\n\n        return outputs\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'n_units':self.n_units,\n            'n_outputs':self.n_outputs,\n        })\n\n        return config\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainingModel(tf.keras.Model):\n    def __init__(self, convnets, heads):\n        super().__init__()\n        self.convnets = convnets\n        self.heads = heads\n        \n    def call(self, inputs):\n        x = {lvgp: self.convnets[lvgp](inputs[lvgp]) for lvgp in LEVEL_GROUPS}\n        outputs = {}\n        outputs['0-4'] = self.heads['0-4'](x['0-4'])\n        outputs['5-12'] = self.heads['5-12'](\n            tf.keras.layers.Concatenate(name='concat_5_12')(\n                [x['0-4'], x['5-12']]\n            )\n        )\n        outputs['13-22'] = self.heads['13-22'](\n            tf.keras.layers.Concatenate(name='concat_13_22')(\n                [x['0-4'], x['5-12'], x['13-22']]\n            )\n        )\n        \n        return outputs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = os.path.join(os.getcwd(), 'model', PRETRAIN_VERSION)\nprint(PATH)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_convnets(fold):\n    # compile=True: you will load the model and compile with the same setting as saved.\n    # (e.g. loss function, optimizer/learning rate and metrics)\n    # compile=False: you will load only the model without the settings.\n    convnets = {}\n    with tf.keras.utils.custom_object_scope({'ConvNet': ConvNet}):\n        for lvgp in LEVEL_GROUPS:\n            suffix = f\"f{fold}_{lvgp.replace('-', '_')}\"\n            f_name = os.path.join(PATH, f'convnet_{suffix}.h5')\n            convnets[lvgp] = tf.keras.models.load_model(f_name, compile=False)\n\n    return convnets","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_addons.optimizers import CyclicalLearningRate\nimport math\n\ndef build_training_model(fold, input_len, max_lr, trainable=False):\n\n    convnets = get_convnets(fold)\n    for lvgp in LEVEL_GROUPS:\n        convnets[lvgp].trainable = trainable\n    heads = {lvgp: SimpleHead(n_units=N_UNITS, n_outputs=len(QNS[lvgp]), name=f\"head_f{fold}_{lvgp.replace('-', '_')}\") for lvgp in LEVEL_GROUPS}\n    training_model = TrainingModel(convnets, heads)\n\n    iterations_per_epoch = math.ceil(input_len / BATCH_SIZE)\n    clr = CyclicalLearningRate(initial_learning_rate=0.01 * max_lr,\n                               maximal_learning_rate=max_lr,\n                               scale_fn=lambda step: 1/(1.2**(step-1)),\n                               scale_mode='cycle',\n                               step_size=EPOCHS_PER_STEP * iterations_per_epoch)\n    training_model.compile(optimizer=tf.keras.optimizers.Adam(clr, clipnorm=1), loss='binary_crossentropy')\n    \n    return training_model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_scores(logs):\n    loss = logs['loss']\n    val_loss = logs['val_loss']\n    xticks = range(1, len(loss) + 1)\n    plt.figure(figsize=(8, 3))\n    plt.plot(xticks, loss)\n    plt.plot(xticks, val_loss)\n    plt.legend(['loss', 'val_loss'])\n    plt.xlabel('epoch')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fast_f1_score(trues, preds):\n    all_positives = (trues + preds == 2).mean()\n    all_negatives = (trues + preds == 0).mean()\n    score = 1 - (1 - all_negatives - all_positives) / (1 - (all_negatives - all_positives) ** 2)\n    \n    return np.round(score, 5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize_threshold(trues, preds, step_size=0.0025):\n    best_score = 0\n    best_threshold = 0\n    for threshold in np.arange(0.55, 0.70, step_size):\n        binarized_preds = (preds > threshold).astype('int')\n        score = fast_f1_score(trues.reshape(-1), binarized_preds.reshape(-1))\n        if score > best_score:\n            best_score = score\n            best_threshold = threshold\n\n    return best_score, best_threshold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_valid_score(model, valid_dataset, y_valid):\n    oof_preds = model.predict(valid_dataset, verbose=False)\n    oof_trues = y_valid\n    \n    preds = np.concatenate([oof_pred for lvgp, oof_pred in oof_preds.items()], axis=1)\n    trues = np.concatenate([oof_true for lvgp, oof_true in oof_trues.items()], axis=1)\n    ova_score, threshold = optimize_threshold(trues, preds)\n    return ova_score, threshold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nclass LogCallback(tf.keras.callbacks.Callback):\n    def __init__(self, valid_dataset, y_valid):\n        self.history = []\n        self.best_loss = 100\n        self.best_score = 0\n        self.t_0 = time.time()\n        \n    def on_epoch_end(self, epoch, logs=None):\n        score, threshold = compute_valid_score(self.model, valid_dataset, y_valid)\n        self.history.append({\n            'time': int(time.time() - self.t_0),\n            'epoch': epoch,\n            'epoch_end_lr': self.model.optimizer._decayed_lr(tf.float32).numpy(),\n            'loss': logs['loss'],\n            'val_loss': logs['val_loss'],\n            'val_score': score,\n            'val_score_thr': threshold\n        })\n        logs['val_score'] = score\n\n        if epoch == 0:\n            print(f\"{'time' :^11} {'epoch' :^9} {'epoch_end_lr' :^12} {'loss' :^10} {'val_loss' :^10} {'val_score' :^10}  {'val_score_thr' :^10}\")\n\n        info = self.history[-1]\n        hour = f\"{info['time'] // 3600 :02}\"\n        minute = f\"{info['time'] // 60 % 60 :02}\"\n        second = f\"{info['time'] % 60 :02}\"\n\n        if self.best_loss > info['val_loss']:\n            self.best_loss = info['val_loss']\n        if self.best_score < info['val_score']:\n            self.best_score = info['val_score']\n            self.best_thr = info['val_score_thr']\n\n        print(\n            f\"{f'{hour}:{minute}:{second}' :>11} {epoch + 1 :>9} {info['epoch_end_lr'] :>12.4f} {info['loss'] :>10.5f} {info['val_loss'] :>10.5f} {info['val_score'] :>10.5f} {'-' if info['val_score'] == self.best_score else ' '} {info['val_score_thr'] :>10.5f}\"\n        )\n\n    def on_train_end(self, logs=None):\n        h = pd.DataFrame(self.history)\n        plot_scores(h)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_score', \n                                                  patience=4*EPOCHS_PER_CYCLE, \n                                                  mode='max',\n                                                  verbose=0, \n                                                  restore_best_weights=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nmodels = {}\nthr = {}\n\nfor f in range(N_FOLDS):\n    print('-----------------------------')\n    print(f'f{f}')\n    print()\n    models[f'f{f}'] = {}\n    \n    (X_train, y_train), (X_valid, y_valid) = dataset[f'f{f}']\n\n    input_len = len(y_train['0-4'])\n    train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n    train_dataset = train_dataset.batch(BATCH_SIZE).prefetch(buffer_size=tf.data.AUTOTUNE)\n    valid_dataset = tf.data.Dataset.from_tensor_slices((X_valid, y_valid))\n    valid_dataset = valid_dataset.batch(BATCH_SIZE).prefetch(buffer_size=tf.data.AUTOTUNE)\n\n    training_model = build_training_model(fold=f, input_len=input_len, max_lr=MAX_LR)\n    callbacks = [LogCallback(valid_dataset, y_valid), early_stopping]\n    history = training_model.fit(x=train_dataset, validation_data=valid_dataset, epochs=EPOCHS, verbose=0, callbacks=callbacks)\n    thr[f'f{f}'] = callbacks[0].best_thr\n    \n    for lvgp in LEVEL_GROUPS:\n        models[f'f{f}'][lvgp] = {}\n        models[f'f{f}'][lvgp]['convnet'] = training_model.convnets[lvgp]\n        models[f'f{f}'][lvgp]['head'] = training_model.heads[lvgp]\n\n    if f == 0:\n        display(training_model.summary(expand_nested=True))\n\n    tf.keras.backend.clear_session()\n    del X_train, y_train, X_valid, y_valid, train_dataset, valid_dataset, training_model, callbacks, history\n    gc.collect()","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = os.path.join(os.getcwd(), 'model', TRAIN_VERSION)\nprint(path)\nos.makedirs(path, exist_ok=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_name = os.path.join(path, f'{THR_VERSION}.pkl')\nwith open(f_name, \"wb\") as f:\n    pickle.dump(thr, f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\n# include_optimizer=True: this allows you to restart training in exactly the same state as you saved the checkpoint.\n# include_optimizer=False: it's not required for inference.\n\nfor f in range(N_FOLDS):\n    for i, lvgp in enumerate(LEVEL_GROUPS):\n        suffix = f\"f{f}_{lvgp.replace('-', '_')}\"\n        inputs = {feature: tf.keras.Input(shape=(LENGTHS[lvgp], ), name=f'input_{feature}_{suffix}') for feature in FEATURES}\n        outputs = models[f'f{f}'][lvgp]['convnet'](inputs)\n        convnet = tf.keras.Model(inputs=inputs, outputs=outputs, name=f'convnet_{suffix}')\n        f_name = os.path.join(path, f'convnet_{suffix}.h5')\n        convnet.save(f_name, include_optimizer=False)\n    \n        inputs = tf.keras.Input(shape=(24 * (i + 1)), name=f'input_{suffix}')\n        outputs = models[f'f{f}'][lvgp]['head'](inputs)\n        head = tf.keras.Model(inputs=inputs, outputs=outputs, name=f'head_{suffix}')\n        f_name = os.path.join(path, f'head_{suffix}.h5')\n        head.save(f_name, include_optimizer=False)","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nfor fold in range(N_FOLDS):\n    for lvgp in LEVEL_GROUPS:\n        suffix = f\"f{fold}_{lvgp.replace('-', '_')}\"\n        f_name = os.path.join(path, f'convnet_{suffix}.h5')\n        with tf.keras.utils.custom_object_scope({'ConvNet': ConvNet}):\n            convnet = tf.keras.models.load_model(f_name, compile=False)\n        converter = tf.lite.TFLiteConverter.from_keras_model(convnet)\n        tflite_model = converter.convert()\n        f_name = os.path.join(path, f'convnet_{suffix}.tflite')\n        with open(f_name, 'wb') as f:\n            f.write(tflite_model)\n\n        f_name = os.path.join(path, f'head_{suffix}.h5')\n        with tf.keras.utils.custom_object_scope({'SimpleHead': SimpleHead}):\n            head = tf.keras.models.load_model(f_name, compile=False)\n        converter = tf.lite.TFLiteConverter.from_keras_model(head)\n        tflite_model = converter.convert()\n        f_name = os.path.join(path, f'head_{suffix}.tflite')\n        with open(f_name, 'wb') as f:\n            f.write(tflite_model)","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]}]}