{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feedback Prize - BERT Inference\n\nThis is the **inference** notebook to the training notebook [Feedback Prize - BERT](https://www.kaggle.com/morodertobias/feedback-prize-bert/). \n\nThe current version uses a fine-tuned BERT model on the input ``discourse_type [SEP] discourse_text``.\n\nNote, data preparation and model creation function need to be identical as in the corresponding training notebook. Also be careful that added notebook outputs or dataset might change over time.\n\n- Notebook version ``feedback-prize-bert``: v24","metadata":{}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport transformers","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:01.639547Z","iopub.execute_input":"2022-07-18T23:24:01.640284Z","iopub.status.idle":"2022-07-18T23:24:01.646064Z","shell.execute_reply.started":"2022-07-18T23:24:01.640249Z","shell.execute_reply":"2022-07-18T23:24:01.645181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)\nprint(transformers.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:01.942368Z","iopub.execute_input":"2022-07-18T23:24:01.942776Z","iopub.status.idle":"2022-07-18T23:24:01.947924Z","shell.execute_reply.started":"2022-07-18T23:24:01.942741Z","shell.execute_reply":"2022-07-18T23:24:01.947066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = tf.distribute.get_strategy()\nstrategy","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:02.486203Z","iopub.execute_input":"2022-07-18T23:24:02.487158Z","iopub.status.idle":"2022-07-18T23:24:02.493631Z","shell.execute_reply.started":"2022-07-18T23:24:02.487112Z","shell.execute_reply":"2022-07-18T23:24:02.492578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config\n\nLet us use a config object holding all parameters, settings and configs.","metadata":{}},{"cell_type":"code","source":"class Config:\n    model_name = \"tpu_bert_v15\"\n    n_fold = 5\n    # inputs\n    input_dir = pathlib.Path(\"/kaggle/input/feedback-prize-effectiveness/\")\n    path_train = input_dir / \"train.csv\"\n    train_dir = input_dir / \"train\"\n    path_test = input_dir / \"test.csv\"\n    test_dir = input_dir / \"test\"\n    path_submission = input_dir / \"sample_submission.csv\"\n    labels = [\"Ineffective\", \"Adequate\", \"Effective\"]\n    label_dict = {v: i for i, v in enumerate(labels)}\n    num_classes = len(labels)\n    id_col = \"discourse_id\"\n    # model\n    model_dir = pathlib.Path(\"/kaggle/input/feedback-prize-bert\")\n    pretrained_dir = model_dir / \"pretrained\"\n    path_tokenizer = pretrained_dir\n    fmt_weights_name = f\"weights__{model_name}__fold-\" + \"{fold}.h5\"\n    max_len = 512\n    dropout = 0.4\n    # train\n    batch_size = 32\n    verbose = 2\n    \ncfg = Config()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-18T23:24:03.525113Z","iopub.execute_input":"2022-07-18T23:24:03.525984Z","iopub.status.idle":"2022-07-18T23:24:03.532669Z","shell.execute_reply.started":"2022-07-18T23:24:03.525947Z","shell.execute_reply":"2022-07-18T23:24:03.531909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted(os.listdir(cfg.model_dir))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:04.279421Z","iopub.execute_input":"2022-07-18T23:24:04.279816Z","iopub.status.idle":"2022-07-18T23:24:04.291563Z","shell.execute_reply.started":"2022-07-18T23:24:04.279782Z","shell.execute_reply":"2022-07-18T23:24:04.290826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation\n\nLoad data and tokenizer, prepare the main input ``discourse_type [SEP] discourse_text`` and create dataset.","metadata":{}},{"cell_type":"code","source":"tokenizer = transformers.AutoTokenizer.from_pretrained(cfg.path_tokenizer)\ntokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:05.719774Z","iopub.execute_input":"2022-07-18T23:24:05.720136Z","iopub.status.idle":"2022-07-18T23:24:06.080403Z","shell.execute_reply.started":"2022-07-18T23:24:05.720105Z","shell.execute_reply":"2022-07-18T23:24:06.079523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(cfg.path_test)\ndata[\"text\"] = data[\"discourse_type\"] + tokenizer.sep_token + data[\"discourse_text\"]\ndata","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:06.081919Z","iopub.execute_input":"2022-07-18T23:24:06.082806Z","iopub.status.idle":"2022-07-18T23:24:06.111158Z","shell.execute_reply.started":"2022-07-18T23:24:06.082763Z","shell.execute_reply":"2022-07-18T23:24:06.110273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"options = tf.data.Options()\noptions.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n\n\ndef encode_text(text):\n    \"\"\"Encode text with tokenizer and return dictionary of numpy results.\"\"\"\n    encoded = tokenizer.batch_encode_plus(\n        text,\n        max_length=cfg.max_len,\n        padding='max_length',\n        truncation=True,\n        return_attention_mask=True,\n        return_token_type_ids=True,\n        return_tensors=\"tf\",\n    )\n    return {\n        \"input_ids\": encoded[\"input_ids\"].numpy(),\n        \"attention_masks\": encoded[\"attention_mask\"].numpy(),\n        \"token_type_ids\": encoded[\"token_type_ids\"].numpy(),\n    }\n\n\ndef get_dataset(data, batch_size=cfg.batch_size, shuffle=False, repeat=False, include_label=True):\n    \"\"\"Get dataset\"\"\"\n    encoded_text = encode_text(data['text'].to_list())\n    tensor_slices = encoded_text\n    if include_label:\n        label = tf.one_hot(data[\"label\"].to_list(), cfg.num_classes)\n        tensor_slices = (encoded_text, label)\n    ds = tf.data.Dataset.from_tensor_slices(tensor_slices)\n    ds = ds.with_options(options)\n    if repeat:\n        ds = ds.repeat()\n    if shuffle:\n        ds = ds.shuffle(2048)\n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(tf.data.AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:06.615053Z","iopub.execute_input":"2022-07-18T23:24:06.615413Z","iopub.status.idle":"2022-07-18T23:24:06.625513Z","shell.execute_reply.started":"2022-07-18T23:24:06.615382Z","shell.execute_reply":"2022-07-18T23:24:06.624671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_dataset(data, include_label=False, shuffle=False)\nds","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:07.133424Z","iopub.execute_input":"2022-07-18T23:24:07.133812Z","iopub.status.idle":"2022-07-18T23:24:12.370362Z","shell.execute_reply.started":"2022-07-18T23:24:07.133778Z","shell.execute_reply":"2022-07-18T23:24:12.369550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model\n\nDirectly copy and paste the ``create_model`` function from the training notebook; compilation can be skipped.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import Model, layers, losses, optimizers, metrics, callbacks, backend","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:12.372157Z","iopub.execute_input":"2022-07-18T23:24:12.372787Z","iopub.status.idle":"2022-07-18T23:24:13.289038Z","shell.execute_reply.started":"2022-07-18T23:24:12.372748Z","shell.execute_reply":"2022-07-18T23:24:13.288080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idea from https://www.kaggle.com/code/abhishek/tez-for-feedback-v2-0\n# careful, own implementation following https://www.tensorflow.org/guide/keras/masking_and_padding\n\n\nclass MeanPooler(layers.Layer):\n    def call(self, inputs, mask=None):\n        broadcast_float_mask = tf.expand_dims(tf.cast(mask, \"float32\"), -1)\n        masked_inputs = inputs * broadcast_float_mask\n        inputs_sum = tf.reduce_sum(masked_inputs, axis=1)\n        mask_sum = tf.reduce_sum(broadcast_float_mask, axis=1)\n        mask_sum = tf.math.maximum(mask_sum, 1e-9)\n        return inputs_sum / mask_sum","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:13.290318Z","iopub.execute_input":"2022-07-18T23:24:13.290734Z","iopub.status.idle":"2022-07-18T23:24:13.300720Z","shell.execute_reply.started":"2022-07-18T23:24:13.290685Z","shell.execute_reply":"2022-07-18T23:24:13.299120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    # inputs\n    input_ids = layers.Input(shape=(cfg.max_len,), dtype=\"int32\", name=\"input_ids\")\n    attention_masks = layers.Input(shape=(cfg.max_len,), dtype=\"int32\", name=\"attention_masks\")\n    token_type_ids = layers.Input(shape=(cfg.max_len,), dtype=\"int32\", name=\"token_type_ids\")\n    # base_model\n    base_model_config = transformers.AutoConfig.from_pretrained(\n        cfg.pretrained_dir / \"config.json\"\n    )\n    base_model = transformers.TFAutoModel.from_pretrained(\n        cfg.pretrained_dir / \"tf_model.h5\", config=base_model_config\n    )\n    # base_model.trainable = False\n    base_model_output = base_model(\n        input_ids, attention_mask=attention_masks, token_type_ids=token_type_ids\n    )\n    # x = base_model_output.last_hidden_state[:, 0, :]\n    x = MeanPooler()(base_model_output.last_hidden_state, mask=attention_masks)\n    # head\n    x = layers.Dropout(cfg.dropout)(x)\n    output = layers.Dense(cfg.num_classes, activation='softmax')(x)\n    model = Model(\n        inputs=[input_ids, attention_masks, token_type_ids],\n        outputs=output,\n        name=cfg.model_name,\n    )\n#     # compile\n#     model.compile(\n#         optimizer=optimizers.Adam(cfg.learning_rate),\n#         loss=losses.CategoricalCrossentropy(label_smoothing=cfg.label_smoothing),\n#         metrics=[\"acc\", metrics.CategoricalCrossentropy(name='xentropy')],\n#     )\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:13.302845Z","iopub.execute_input":"2022-07-18T23:24:13.303663Z","iopub.status.idle":"2022-07-18T23:24:13.315338Z","shell.execute_reply.started":"2022-07-18T23:24:13.303607Z","shell.execute_reply":"2022-07-18T23:24:13.314475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backend.clear_session()\nwith strategy.scope():\n    model = create_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:13.318256Z","iopub.execute_input":"2022-07-18T23:24:13.318764Z","iopub.status.idle":"2022-07-18T23:24:27.077382Z","shell.execute_reply.started":"2022-07-18T23:24:13.318739Z","shell.execute_reply":"2022-07-18T23:24:27.075713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and create submission","metadata":{}},{"cell_type":"code","source":"pred_avg = pd.DataFrame(0.0, columns=cfg.labels, index=data[cfg.id_col])\nfor fold in range(cfg.n_fold):\n    print(f\"fold: {fold}\")\n    path_weights = cfg.model_dir / cfg.fmt_weights_name.format(fold=fold)\n    print(f\"path_weights: {path_weights}\")\n    model.load_weights(path_weights)\n    pred = model.predict(ds, verbose=1)\n    pred = pd.DataFrame(pred, columns=cfg.labels, index=data[cfg.id_col])\n    display(pred.head())\n    pred_avg += pred\npred_avg = pred_avg / cfg.n_fold","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:27.078640Z","iopub.execute_input":"2022-07-18T23:24:27.079382Z","iopub.status.idle":"2022-07-18T23:24:49.529125Z","shell.execute_reply.started":"2022-07-18T23:24:27.079344Z","shell.execute_reply":"2022-07-18T23:24:49.528368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_avg = pred_avg.reset_index()\npred_avg","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:49.530471Z","iopub.execute_input":"2022-07-18T23:24:49.530839Z","iopub.status.idle":"2022-07-18T23:24:49.545293Z","shell.execute_reply.started":"2022-07-18T23:24:49.530802Z","shell.execute_reply":"2022-07-18T23:24:49.544569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_avg.to_csv(\"submission.csv\", index=False)\n!head -4 \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-07-18T23:24:49.546387Z","iopub.execute_input":"2022-07-18T23:24:49.546675Z","iopub.status.idle":"2022-07-18T23:24:50.266283Z","shell.execute_reply.started":"2022-07-18T23:24:49.546630Z","shell.execute_reply":"2022-07-18T23:24:50.265175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}