{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model\n\nfrom transformers import AutoTokenizer, TFAutoModel\n\ntqdm.pandas()\nnp.random.seed(2022)\ntf.random.set_seed(2022)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:29.838379Z","iopub.execute_input":"2022-08-03T17:16:29.838771Z","iopub.status.idle":"2022-08-03T17:16:40.210261Z","shell.execute_reply.started":"2022-08-03T17:16:29.838668Z","shell.execute_reply":"2022-08-03T17:16:40.209088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hardware Config","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    BATCH_SIZE = strategy.num_replicas_in_sync * 8\n    print(\"Running on TPU:\", tpu.master())\n    print(f\"Batch Size: {BATCH_SIZE}\")\n    \nexcept ValueError:\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE = 64\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    print(f\"Batch Size: {BATCH_SIZE}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:40.211968Z","iopub.execute_input":"2022-08-03T17:16:40.212269Z","iopub.status.idle":"2022-08-03T17:16:46.623173Z","shell.execute_reply.started":"2022-08-03T17:16:40.212224Z","shell.execute_reply":"2022-08-03T17:16:46.622217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{}},{"cell_type":"code","source":"class Config:\n    \n    FOLDS = 5\n    VERBOSE = 0\n    NUM_EPOCH = 7\n    MAX_LEN = 512\n    LR_START = 8e-6\n    LR_END = 1e-7\n    BATCH_SIZE = BATCH_SIZE\n    MODEL_NAME = 'roberta-large'\n\nconfig = Config()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:46.624548Z","iopub.execute_input":"2022-08-03T17:16:46.624811Z","iopub.status.idle":"2022-08-03T17:16:46.630779Z","shell.execute_reply.started":"2022-08-03T17:16:46.624774Z","shell.execute_reply":"2022-08-03T17:16:46.629884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load training dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/feedback-train-dataset-with-folds-v2/train.csv')\nprint(f\"train: {train.shape}\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:46.632456Z","iopub.execute_input":"2022-08-03T17:16:46.632765Z","iopub.status.idle":"2022-08-03T17:16:53.527657Z","shell.execute_reply.started":"2022-08-03T17:16:46.632735Z","shell.execute_reply":"2022-08-03T17:16:53.526721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['discourse_type'] = train['discourse_type'].map({\n    'Claim': 0,\n    'Evidence': 1,\n    'Position': 2,\n    'Concluding Statement': 3,\n    'Lead': 4,\n    'Counterclaim': 5,\n    'Rebuttal': 6\n})","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:53.528962Z","iopub.execute_input":"2022-08-03T17:16:53.529223Z","iopub.status.idle":"2022-08-03T17:16:53.572381Z","shell.execute_reply.started":"2022-08-03T17:16:53.529195Z","shell.execute_reply":"2022-08-03T17:16:53.571252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['final_text'] = train.progress_apply(lambda x: f\"{x['discourse_text']} </s> {x['essay_text']}\", axis=1)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:16:53.574066Z","iopub.execute_input":"2022-08-03T17:16:53.574336Z","iopub.status.idle":"2022-08-03T17:16:56.524500Z","shell.execute_reply.started":"2022-08-03T17:16:53.574309Z","shell.execute_reply":"2022-08-03T17:16:56.523560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=5, test_size=0.6, random_state=42)\nfor train_index, test_index in sss.split(train, train['kfold']):\n    _, X_test = train.iloc[train_index], train.iloc[test_index]\n\nprint(X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:17:27.473384Z","iopub.execute_input":"2022-08-03T17:17:27.474662Z","iopub.status.idle":"2022-08-03T17:17:28.215043Z","shell.execute_reply.started":"2022-08-03T17:17:27.474611Z","shell.execute_reply":"2022-08-03T17:17:28.214037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=5, test_size=0.35, random_state=42)\nfor train_index, test_index in sss.split(X_test, X_test['kfold']):\n    X_train, _ = train.iloc[train_index], train.iloc[test_index]\n\nprint(X_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:10.693297Z","iopub.execute_input":"2022-08-03T17:18:10.693866Z","iopub.status.idle":"2022-08-03T17:18:11.081405Z","shell.execute_reply.started":"2022-08-03T17:18:10.693833Z","shell.execute_reply":"2022-08-03T17:18:11.080328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = X_train.copy()\nprint(f\"train: {train.shape}\")\n\ndel X_train, X_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:21.463040Z","iopub.execute_input":"2022-08-03T17:18:21.463335Z","iopub.status.idle":"2022-08-03T17:18:21.690489Z","shell.execute_reply.started":"2022-08-03T17:18:21.463307Z","shell.execute_reply":"2022-08-03T17:18:21.689580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build the model","metadata":{}},{"cell_type":"code","source":"def encode_text(text, tokenizer):\n    \n    encoded = tokenizer.batch_encode_plus(\n        text,\n        add_special_tokens=True,\n        max_length=config.MAX_LEN,\n        padding='max_length',\n        truncation=True,\n        return_attention_mask=True,\n        return_tensors=\"tf\",\n    )\n\n    input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n    attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n\n    return {\n        \"input_ids\": input_ids,\n        \"attention_masks\": attention_masks\n    }","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:32.617985Z","iopub.execute_input":"2022-08-03T17:18:32.618274Z","iopub.status.idle":"2022-08-03T17:18:32.624681Z","shell.execute_reply.started":"2022-08-03T17:18:32.618247Z","shell.execute_reply":"2022-08-03T17:18:32.623630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feedback_model(transformer_model):\n    \n    input_ids = layers.Input(shape=(config.MAX_LEN,), dtype=tf.int32, name=\"input_ids\")\n    attention_mask = layers.Input(shape=(config.MAX_LEN,), dtype=tf.int32, name=\"attention_mask\")\n\n    bert_model = transformer_model(input_ids, attention_mask=attention_mask)\n    \n    last_hidden_state, pooler_output = bert_model[0], bert_model[1]\n    \n    x = layers.Concatenate()([\n        pooler_output,\n        layers.GlobalAveragePooling1D()(last_hidden_state),\n        layers.GlobalMaxPooling1D()(last_hidden_state)\n    ])\n    x = layers.Dropout(rate=0.35)(x)\n    \n    x = layers.Dense(units=1024, activation='gelu')(x)\n    x = layers.Dropout(rate=0.25)(x)\n    \n    x_output = layers.Dense(units=7, activation='softmax')(x)\n\n    model = Model(inputs=[input_ids, attention_mask], \n                  outputs=x_output, \n                  name='Feedback_TFRoberta_Large_Model')\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:33.317002Z","iopub.execute_input":"2022-08-03T17:18:33.317601Z","iopub.status.idle":"2022-08-03T17:18:33.326803Z","shell.execute_reply.started":"2022-08-03T17:18:33.317539Z","shell.execute_reply":"2022-08-03T17:18:33.325618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(config.MODEL_NAME)\ntokenizer.save_pretrained(f'./{config.MODEL_NAME}-tokenizer')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:34.032093Z","iopub.execute_input":"2022-08-03T17:18:34.032414Z","iopub.status.idle":"2022-08-03T17:18:36.848476Z","shell.execute_reply.started":"2022-08-03T17:18:34.032382Z","shell.execute_reply":"2022-08-03T17:18:36.847349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformer_model = TFAutoModel.from_pretrained(config.MODEL_NAME)\ntransformer_model.save_pretrained(f'./{config.MODEL_NAME}-model')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:18:36.850002Z","iopub.execute_input":"2022-08-03T17:18:36.850253Z","iopub.status.idle":"2022-08-03T17:19:21.337803Z","shell.execute_reply.started":"2022-08-03T17:18:36.850224Z","shell.execute_reply":"2022-08-03T17:19:21.336710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = feedback_model(transformer_model)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:19:21.339867Z","iopub.execute_input":"2022-08-03T17:19:21.340185Z","iopub.status.idle":"2022-08-03T17:19:30.143815Z","shell.execute_reply.started":"2022-08-03T17:19:21.340154Z","shell.execute_reply":"2022-08-03T17:19:30.142854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(\n    model, to_file='./Feedback_TFRoberta_Large_Model.png', \n    show_shapes=True, show_layer_names=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T04:59:57.195586Z","iopub.execute_input":"2022-07-19T04:59:57.195909Z","iopub.status.idle":"2022-07-19T04:59:59.010299Z","shell.execute_reply.started":"2022-07-19T04:59:57.195878Z","shell.execute_reply":"2022-07-19T04:59:59.00914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    \n    tokenizer = AutoTokenizer.from_pretrained(config.MODEL_NAME)\n    transformer_model = TFAutoModel.from_pretrained(config.MODEL_NAME)\n    \n    counter = 0\n    oof_score = 0\n    loss_values = {}\n    \n\n    for fold in range(config.FOLDS):\n        counter += 1\n\n        train_data = encode_text(train[train['kfold']!=fold]['final_text'].tolist(), tokenizer)\n        val_data = encode_text(train[train['kfold']==fold]['final_text'].tolist(), tokenizer)\n        \n        train_labels, val_labels = pd.get_dummies(train[train['kfold']!=fold]['discourse_type']), \\\n                                   pd.get_dummies(train[train['kfold']==fold]['discourse_type'])\n        \n        model = feedback_model(transformer_model)\n        model.load_weights(f'../input/feedback-roberta-large-pretraining-v2/Feedback_TFRoberta_Large_Model_{counter}C.h5')\n        \n        model.compile(loss='categorical_crossentropy', \n                      optimizer=optimizers.Adam(learning_rate=config.LR_START))\n\n        early = callbacks.EarlyStopping(monitor=\"val_loss\", mode=\"min\", \n                                        restore_best_weights=True, \n                                        patience=5, verbose=config.VERBOSE)\n\n        reduce_lr = callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, \n                                                min_lr=config.LR_END, patience=1, \n                                                verbose=config.VERBOSE, mode='min')\n\n        chk_point = callbacks.ModelCheckpoint(f'./Feedback_TFRoberta_Large_Model_{counter}C.h5', \n                                              monitor='val_loss', verbose=config.VERBOSE, \n                                              save_best_only=True, mode='min',\n                                              save_weights_only=True)\n\n        history = model.fit(\n            (np.asarray(train_data['input_ids']),\n             np.asarray(train_data['attention_masks'])), \n            train_labels, \n            batch_size=config.BATCH_SIZE,\n            epochs=config.NUM_EPOCH, \n            verbose=config.VERBOSE, \n            callbacks=[reduce_lr, early, chk_point], \n            validation_data=(\n                (np.asarray(val_data['input_ids']),\n                 np.asarray(val_data['attention_masks'])), \n                val_labels\n            )\n        )\n\n        loss_values[\"train_loss_\"+str(counter)] = history.history['loss']\n        loss_values[\"valid_loss_\"+str(counter)] = history.history['val_loss']\n        \n        model.load_weights(f'./Feedback_TFRoberta_Large_Model_{counter}C.h5')\n\n        y_pred = model.predict(\n            (np.asarray(val_data['input_ids']),\n             np.asarray(val_data['attention_masks'])), \n            batch_size=config.BATCH_SIZE, \n            verbose=config.VERBOSE\n        )\n        \n        score = log_loss(val_labels, y_pred)\n        oof_score += score\n        print(f\"Fold-{counter} | OOF Score: {score}\")\n        \n        del model, y_pred\n        del val_data, val_labels\n        del train_data, train_labels\n        gc.collect()\n\n\noof_score /= float(counter)\nprint(f\"Aggregate OOF Score: {oof_score}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T17:19:33.654563Z","iopub.execute_input":"2022-08-03T17:19:33.655020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 7))\nplt.title(\"Model Loss Curve\", fontweight='bold', pad=15)\n\nfor i in range(config.FOLDS):\n    plt.plot(loss_values[\"train_loss_\"+str(i+1)], label='train_loss_'+str(i+1))\n    plt.plot(loss_values[\"valid_loss_\"+str(i+1)], label='valid_loss_'+str(i+1))\n\nplt.ylabel('Model Loss')\nplt.xlabel('Epochs')\nplt.legend()\nplt.grid();","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Good Day!!","metadata":{},"execution_count":null,"outputs":[]}]}