{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch sucks am I right (the sequel)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nimport transformers\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Conv1D, GlobalMaxPooling1D, Dropout, BatchNormalization\nfrom tensorflow.keras.activations import sigmoid\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.regularizers import l1, l2\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.keras.wrappers.scikit_learn import KerasClassifier\nfrom sklearn.ensemble import AdaBoostClassifier","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading training/validation/test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle = '/kaggle/input/undersampling-xlmtokenized/'\nx_train = np.load(f'{kaggle}x_train.npy')\ny_train = np.load(f'{kaggle}y_train.npy')\n\nx_valid = np.load(f'{kaggle}x_valid.npy')\ny_valid = np.load(f'{kaggle}y_valid.npy')\n\nx_test = np.load(f'{kaggle}x_test.npy')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TPU Configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global Variables\nBATCHSIZE = 128\nEPOCHS = 10\nAUTO = tf.data.experimental.AUTOTUNE\nNUMCORES = strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Helper function to build model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build(transformer, maxlen=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_ids = Input(shape=(maxlen,), dtype=tf.int32, name=\"input_word_ids\")\n    cls_token = transformer(input_ids)[0][:,0,:]\n    out = tf.reshape(cls_token, [-1, cls_token.shape[1], 1])\n    out = Dropout(0.25)(out)\n    out = Conv1D(100, 3, padding='valid', activation='relu', strides=1)(out)\n    out = BatchNormalization(axis=2)(out)\n    out = Conv1D(100, 4, padding='valid', activation='relu', strides=1)(out)\n    out = BatchNormalization(axis=2)(out)\n    out = Conv1D(100, 5, padding='valid', activation='relu', strides=1)(out)\n    out = BatchNormalization(axis=2)(out)\n    out = GlobalMaxPooling1D()(out)\n    out = Dropout(0.5)(out)\n    out = Dense(1, activation='sigmoid')(out)\n    model = Model(inputs=input_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), \n                  loss=BinaryCrossentropy(label_smoothing=0.1), metrics=['accuracy', AUC()])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create TF dataset objects.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n        .from_tensor_slices((x_train, y_train))\n        .repeat()\n        .shuffle(x_train.shape[0])\n        .batch(BATCHSIZE)\n        .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n        .from_tensor_slices((x_valid, y_valid))\n        .batch(BATCHSIZE)\n        .cache()\n        .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n        .from_tensor_slices(x_test)\n        .batch(BATCHSIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load model into the TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = transformers.TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-large')\n    model = build(transformer_layer, maxlen=192)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = 680\n\n# Treat every instance of class 1 as 20 instances of class 0.\n# 5% of training data is composed of toxic data.\n# CLASS_WEIGHT = [1., 20.]\ntraining_history = model.fit(train_dataset, \n                             steps_per_epoch=n_steps, \n                             validation_data=valid_dataset, \n                             epochs=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After saturating the learning potential of the model on english data only, we can train it for a few more epochs on the validation set to fine tune the translation process.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_valid.shape[0] // 128\n\n# Treat every instance of class 1 as 7 instances of class 0.\n# 15% of validation data is composed of toxic data\nCLASS_WEIGHT = [1., 7.]\nvalid_history = model.fit(valid_dataset.repeat(), \n                          steps_per_epoch=n_steps, \n                          epochs=4,\n                          class_weight=CLASS_WEIGHT)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting Training History","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nplt.plot(training_history.history['auc'])\nplt.plot(training_history.history['val_auc'])\nplt.title('Model AUC vs. Epoch')\nplt.ylabel('AUC')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Valid'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(training_history.history['accuracy'])\nplt.plot(training_history.history['val_accuracy'])\nplt.title('Model Accuracy vs. Epoch')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Valid'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(training_history.history['loss'])\nplt.plot(training_history.history['val_loss'])\nplt.title('Model Loss vs. Epoch')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Valid'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Model\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsub['toxic'] = model.predict(test_dataset, verbose=1)[0:len(sub['toxic'])]\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}