{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# IndoXTC - Fine-tune Full Toxic [XLM-R] (Simpler)\nExploring Indonesian hate speech/abusive & sentiment text classification using multilingual language model.\n\nThis kernel is a part of my undergraduate final year project.\nCheckout the full github repository:\nhttps://github.com/ilhamfp/indonesian-text-classification-multilingual","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#############################\n# Experiment configurations #\n#############################\n\nINDO_DATA_NAME = 'toxic'\nDATA_PATH_INDO = '../input/simpler-preprocess-indonesian-hate-abusive-text'\n\nFOREIGN_DATA_NAME = 'toxic'\nDATA_PATH_FOREIGN = '../input/jigsaw-multilingual-toxic-comment-classification'\n\nMODEL_NAME = 'XLM_R'\n\nEXPERIMENT_TYPE = 'C' # A / B / C\nTOTAL_DATA = 11852 # 500 / 1000 / 2500 / 5000 / 7500 / 11852\nFOREIGN_LANG_DATA_MULT = 3 # 0.5 / 1 / 1.5 / 2 / 3\nRANDOM_SEED = 1\nVALIDATION_DATA = 0.1\nEPOCHS = 19\nLEARNING_RATE = 5e-6\nUSE_TPU = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from model_full import set_seed, regular_encode, build_model, callback\nfrom load_data import load_experiment_dataset\nimport os\nimport random\nimport pandas as pd\nimport numpy as np\nimport torch\nimport tensorflow as tf\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\n\nset_seed(seed=RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_TPU:\n    # Detect hardware, return appropriate distribution strategy\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    BATCH_SIZE = 8 * strategy.num_replicas_in_sync\n    \nelse:\n    BATCH_SIZE = 8 * 8\n\nAUTO = tf.data.experimental.AUTOTUNE\nMAX_LEN = 512\n\nif MODEL_NAME == 'XLM_R':\n    MODEL = 'jplu/tf-xlm-roberta-large'\nelif MODEL_NAME == 'mBERT':\n    MODEL = 'bert-base-multilingual-cased'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"(x_train, y_train), (x_valid, y_valid), (x_test, y_test) = load_experiment_dataset(INDO_DATA_NAME,\n                                                                                   FOREIGN_DATA_NAME,\n                                                                                   tipe=EXPERIMENT_TYPE, \n                                                                                   total_data=TOTAL_DATA, \n                                                                                   foreign_mult=FOREIGN_LANG_DATA_MULT, \n                                                                                   valid_size=VALIDATION_DATA)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nx_train = regular_encode(x_train, tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(x_valid, tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(x_test, tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Datasets Objects","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(len(x_train),\n             seed=RANDOM_SEED)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif USE_TPU:\n    with strategy.scope():\n        transformer_layer = TFAutoModel.from_pretrained(MODEL)\n        model = build_model(transformer_layer, max_len=MAX_LEN, learning_rate=LEARNING_RATE)\n        \nelse:\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN, learning_rate=LEARNING_RATE)\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\nprint(n_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    callbacks = callback(), \n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights('model.h5') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = pd.DataFrame()\nresult['y_pred'] = model.predict(test_dataset, verbose=1).flatten()\nresult['y_true'] = y_test\nresult.to_csv('result_{}_{}_{}_{}_{}_{}_full.csv'.format(INDO_DATA_NAME,\n                                                    FOREIGN_DATA_NAME,\n                                                    MODEL_NAME,\n                                                    EXPERIMENT_TYPE,\n                                                    TOTAL_DATA,\n                                                    FOREIGN_LANG_DATA_MULT),\n              index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}