{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# IndoXTC - Fine-tune Full Toxic [XLM-R] Comparable\nExploring Indonesian hate speech/abusive & sentiment text classification using multilingual language model.\n\nThis kernel is a part of my undergraduate final year project.\nCheckout the full github repository:\nhttps://github.com/ilhamfp/indonesian-text-classification-multilingual","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#############################\n# Experiment configurations #\n#############################\n\nINDO_DATA_NAME = 'toxic'\nDATA_PATH_INDO = '../input/simpler-preprocess-indonesian-hate-abusive-text'\n\nFOREIGN_DATA_NAME = 'toxic'\nDATA_PATH_FOREIGN = '../input/jigsaw-multilingual-toxic-comment-classification'\n\nMODEL_NAME = 'XLM_R'\n\nEXPERIMENT_TYPE = 'A' # A / B / C\nTOTAL_DATA = 11852 # 500 / 1000 / 2500 / 5000 / 7500 / 11852\nFOREIGN_LANG_DATA_MULT = 0.5 # 0.5 / 1 / 1.5 / 2 / 3\nRANDOM_SEED = 1\nVALIDATION_DATA = 0.1\nEPOCHS = 25\nLEARNING_RATE = 5e-6\nUSE_TPU = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"######\n## model_full\n######\n\nimport pandas as pd\nimport os\nimport random\nimport numpy as np\nimport torch\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger, EarlyStopping\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors, SentencePieceBPETokenizer\nfrom tensorflow.keras import backend as K\n    \ndef set_seed(seed=1):\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.backends.cudnn.deterministic = True\n    \ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\ndef build_model(transformer, learning_rate=1e-5, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    cls_token = Dropout(0.2)(cls_token)\n    out = Dense(2, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model\n\ndef callback():\n    cb = []\n\n    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',  \n                                       factor=0.5, patience=2, \n                                       verbose=1, mode='min', \n                                       epsilon=0.0001, min_lr=0,\n                                       restore_best_weights=True)\n    cb.append(reduceLROnPlat)\n    \n    log = CSVLogger('log.csv')\n    cb.append(log)\n    \n    es = EarlyStopping(monitor='val_loss', patience=4, verbose=0,\n                       mode='min', restore_best_weights=True)\n    \n    cb.append(es)\n    \n    return cb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"######\n## load_data\n######\n\n# This source code is part of a final year undergraduate project\n# on exploring Indonesian hate speech/abusive & sentiment text \n# classification using a multilingual language model\n# \n# Checkout the full github repository: \n# https://github.com/ilhamfp/indonesian-text-classification-multilingual\n\nimport pandas as pd\nimport numpy as np\nimport re\nfrom sklearn.model_selection import train_test_split\nimport torch\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.utils.data import TensorDataset, DataLoader\n\nRANDOM_SEED=1\n\ndef lowercase(text):\n    return text.lower()\n\ndef remove_nonaplhanumeric(text):\n    text = re.sub('[^0-9a-zA-Z]+', ' ', text) \n    return text\n\ndef remove_unnecessary_char(text):\n    text = re.sub('\\n',' ',text) # Remove every '\\n'\n    text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+)|(http?://[^\\s]+))',' ',text) # Remove every URL\n    text = re.sub('  +', ' ', text) # Remove extra spaces\n    text = text.strip()\n    return text\n\ndef preprocess_text(text):\n    text = lowercase(text)\n    text = remove_nonaplhanumeric(text)\n    text = remove_unnecessary_char(text)\n    return text\n\ndef load_dataset_indonesian(data_name='prosa', data_path=None, data_path_test=None):\n    if data_name == 'prosa':\n        train = pd.read_csv('../input/dataset-prosa/data_train_full.tsv', sep='\\t', header=None)\n        train = train.rename(columns={0: \"text\", 1: \"label\"})\n        train = train[train['label'] != 'neutral']\n        train['label'] = train['label'].apply(lambda x: 1 if x=='positive' else 0)\n        train['text'] = train['text'].apply(lambda x: preprocess_text(x))\n\n        test = pd.read_csv('../input/dataset-prosa/data_testing_full.tsv', sep='\\t', header=None)\n        test = test.rename(columns={0: \"text\", 1: \"label\"})\n        test = test[test['label'] != 'neutral']\n        test['label'] = test['label'].apply(lambda x: 1 if x=='positive' else 0)\n        test['text'] = test['text'].apply(lambda x: preprocess_text(x))\n            \n    elif data_name == 'trip_advisor':\n        if data_path == None:\n            train = pd.read_csv('../input/dataset-tripadvisor/train_set.csv')\n#             train = pd.read_csv('../input/remove-duplicate-tripadvisor/train_set.csv')\n        else:\n            train = pd.read_csv(data_path)\n            \n        train = train.rename(columns={\"content\": \"text\", \"polarity\": \"label\"})\n        train['label'] = train['label'].apply(lambda x: 1 if x==\"positive\" else 0)\n        train['text'] = train['text'].apply(lambda x: preprocess_text(x))\n        \n        if data_path_test == None:\n            test = pd.read_csv('../input/dataset-tripadvisor/test_set.csv')\n#             test = pd.read_csv('../input/remove-duplicate-tripadvisor/test_set.csv')\n        else:\n            test = pd.read_csv(data_path_test)\n            \n        test = test.rename(columns={\"content\": \"text\", \"polarity\": \"label\"})\n        test['label'] = test['label'].apply(lambda x: 1 if x==\"positive\" else 0)\n        test['text'] = test['text'].apply(lambda x: preprocess_text(x))\n\n    elif data_name == 'toxic':\n        if data_path == None:\n            data = pd.read_csv('../input/simpler-preprocess-indonesian-hate-abusive-text/preprocessed_indonesian_toxic_tweet.csv')\n        else:\n            data = pd.read_csv(data_path)\n            \n        temp = pd.DataFrame({\n                   'HS': data['HS'].values,\n                   'Abusive': data['Abusive'].values})\n\n        data['label'] = temp.apply(lambda r: tuple(r), axis=1).apply(np.array)\n            \n        data = data[['Tweet', 'label']]\n        data = data.rename(columns={'Tweet': 'text'})\n\n        X_train, X_test, y_train, y_test = train_test_split(data.text.values, \n                                                            data.label.values, \n                                                            test_size=0.1,\n                                                            random_state=RANDOM_SEED)\n        train = pd.DataFrame({'text': X_train,\n                              'label': y_train})\n\n        test = pd.DataFrame({'text': X_test,\n                             'label': y_test})\n        \n    print(\"~~~Train Data~~~\")\n    print('Shape: ', train.shape)\n    print(train[0:2])\n    print(\"\\nLabel:\")\n#     print(train.label.value_counts())\n    \n    print(\"\\n~~~Test Data~~~\")\n    print('Shape: ', test.shape)\n    print(test[0:4])\n    print(\"\\nLabel:\")\n#     print(test.label.value_counts())\n    return train, test\n    \ndef load_dataset_foreign(data_name='yelp'):\n    train = None\n    if data_name == 'yelp':\n        train = pd.read_csv('../input/yelp-review-dataset/yelp_review_polarity_csv/train.csv', header=None)\n        train = train.rename(columns={0: \"label\", 1: \"text\"})\n        train['label'] = train['label'].apply(lambda x: 1 if x==2 else 0)\n        train['text'] = train['text'].apply(lambda x: preprocess_text(x))\n    \n    elif data_name == 'toxic':\n        data = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\n        data['toxic'] = data['toxic'].apply(lambda x: 1 if x>=0.5 else 0)\n\n        data = data[['comment_text', 'toxic']]\n        data = data.rename(columns={'comment_text': 'text',\n                                    'toxic': 'label'})\n\n        data_pos = data[data['label'] == 1]\n        data_neg = data[data['label'] == 0]\n        train = pd.concat([data_pos[0:152111], \n                           data_neg[0:152111]]).reset_index(drop=True)\n        \n        train['text'] = train['text'].apply(lambda x: preprocess_text(x))\n\n     \n    print(\"~~~Data~~~\")\n    print('Shape: ', train.shape)\n    print(train[0:2])\n    print(\"\\nLabel:\")\n    print(train.label.value_counts())\n    return train\n\ndef split_train_test(train_x, train_y, total_data=50, valid_size=0.2):\n    train_x_split, valid_x_split, train_y_split, valid_y_split = train_test_split(train_x, \n                                                                                  train_y, \n                                                                                  test_size=valid_size,\n                                                                                  random_state=RANDOM_SEED)\n    \n        \n    total_data_valid = int(np.floor(valid_size * total_data))\n    total_data_train = total_data-total_data_valid\n\n    train_x_split = train_x_split[:total_data_train]\n    train_y_split = train_y_split[:total_data_train]\n    valid_x_split = valid_x_split[:total_data_valid]\n    valid_y_split = valid_y_split[:total_data_valid]\n    \n    return train_x_split, train_y_split, valid_x_split, valid_y_split\n    \ndef load_features(data_path, total_data=50, valid_size=0.2):\n    train_x = np.array([x for x in np.load('{}/train_text.npy'.format(data_path), allow_pickle=True)])\n    train_y = pd.read_csv('{}/train_label.csv'.format(data_path)).label.values\n    \n    train_x_split, train_y_split, valid_x_split, valid_y_split = split_train_test(train_x,\n                                                                                  train_y,\n                                                                                  total_data=total_data,\n                                                                                  valid_size=valid_size)\n    return train_x_split, train_y_split, valid_x_split, valid_y_split\n    \n\ndef load_experiment_features(data_path_indo,\n                             data_path_foreign,\n                             tipe='A', \n                             total_data=50, \n                             foreign_mult=1, \n                             valid_size=0.2,\n                             ):\n    ##########################\n    # Load Preprocessed Data #\n    ##########################\n    if tipe == 'A':\n        train_x, train_y, valid_x, valid_y = load_features(data_path_indo,\n                                                           total_data=total_data, \n                                                           valid_size=valid_size)\n        \n    elif tipe == 'B':\n        train_x, train_y, _, _ = load_features(data_path_foreign,\n                                               total_data=total_data, \n                                               valid_size=valid_size)\n        \n        _, _, valid_x, valid_y = load_features(data_path_indo,\n                                               total_data=total_data, \n                                               valid_size=valid_size)\n        \n    elif tipe == 'C':\n        train_x_indo, train_y_indo, valid_x_indo, valid_y_indo = load_features(data_path_indo,\n                                                                                total_data=total_data, \n                                                                                valid_size=valid_size)\n\n        train_x_foreign, train_y_foreign, valid_x_foreign, valid_y_foreign = load_features(data_path_foreign,\n                                                                                           total_data=int(total_data*foreign_mult), \n                                                                                           valid_size=valid_size)\n\n        train_x = np.concatenate([\n                    train_x_indo,\n                    train_x_foreign,\n                    ])\n\n        train_y = np.concatenate([\n                    train_y_indo,\n                    train_y_foreign,\n                ])\n\n        valid_x = valid_x_indo\n\n        valid_y = valid_y_indo\n        \n\n    test_x = np.array([x for x in np.load('{}/test_text.npy'.format(data_path_indo), allow_pickle=True)])\n    test_y = pd.read_csv('{}/test_label.csv'.format(data_path_indo)).label.values\n\n    #########################\n    # Convert to dataloader #\n    #########################\n    batch_size = 32\n\n    train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n    valid_data = TensorDataset(torch.from_numpy(valid_x), torch.from_numpy(valid_y))\n    test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))\n    \n    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size)\n    test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)\n\n    return train_loader, valid_loader, test_loader\n\ndef load_train_dataset(data_name, total_data=50, valid_size=0.2, is_foreign=False):\n    print('sampai')\n\n    train = None\n    if is_foreign:\n        train = load_dataset_foreign(data_name)\n    else:\n        train, test = load_dataset_indonesian(data_name)\n\n    print('sampai2')\n\n    train_x_split, train_y_split, valid_x_split, valid_y_split = split_train_test(train.text.values,\n                                                                                  train.label.values,\n                                                                                  total_data=total_data,\n                                                                                  valid_size=valid_size)\n    print('sampai3')\n\n    train_x_split = np.array([x for x in train_x_split])\n    valid_x_split = np.array([x for x in valid_x_split])\n    return train_x_split, train_y_split, valid_x_split, valid_y_split\n\ndef load_experiment_dataset(data_name_indo,\n                            data_name_foreign,\n                            tipe='A', \n                            total_data=50, \n                            foreign_mult=1, \n                            valid_size=0.2):\n    \n    #################\n    # Load Raw Data #\n    #################\n    if tipe == 'A':\n        train_x, train_y, valid_x, valid_y = load_train_dataset(data_name_indo,\n                                                                total_data=total_data, \n                                                                valid_size=valid_size,\n                                                                is_foreign=False)\n        \n    elif tipe == 'B':\n        train_x, train_y, _, _ = load_train_dataset(data_name_foreign,\n                                                    total_data=total_data, \n                                                    valid_size=valid_size,\n                                                    is_foreign=True)\n        \n        _, _, valid_x, valid_y = load_train_dataset(data_name_indo,\n                                                    total_data=total_data, \n                                                    valid_size=valid_size,\n                                                    is_foreign=False)\n        \n    elif tipe == 'C':\n        train_x_indo, train_y_indo, valid_x_indo, valid_y_indo = load_train_dataset(data_name_indo,\n                                                                                    total_data=total_data, \n                                                                                    valid_size=valid_size,\n                                                                                    is_foreign=False)\n\n        train_x_foreign, train_y_foreign, valid_x_foreign, valid_y_foreign = load_train_dataset(data_name_foreign,\n                                                                                                total_data=int(total_data*foreign_mult), \n                                                                                                valid_size=valid_size,\n                                                                                                is_foreign=True)\n\n        train_x = np.concatenate([\n                    train_x_indo,\n                    train_x_foreign,\n                    ])\n\n        train_y = np.concatenate([\n                    train_y_indo,\n                    train_y_foreign,\n                ])\n\n        valid_x = valid_x_indo\n\n        valid_y = valid_y_indo\n        \n    \n\n    _, test = load_dataset_indonesian(data_name=data_name_indo)\n    test_x = test.text.values\n    test_x = np.array([x for x in test_x])\n    test_y = test.label.values\n    \n    indices = np.arange(len(train_x))\n    np.random.seed(RANDOM_SEED)\n    np.random.shuffle(indices)\n    train_x = train_x[indices]\n    train_y = train_y[indices]\n\n    return (train_x, train_y), (valid_x, valid_y), (test_x, test_y)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport torch\nimport tensorflow as tf\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nset_seed(seed=RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_TPU:\n    # Detect hardware, return appropriate distribution strategy\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    BATCH_SIZE = 8 * strategy.num_replicas_in_sync\n    \nelse:\n    BATCH_SIZE = 8 * 8\n\nAUTO = tf.data.experimental.AUTOTUNE\nMAX_LEN = 512\n\nif MODEL_NAME == 'XLM_R':\n    MODEL = 'jplu/tf-xlm-roberta-large'\nelif MODEL_NAME == 'mBERT':\n    MODEL = 'bert-base-multilingual-cased'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"(x_train, y_train), (x_valid, y_valid), (x_test, y_test) = load_experiment_dataset(INDO_DATA_NAME,\n                                                                                   FOREIGN_DATA_NAME,\n                                                                                   tipe=EXPERIMENT_TYPE, \n                                                                                   total_data=TOTAL_DATA, \n                                                                                   foreign_mult=FOREIGN_LANG_DATA_MULT, \n                                                                                   valid_size=VALIDATION_DATA)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.array([np.array([x[0], x[1]]) for x in y_train])\ny_valid = np.array([np.array([x[0], x[1]]) for x in y_valid])\ny_test = np.array([np.array([x[0], x[1]]) for x in y_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nx_train = regular_encode(x_train, tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(x_valid, tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(x_test, tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Datasets Objects","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(len(x_train),\n             seed=RANDOM_SEED)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif USE_TPU:\n    with strategy.scope():\n        transformer_layer = TFAutoModel.from_pretrained(MODEL)\n        model = build_model(transformer_layer, max_len=MAX_LEN, learning_rate=LEARNING_RATE)\n        \nelse:\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN, learning_rate=LEARNING_RATE)\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\nprint(n_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    callbacks = callback(), \n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights('model.h5') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_prediction = model.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_HS = pd.DataFrame()\nresult_HS['y_pred'] = test_prediction[:, 0]\nresult_HS['y_true'] = y_test[:, 0]\nresult_HS.to_csv('result_HS_{}_{}_{}_{}_{}_{}_full.csv'.format(INDO_DATA_NAME,\n                                                    FOREIGN_DATA_NAME,\n                                                    MODEL_NAME,\n                                                    EXPERIMENT_TYPE,\n                                                    TOTAL_DATA,\n                                                    FOREIGN_LANG_DATA_MULT),\n              index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_HS.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_Abusive = pd.DataFrame()\nresult_Abusive['y_pred'] = test_prediction[:, 1]\nresult_Abusive['y_true'] = y_test[:, 1]\nresult_Abusive.to_csv('result_Abusive_{}_{}_{}_{}_{}_{}_full.csv'.format(INDO_DATA_NAME,\n                                                    FOREIGN_DATA_NAME,\n                                                    MODEL_NAME,\n                                                    EXPERIMENT_TYPE,\n                                                    TOTAL_DATA,\n                                                    FOREIGN_LANG_DATA_MULT),\n              index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_Abusive.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}