{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install pyspellchecker\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\nimport re\nfrom spellchecker import SpellChecker\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2.toxic = train2.toxic.round().astype(int)\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Clean data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"remove username, urls, etc","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1[\"comment_text\"] = clean(train1[\"comment_text\"])\ntrain2[\"comment_text\"] = clean(train2[\"comment_text\"])\nvalid[\"comment_text\"] = clean(valid[\"comment_text\"])\ntest[\"content\"] = clean(test[\"content\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Remove entries with more misspells","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"spell = SpellChecker()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n# Counting the number of spelling errors\ntrain1['mispell_count'] = train1['comment_text'].progress_apply(lambda x: len(spell.unknown(x.split())))\ntrain2['mispell_count'] = train2['comment_text'].progress_apply(lambda x: len(spell.unknown(x.split())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1[train1['mispell_count'] < 100]['mispell_count'].hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train2[train2['mispell_count'] < 100]['mispell_count'].hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check how many toxic comments have more than 20 mispells (Labels are already imbalanced; We do not want to lose them)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train1[train1['mispell_count'] >= 20].toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train2[train2['mispell_count'] >= 20].toxic)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Remove non-toxic comments with more than 20 misspells","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = train1.query('toxic==0 & mispell_count < 20 | toxic==1')\ntrain2 = train2.query('toxic==0 & mispell_count < 20 | toxic==1')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Subsample the data labeled 0 to balance the labeling**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examine number of data labeled 1 or 0\nprint('train2 has %d entries labeled 1' % train2[['comment_text', 'toxic']].query('toxic==1').shape[0])\nprint('train1 has %d entries labeled 1' % train1[['comment_text', 'toxic']].query('toxic==1').shape[0])\nprint('train2 has %d entries labeled 0' % train2[['comment_text', 'toxic']].query('toxic==0').shape[0])\nprint('train1 has %d entries labeled 0' % train1[['comment_text', 'toxic']].query('toxic==0').shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine train1 with a subset of train2 \n# (decided to subsample 100,000 samples from train2 labeld 0)\ntrain = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Encode data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\nMODEL = 'jplu/tf-xlm-roberta-large'\n# Configuration\nMAX_LEN = 192\ntokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nx_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU configs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\nEPOCHS = 2\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(len(train))\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', \n                  metrics=[\n                          keras.metrics.TruePositives(name='tp'),\n                          keras.metrics.FalsePositives(name='fp'),\n                          keras.metrics.TrueNegatives(name='tn'),\n                          keras.metrics.FalseNegatives(name='fn'), \n                          keras.metrics.BinaryAccuracy(name='accuracy'),\n                          keras.metrics.Precision(name='precision'),\n                          keras.metrics.Recall(name='recall'),\n                          keras.metrics.AUC(name='auc')\n                  ])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (12, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metrics(history, val=True):\n    metrics =  ['accuracy', 'auc', 'precision', 'recall']\n    colors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n    for n, metric in enumerate(metrics):\n        name = metric.replace(\"_\",\" \").capitalize()\n        plt.subplot(2, 2, n+1)\n        plt.plot(history.epoch,  history.history[metric], color=colors[0], label='Train')\n        if val:\n            plt.plot(history.epoch, history.history['val_' + metric],\n                     color=colors[0], linestyle=\"--\", label='Val')\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        plt.ylim([0, 1])\n\n        plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_metrics(train_history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_metrics(train_history_2, val=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}