{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\nimport string\nimport re\nimport random\nimport missingno as msno\nimport tensorflow as tf\nfrom transformers import TFAutoModel, AutoTokenizer\nimport os\nfrom tqdm.notebook import tqdm\nimport tensorflow_hub as hub\nfrom sklearn.model_selection import StratifiedKFold\nimport tensorflow.keras.backend as K\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading DataSets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n\ntrain2.toxic = train2.toxic.round().astype(int)\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']]\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Class Imbalance Problem","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 8))\nplt.title(\"Count of labels 0 vs 1\")\nplt.xlabel(\"Toxic\")\nplt.ylabel(\"Count\")\nsns.countplot(x = \"toxic\", data = train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1')\n#     train2[['comment_text', 'toxic']].query('toxic==0').sample(n=30000, random_state=0)\n]).sample(100000, random_state = 1)\ntrain = pd.concat([\n    train,\n    valid[[\"comment_text\", \"toxic\"]]\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 8))\nplt.title(\"Count of labels 0 vs 1\")\nplt.xlabel(\"Toxic\")\nplt.ylabel(\"Count\")\nsns.countplot(x = \"toxic\", data = train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (12, 8))\nmsno.bar(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef  word_cloud(data, title =None):\n    data = data.apply(lambda x : x.lower())\n    cloud = WordCloud(\n    background_color = \"black\",\n    stopwords = stopwords,\n    max_words = 200,\n    max_font_size = 40,\n    scale = 3).generate(str(data))\n    \n    fig = plt.figure(figsize= (15, 15))\n    plt.axis(\"off\")\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.25)\n\n    plt.imshow(cloud)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_cloud(train[\"comment_text\"], \"WordCloud for train data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_cloud(valid[\"comment_text\"].apply(str), \"WordCloud for valid data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_cloud(test[\"content\"], \"WordCloud for test data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 8))\nlen_sent = train[\"comment_text\"].apply(lambda x : len(x.split()))\nsns.distplot(len_sent.values)\nplt.title(\"Distribution of length of words\")\nplt.xlabel(\"Length of words\")\nplt.ylabel(\"Probability of occurance\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Max length of characters = {len_sent.values.max()}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tokenizer = AutoTokenizer.from_pretrained(\"bert-base-multilingual-cased\")\ntokenizer = AutoTokenizer.from_pretrained(\"jplu/tf-xlm-roberta-large\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"exp_text = \"I am currently participating in Jigsaw competition\"\ntokenizer.tokenize(exp_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(data, max_seq_length = MAX_LEN, tokenizer = tokenizer):    \n    ids = []\n    masks = []\n    segment = []\n    for i in tqdm(range(len(data))):\n        \n        tokens = tokenizer.tokenize(data[i])\n        if len(tokens) > max_seq_length - 2:\n            tokens = tokens[ : max_seq_length - 2]\n\n        # Converting tokens to ids\n        input_ids = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + tokens + [\"[SEP]\"])\n\n        # Input mask\n        input_masks = [1] * len(input_ids)\n\n        # padding upto max length\n        padding = max_seq_length - len(input_ids)\n        input_ids.extend([0] * padding)\n        input_masks.extend([0] * padding)\n        segment_ids =[0]* max_seq_length\n        \n        \n        ids.append(input_ids)\n        masks.append(input_masks)\n        segment.append(segment_ids)\n    \n    return (np.array(ids), np.array(masks), np.array(segment))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids, train_masks, train_segment =  preprocess(train[\"comment_text\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids, test_masks, test_segment =  preprocess(test[\"content\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ids, valid_masks, valid_segment =  preprocess(valid[\"comment_text\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train[\"toxic\"].values\ny_valid = valid[\"toxic\"].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Detecting TPUs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model(roberta_layer, max_len = MAX_LEN):\n    \n        input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n        input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n        segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n\n        pooled_output, sequence_output = roberta_layer([input_word_ids, input_mask, segment_ids])\n\n        # There are two outputs: a pooled_output of shape [batch_size, 768] with representations for \n        # the entire input sequences and a sequence_output of shape [batch_size, max_seq_length, 768] \n        # with representations for each input token (in context)\n\n\n        x = pooled_output\n        x = tf.keras.layers.Flatten()(x)\n        x = tf.keras.layers.Dense(128, activation = \"relu\")(x)\n        x = tf.keras.layers.Dense(1, activation = \"sigmoid\")(x)\n\n        model = tf.keras.Model(inputs = [input_word_ids, input_mask, segment_ids], outputs = x)\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    roberta_layer = TFAutoModel.from_pretrained(\"jplu/tf-xlm-roberta-large\", trainable = True)\n    model = model(roberta_layer)\n    model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle = True)\nskf.get_n_splits(train_ids, y_train)\nskf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 1\npreds = []\nfor train_index, test_index in skf.split(train_ids, y_train):\n    print(\"\\n\")\n    print(\"#\" * 20)\n    print(f\"FOLD No {i}\")\n    print(\"#\" * 20)\n    \n    \n    tr_ids = train_ids[train_index]\n    tr_masks = train_masks[train_index]\n    tr_segment = train_segment[train_index]\n    \n    vd_ids = train_ids[test_index]\n    vd_masks = train_masks[test_index]\n    vd_segment = train_segment[test_index]\n    \n    y_tr = y_train[train_index]\n    y_vd = y_train[test_index]\n    \n    \n    history = model.fit(\n    (tr_ids, tr_masks, tr_segment), y_tr,\n    epochs=2,\n    batch_size=BATCH_SIZE,\n    validation_data = ((vd_ids, vd_masks, vd_segment), y_vd),\n    steps_per_epoch = len(tr_ids)//BATCH_SIZE)\n\n    predictions = model.predict((test_ids, test_masks, test_segment))\n    preds.append(predictions)\n    \n    i += 1\n    K.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions =  (preds[0] + preds[1] + preds[2] + preds[3] + preds[4])/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predictions = model.predict((test_ids, test_masks, test_segment))\nsub[\"toxic\"] = predictions\nsub.set_index(\"id\", inplace = True)\nsub.to_csv(\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# References:\n1. [reference](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}