{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install bert-for-tf2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport os\nimport numpy as np\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Input, Dense, Embedding, Activation, LSTM, SimpleRNN, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport bert\nfrom tqdm import tqdm\nfrom tensorflow.keras import backend as K\nimport tensorflow as tf\nimport tensorflow_hub as hub\nprint(\"TensorFlow Version:\",tf.__version__)\nprint(\"Hub version: \",hub.__version__)\n# Params for bert model and tokenization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LoadingData():\n            \n    def __init__(self):\n        train_file_path = os.path.join(\"..\",\"input\",\"nlp-benchmarking-data-for-intent-and-entity\",\"benchmarking_data\",\"Train\")\n        validation_file_path = os.path.join(\"..\",\"input\",\"nlp-benchmarking-data-for-intent-and-entity\",\"benchmarking_data\",\"Validate\")\n        \n        self.train_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\n        self.validation_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n        self.test_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n        self.train_df = self.train_df.sample(frac = 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_data_obj = LoadingData()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_data_obj.train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertModel(object):\n    \n    def __init__(self):\n        \n        self.max_len = 128\n        #bert_path = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\"\n        bert_path = \"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\"\n        FullTokenizer=bert.bert_tokenization.FullTokenizer\n        \n        self.bert_module = hub.KerasLayer(bert_path,trainable=True)\n\n        self.vocab_file = self.bert_module.resolved_object.vocab_file.asset_path.numpy()\n\n        self.do_lower_case = self.bert_module.resolved_object.do_lower_case.numpy()\n\n        self.tokenizer = FullTokenizer(self.vocab_file,self.do_lower_case)\n        \n    def get_masks(self,tokens, max_seq_length):\n        return [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n\n    def get_segments(self,tokens, max_seq_length):\n        \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n        segments = []\n        current_segment_id = 0\n        for token in tokens:\n            segments.append(current_segment_id)\n            if token == \"[SEP]\":\n                current_segment_id = 1\n        return segments + [0] * (max_seq_length - len(tokens))\n    def get_ids(self,tokens, tokenizer, max_seq_length):\n        \"\"\"Token ids from Tokenizer vocab\"\"\"\n        token_ids = tokenizer.convert_tokens_to_ids(tokens,)\n        input_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n        return input_ids\n    def create_single_input(self,sentence,maxlen):\n\n        stokens = self.tokenizer.tokenize(sentence)\n\n        stokens = stokens[:maxlen]\n\n        stokens = [\"[CLS]\"] + stokens + [\"[SEP]\"]\n\n        ids = self.get_ids(stokens, self.tokenizer, self.max_len)\n        masks = self.get_masks(stokens, self.max_len)\n        segments = self.get_segments(stokens, self.max_len)\n\n        return ids,masks,segments\n\n    def create_input_array(self,sentences):\n        \n        input_ids, input_masks, input_segments = [], [], []\n\n        for sentence in tqdm(sentences,position=0, leave=True):\n            ids,masks,segments=self.create_single_input(sentence,self.max_len-2)\n\n            input_ids.append(ids)\n            input_masks.append(masks)\n            input_segments.append(segments)\n            \n        tensor = [np.asarray(input_ids, dtype=np.int32), \n                np.asarray(input_masks, dtype=np.int32), \n                np.asarray(input_segments, dtype=np.int32)]\n        return tensor\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PreprocessingBertData():\n    \n    def prepare_data_x(self,train_sentences):\n        x = bert_model_obj.create_input_array(train_sentences)\n        return x\n    \n    def prepare_data_y(self,train_labels):\n        y = list()\n        for item in train_labels:\n            label = item\n            y.append(label)\n        y = np.array(y)\n        return y\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_model_obj = BertModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_classes = [\"toxic\"]\n\ntrain_sentences = load_data_obj.train_df[\"comment_text\"].values\ntrain_labels = load_data_obj.train_df[\"toxic\"].values\n\npreprocess_bert_data_obj = PreprocessingBertData()\nx = preprocess_bert_data_obj.prepare_data_x(train_sentences)\ny = preprocess_bert_data_obj.prepare_data_y(train_labels)\n\ntrain_input_ids, train_input_masks, train_segment_ids = x\ntrain_labels = y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DesignModel():\n    def __init__(self):\n        self.model = None        \n        self.train_data = [train_input_ids, train_input_masks, train_segment_ids]\n        self.train_labels = train_labels\n        \n    def bert_model(self,max_seq_length): \n        in_id = Input(shape=(max_seq_length,), dtype=tf.int32, name=\"input_ids\")\n        in_mask = Input(shape=(max_seq_length,), dtype=tf.int32, name=\"input_masks\")\n        in_segment = Input(shape=(max_seq_length,), dtype=tf.int32, name=\"segment_ids\")\n        \n        bert_inputs = [in_id, in_mask, in_segment]\n        pooled_output, sequence_output = bert_model_obj.bert_module(bert_inputs)\n        \n        x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)\n        x = tf.keras.layers.Dropout(0.2)(x)\n        out = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n        self.model = tf.keras.models.Model(inputs=bert_inputs, outputs=out)\n        \n        self.model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n        \n        self.model.summary()\n    \n    def model_train(self,batch_size,num_epoch):\n        n_steps = len(self.train_data) // batch_size\n        print(\"Fitting to model\")\n        \n        self.model.fit(self.train_data,self.train_labels,steps_per_epoch=n_steps,epochs=num_epoch,batch_size=batch_size,validation_split=0.2,shuffle=True)\n        \n        print(\"Model Training complete.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_obj = DesignModel()\nmodel_obj.bert_model(bert_model_obj.max_len)\nmodel_obj.model_train(32,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_sentences = load_data_obj.test_df[\"content\"].values\ninputs = preprocess_bert_data_obj.prepare_data_x(test_sentences)\nresult = model_obj.model.predict(inputs,verbose=1)\nprint(result[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsub['toxic'] = result\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}