{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport pandas as pd\nimport os\nimport numpy as np\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Input, Dense, Embedding, Activation, LSTM, SimpleRNN, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras import backend as K\n\nimport transformers\nfrom tokenizers import BertWordPieceTokenizer\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LoadingData():\n            \n    def __init__(self):\n        train_file_path = os.path.join(\"..\",\"input\",\"nlp-benchmarking-data-for-intent-and-entity\",\"benchmarking_data\",\"Train\")\n        validation_file_path = os.path.join(\"..\",\"input\",\"nlp-benchmarking-data-for-intent-and-entity\",\"benchmarking_data\",\"Validate\")\n        \n        self.train_df1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\n        self.train_df2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\n        self.train_df2.toxic = self.train_df2.toxic.round().astype(int)\n\n        self.train_df = pd.concat([self.train_df1[['comment_text', 'toxic']],self.train_df2[['comment_text', 'toxic']].query('toxic==1'),\n                           self.train_df2[['comment_text', 'toxic']].query('toxic==0').sample(n=150000, random_state=3982)])\n        \n        self.validation_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n        \n        self.test_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_data_obj = LoadingData()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_data_obj.train_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertModel(object):\n    def __init__(self):\n        self.tokenizer = None\n        self.model_name = 'distilbert-base-multilingual-cased'\n    def set_tokenizer(self):\n        self.tokenizer = transformers.DistilBertTokenizerFast.from_pretrained(self.model_name)\n        # Save it locally\n        self.tokenizer.save_pretrained('.')\n        # Load it with huggingface transformer library\n        self.fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_model_obj = BertModel()\nbert_model_obj.set_tokenizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PreprocessingBertData():\n    def __init__(self):\n        self.nb_epochs = 1\n        self.batch_size = 16 * strategy.num_replicas_in_sync\n        self.maxLen = 192\n        self.x_train = None\n        self.y_train = None\n        self.x_valid = None\n        self.y_valid = None\n        self.x_test = None\n        \n    def encode_text(self,text, tokenizer, chunk_size=256, maxlen=512):\n        tokenizer.enable_truncation(max_length=maxlen)\n        tokenizer.enable_padding(max_length=maxlen)\n        ids = list()\n        for i in tqdm(range(0, len(text), chunk_size)):\n            current_chunk = text[i:i+chunk_size].tolist()\n            encs = tokenizer.encode_batch(current_chunk)\n            ids.extend([enc.ids for enc in encs])\n        return np.array(ids)\n    \n    def tokenize_data(self):\n        self.x_train = self.encode_text(load_data_obj.train_df.comment_text.astype(str), tokenizer=bert_model_obj.fast_tokenizer, maxlen=self.maxLen)\n        self.x_test = self.encode_text(load_data_obj.test_df.content.astype(str), tokenizer=bert_model_obj.fast_tokenizer, maxlen=self.maxLen)\n        self.x_valid = self.encode_text(load_data_obj.validation_df.comment_text.astype(str), tokenizer=bert_model_obj.fast_tokenizer, maxlen=self.maxLen)\n\n        self.y_train = load_data_obj.train_df.toxic.values\n        self.y_valid = load_data_obj.validation_df.toxic.values\n        \n        \n    def prepare_data_x(self,train_sentences):\n        x = bert_model_obj.create_input_array(train_sentences)\n        return x\n    \n    def prepare_data_y(self,train_labels):\n        y = list()\n        for item in train_labels:\n            label = item\n            y.append(label)\n        y = np.array(y)\n        return y\n    def get_dataset(self):\n        AUTO = tf.data.experimental.AUTOTUNE\n        train_dataset = (tf.data.Dataset\n                         .from_tensor_slices((self.x_train, self.y_train))\n                         .repeat()\n                         .shuffle(2048)\n                         .batch(self.batch_size)\n                         .prefetch(AUTO)\n                        )\n        valid_dataset = (tf.data.Dataset\n                         .from_tensor_slices((self.x_valid, self.y_valid))\n                         .batch(self.batch_size)\n                         .cache()\n                         .prefetch(AUTO)\n                        )\n        test_dataset = (tf.data.Dataset\n                        .from_tensor_slices(self.x_test)\n                        .batch(self.batch_size)\n                       )\n        return train_dataset,valid_dataset,test_dataset\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess_bert_data_obj = PreprocessingBertData()\npreprocess_bert_data_obj.tokenize_data()\ntrain_data,valid_data,test_data = preprocess_bert_data_obj.get_dataset()\ntrain_len = preprocess_bert_data_obj.x_train.shape[0]\nvalid_len = preprocess_bert_data_obj.x_valid.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DesignModel():\n    def __init__(self):\n        self.model = None \n        self.train_history = None\n        self.valid_history = None\n        \n    def bert_model(self,transformer,max_seq_length=512): \n        \n        bert_inputs = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name=\"input_ids\")\n        sequence_output = transformer(bert_inputs)[0]\n        class_token = seq_output[:, 0, :]\n        bert_out = tf.keras.layers.GlobalAveragePooling1D()(class_token)\n        bert_out = tf.keras.layers.Dropout(0.2)(bert_out)\n        bert_outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(bert_out)\n        self.model = tf.keras.models.Model(inputs=bert_inputs, outputs=bert_outputs)\n        \n        self.model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n        \n        self.model.summary()\n    \n    def model_train(self,batch_size,num_epoch):\n        n_steps = train_len // batch_size\n        print(\"Fitting to model\")\n        self.train_history = self.model.fit(train_data,steps_per_epoch=n_steps,validation_data=valid_data,epochs=num_epoch)        \n        print(\"Model Training complete.\")\n        n_steps = valid_len // batch_size\n        self.valid_history = self.model.fit(valid_data.repeat(),steps_per_epoch=n_steps,epochs=num_epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_len = 192\nnum_epoch = 5\nbatch_size = 16 * strategy.num_replicas_in_sync\nmodel_obj = DesignModel()\nwith strategy.scope():\n    transformer = (transformers.TFDistilBertModel.from_pretrained('distilbert-base-multilingual-cased'))\n    model_obj.bert_model(transformer,max_seq_length=max_len)\n    model_obj.model_train(batch_size,num_epoch)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsub['toxic'] = model_obj.model.predict(test_data, verbose=1)\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}