{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\n\ntf.keras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\n\ndata_path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\n\nTEST_PATH = os.path.join(data_path, \"test.csv\")\nVAL_PATH = os.path.join(data_path, \"validation.csv\")\nTRAIN_PATH = os.path.join(data_path, \"jigsaw-toxic-comment-train.csv\")\n\nval_data = pd.read_csv(VAL_PATH)\ntest_data = pd.read_csv(TEST_PATH)\ntrain_data = pd.read_csv(TRAIN_PATH)\n\nprint(train_data.shape)\n\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Clean Text\nimport re\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    #replace newline characters with space\n    text = text.map(lambda x: re.sub('\\\\n',' ', str(x)))\n    text = text.map(lambda x: re.sub('\\[\\[User.*', '', str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval_data[\"comment_text\"] = clean(val_data[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain_data[\"comment_text\"] = clean(train_data[\"comment_text\"])   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load DistilBERT tokenizer\nimport transformers\n\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport tqdm\n\ndef create_bert_input(tokenizer, docs, max_seq_len):\n    all_input_ids, all_mask_ids = [], []\n    for doc in tqdm.tqdm(docs, desc=\"Converting docs to features\"):\n        tokens = tokenizer.tokenize(doc)\n        if len(tokens) > max_seq_len - 2:\n            tokens = tokens[0: (max_seq_len-2)]\n        tokens = ['[CLS]']+tokens+['[SEP]']\n        ids = tokenizer.convert_tokens_to_ids(tokens)\n        masks = [1]*len(ids)\n        while len(ids) < max_seq_len:\n            ids.append(0)\n            masks.append(0)\n        all_input_ids.append(ids)\n        all_mask_ids.append(masks)\n    \n    encoded = np.array([all_input_ids, all_mask_ids])\n    return encoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_comments = train_data.comment_text.astype(str).values\nval_comments = val_data.comment_text.astype(str).values\ntest_comments = test_data.content.astype(str).values\n\ny_valid = val_data.toxic.values\ny_train = train_data.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Encode the comments in train_set\nMAX_SEQ_LENGTH = 500\n\ntrain_feature_ids, train_feature_masks = create_bert_input(tokenizer, train_comments, max_seq_len=MAX_SEQ_LENGTH)\n\nval_feature_ids, val_feature_masks = create_bert_input(tokenizer, val_comments, max_seq_len=MAX_SEQ_LENGTH)\n\ntest_feature_ids, test_feature_masks = create_bert_input(tokenizer, test_comments, max_seq_len=MAX_SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Configure TPU\nfrom kaggle_datasets import KaggleDatasets\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nEPOCHS = 1\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create TensorFlow datasets for better performance\ntrain_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((train_feature_ids, train_feature_masks), y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)\n    \nvalid_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((val_feature_ids, val_feature_masks), y_valid))\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)\n\ntest_ds = (\n    tf.data.Dataset\n    .from_tensor_slices((test_feature_ids, test_feature_masks))\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create training ready model\ndef get_training_model():\n    # Build the model\n    print('Build model...')\n    model = tf.keras.models.Sequential()\n    model.add(tf.keras.layers.LSTM(units = 32, activation = 'tanh', return_sequences=True))\n    model.add(tf.keras.layers.Dropout(0.2))\n    model.add(tf.keras.layers.LSTM(units = 32, activation = 'tanh', return_sequences=True))\n    model.add(tf.keras.layers.LSTM(units = 32, activation = 'tanh', return_sequences=True))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nimport time\n\nstart = time.time()\n\n# Compile the model with TPU Strategy\nwith strategy.scope():\n    model = get_training_model()\n    \nmodel.fit(train_ds, steps_per_epoch=train_data.shape[0] // BATCH_SIZE, validation_data=valid_ds,validation_steps=val_data.shape[0] // BATCH_SIZE, epochs=EPOCHS, verbose=1)\nend = time.time() - start\nprint(\"Time taken \",end)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}