{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nimport traitlets\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nfrom tokenizers import BertWordPieceTokenizer\nfrom sklearn.metrics import roc_auc_score\n\nwarnings.simplefilter(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, loss='binary_crossentropy', max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    x = tf.keras.layers.Dropout(0.35)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cosine similarity calculates similarity by measuring the cosine of angle between two vectors. This is calculated as:\n![](https://miro.medium.com/max/426/1*hub04IikybZIBkSEcEOtGA.png)\n\nCosine Similarity calculation for two vectors A and B [source]\nWith cosine similarity, we need to convert sentences into vectors. One way to do that is to use bag of words with either TF (term frequency) or TF-IDF (term frequency- inverse document frequency). The choice of TF or TF-IDF depends on application and is immaterial to how cosine similarity is actually performed — which just needs vectors. TF is good for text similarity in general, but TF-IDF is good for search query relevance."},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/questions/8897593/how-to-compute-the-similarity-between-two-text-documents\nimport nltk, string\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nnltk.download('punkt') # if necessary...\n\n\nstemmer = nltk.stem.porter.PorterStemmer()\nremove_punctuation_map = dict((ord(char), None) for char in string.punctuation)\n\ndef stem_tokens(tokens):\n    return [stemmer.stem(item) for item in tokens]\n\n'''remove punctuation, lowercase, stem'''\ndef normalize(text):\n    return stem_tokens(nltk.word_tokenize(text.lower().translate(remove_punctuation_map)))\n\nvectorizer = TfidfVectorizer(tokenizer=normalize, stop_words='english')\n\ndef cosine_sim(text1, text2):\n    tfidf = vectorizer.fit_transform([text1, text2])\n    return ((tfidf * tfidf.T).A)[0,1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Create strategy from tpu\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# Data access\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path('kaggle/input/') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create fast tokenizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased')\n\n# Save the loaded tokenizer locally\nsave_path = '/kaggle/working/distilbert_base_uncased/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)\n\n# Reload it with the huggingface tokenizers library\nfast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True)\nfast_tokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load text data into memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n\nvalid = pd.read_csv('/kaggle/input/val-en-df/validation_en.csv')\ntest1 = pd.read_csv('/kaggle/input/test-en-df/test_en.csv')\ntest2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test2.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test dataset comparision"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 8))\n\nsns.distplot(train1.comment_text.str.len(), label='train')\nsns.distplot(test1.content_en.str.len(), label='test1')\nsns.distplot(test2.translated.str.len(), label='test2')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 8))\n\nsns.distplot(train1.comment_text.str.len(), label='train')\nsns.distplot(test1.content_en.str.len(), label='test1')\nsns.distplot(test2.translated.str.len(), label='test2')\nplt.xlim([0, 512])\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets calculate cosine similarity two translated test datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set_similarity = [cosine_sim(t1, t2) for t1, t2 in tqdm(zip(test1.content_en, test2.translated))]\n\nplt.figure(figsize=(12, 8))\n\nsns.distplot(test_set_similarity);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Fast encode"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=512)\nx_valid = fast_encode(valid.comment_text_en.astype(str), fast_tokenizer, maxlen=512)\nx_test1 = fast_encode(test1.content_en.astype(str), fast_tokenizer, maxlen=512)\nx_test2 = fast_encode(test2.translated.astype(str), fast_tokenizer, maxlen=512)\n\ny_train = train1.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build datasets objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(64)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(64)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = [(\n    tf.data.Dataset\n    .from_tensor_slices(x_test1)\n    .batch(64)\n),\n    (\n    tf.data.Dataset\n    .from_tensor_slices(x_test2)\n    .batch(64)\n)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Focal Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load model into the TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = transformers.TFBertModel.from_pretrained('bert-base-uncased')\n    model = build_model(transformer_layer, loss=focal_loss(gamma=1.5), max_len=512)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RocAuc Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback \n\nclass RocAucCallback(Callback):\n    def __init__(self, test_data, score_thr):\n        self.test_data = test_data\n        self.score_thr = score_thr\n        self.test_pred = []\n        \n    def on_epoch_end(self, epoch, logs=None):\n        if logs['val_auc'] > self.score_thr:\n            print('\\nRun TTA...')\n            for td in self.test_data:\n                self.test_pred.append(self.model.predict(td))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LrScheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.000001, lr_max=0.000002, \n               lr_min=0.0000001, lr_rampup_epochs=7, \n               lr_sustain_epochs=0, lr_exp_decay=.87):\n    lr_max = lr_max * strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(10, 7))\n\n_lrfn = build_lrfn()\nplt.plot([i for i in range(35)], [_lrfn(i) for i in range(35)]);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc = RocAucCallback(test_dataset, 0.9195)\nlrfn = build_lrfn()\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=150,\n    validation_data=valid_dataset,\n    callbacks=[lr_schedule, roc_auc],\n    epochs=35\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = np.mean(roc_auc.test_pred, axis=0)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}