{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os, gc\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import *\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nfrom tqdm.autonotebook import  tqdm\nfrom ast import literal_eval\nimport tensorflow.keras.layers as L","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nAUTO = tf.data.experimental.AUTOTUNE\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nprint(\"PATH: \", GCS_DS_PATH)\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 2\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\ntrain_nobias = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n\ntrain_base = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid_base = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\ntest_base = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\n\ntrain_pre = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv\")\nvalid_pre = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv\")\ntest_pre = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv\")\n\nsub = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_TYPE =  \"bert-base-multilingual-cased\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained(MODEL_TYPE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_base.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenize_all(texts, tokenizer, chunk_size=512 ,max_len=512):\n    ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        chunk_ids = tokenizer.batch_encode_plus(texts[i: i+chunk_size])\n        ids.append(chunk_ids[\"input_ids\"])\n    \n    return ids\n    \n#train_ids = tokenize_all(train_base.comment_text.values, tokenizer)\n#valid_ids = tokenize_all(valid_base.comment_text.values, tokenizer)\n#test_ids = tokenize_all(test_base.comment_text.values, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pre.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_pre[\"input_word_ids\"] = train_pre[\"input_word_ids\"].apply(literal_eval)\nvalid_pre[\"input_word_ids\"] = valid_pre[\"input_word_ids\"].apply(literal_eval)\ntest_pre[\"input_word_ids\"] = test_pre[\"input_word_ids\"].apply(literal_eval)\n\ntrain_pre[\"input_mask\"] = train_pre[\"input_mask\"].apply(literal_eval)\nvalid_pre[\"input_mask\"] = valid_pre[\"input_mask\"].apply(literal_eval)\ntest_pre[\"input_mask\"] = test_pre[\"input_mask\"].apply(literal_eval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = train_pre.loc[:, [\"input_word_ids\", \"input_mask\"]]\nvalid_ids = valid_pre.loc[:, [\"input_word_ids\", \"input_mask\"]]\ntest_ids  = test_pre.loc[:, [\"input_word_ids\", \"input_mask\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_targets = train_pre.loc[:, [\"toxic\"]]#, \"severe_toxic\", \"obscene\", \"threat\",\t\"insult\", \"identity_hate\"]]\nvalid_targets = valid_pre.loc[:, [\"toxic\"]]#, \"severe_toxic\", \"obscene\", \"threat\",\t\"insult\", \"identity_hate\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_ids[\"input_word_ids\"] = train_ids[\"input_word_ids\"].apply(np.array, dtype=np.int32)\ntrain_ids[\"input_mask\"] = train_ids[\"input_mask\"].apply(np.array, dtype=np.int32)\n\nvalid_ids[\"input_word_ids\"] = valid_ids[\"input_word_ids\"].apply(np.array, dtype=np.int32)\nvalid_ids[\"input_mask\"] = valid_ids[\"input_mask\"].apply(np.array, dtype=np.int32)\n\ntest_ids[\"input_word_ids\"] = test_ids[\"input_word_ids\"].apply(np.array, dtype=np.int32)\ntest_ids[\"input_mask\"] = test_ids[\"input_mask\"].apply(np.array, dtype=np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = (tf.convert_to_tensor(train_ids.iloc[:, 0]), tf.convert_to_tensor(train_ids.iloc[:, 1]))\nvalid_data = (tf.convert_to_tensor(valid_ids.iloc[:, 0]), tf.convert_to_tensor(valid_ids.iloc[:, 1]))\ntest_data = (tf.convert_to_tensor(test_ids.iloc[:, 0]), tf.convert_to_tensor(test_ids.iloc[:, 1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_data[0], train_targets.values))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_data[0], valid_targets.values))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(test_data[0])\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Modelling "},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_multi_classifier(base_model, n_classes, max_len=128):\n    input_i = L.Input(shape=(max_len, ), dtype=tf.int32)\n    #input_m = L.Input(shape=(max_len, ), dtype=tf.int32)\n    \n    print(input_i.shape)\n    op1, op2 = base_model(input_i)\n    print(op1.shape)\n    print(op2.shape)\n    cls_token = op1[:, 0, :]\n    \"\"\"\n    cls_token = L.Dense(256, activation=\"relu\")(cls_token)\n    cls_token = L.Dropout(0.1)(cls_token)\n    \"\"\"\n    out = L.Dense(n_classes, activation='sigmoid')(cls_token)\n    \n    model = tf.keras.models.Model(inputs = input_i, outputs = out)\n    model.compile(optimizer=tf.keras.optimizers.Adam(lr=1.5e-5), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n    \n    return model\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    base_model = TFBertModel.from_pretrained(MODEL_TYPE)\n    model_ = get_multi_classifier(base_model, 1, 128)\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_.fit(train_dataset,\n           epochs=15,\n           steps_per_epoch=train_data[0].shape[0]//BATCH_SIZE,\n           validation_data=valid_dataset,\n           validation_steps=valid_data[0].shape[0]//BATCH_SIZE,\n           callbacks = [])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ps = model_.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pre.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ps_s=np.array(ps).squeeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = pd.DataFrame({\"id\":test_pre[\"id\"].values, \"toxic\":ps_s})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ps_s.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(ps_s>0.5).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}