{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pathlib\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-17T10:02:51.666636Z","iopub.execute_input":"2021-08-17T10:02:51.666986Z","iopub.status.idle":"2021-08-17T10:02:51.671475Z","shell.execute_reply.started":"2021-08-17T10:02:51.666955Z","shell.execute_reply":"2021-08-17T10:02:51.670385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Kudos to [this Kaggle kernel](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta).","metadata":{}},{"cell_type":"code","source":"ROOT_PATH = pathlib.Path(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\")\nMODEL = \"distilbert-base-multilingual-cased\"\nBATCH_SIZE = 32\nEPOCHS = 1\nMAX_DOC_LENGTH = 256","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:52:20.862826Z","iopub.execute_input":"2021-08-17T09:52:20.863159Z","iopub.status.idle":"2021-08-17T09:52:20.869429Z","shell.execute_reply.started":"2021-08-17T09:52:20.863123Z","shell.execute_reply":"2021-08-17T09:52:20.868708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(ROOT_PATH / \"jigsaw-toxic-comment-train.csv\")\nvalid_df = pd.read_csv(ROOT_PATH / \"validation.csv\")\ntest_df = pd.read_csv(ROOT_PATH / \"test.csv\").rename(columns={\"content\": \"comment_text\"})\ntrain_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:53:43.264668Z","iopub.execute_input":"2021-08-17T09:53:43.265057Z","iopub.status.idle":"2021-08-17T09:53:46.972984Z","shell.execute_reply.started":"2021-08-17T09:53:43.265025Z","shell.execute_reply":"2021-08-17T09:53:46.972138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"toxic\"].mean(), valid_df[\"toxic\"].mean()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:53:46.974525Z","iopub.execute_input":"2021-08-17T09:53:46.974857Z","iopub.status.idle":"2021-08-17T09:53:46.982260Z","shell.execute_reply.started":"2021-08-17T09:53:46.974823Z","shell.execute_reply":"2021-08-17T09:53:46.981296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Tokenizer:\n    def __init__(\n        self,\n        tokenizer,\n        max_doc_length: int,\n        padding = True,\n    ) -> None:\n        self.tokenizer = tokenizer\n        self.max_doc_length = max_doc_length\n        self.padding = padding\n\n    def __call__(self, x):\n        return self.tokenizer(\n            x,\n            max_length=self.max_doc_length,\n            truncation=True,\n            padding=self.padding,\n            return_tensors=\"tf\",\n        )\n    \ntokenizer = Tokenizer(AutoTokenizer.from_pretrained(MODEL), MAX_DOC_LENGTH)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:53:49.199622Z","iopub.execute_input":"2021-08-17T09:53:49.200047Z","iopub.status.idle":"2021-08-17T09:53:54.693069Z","shell.execute_reply.started":"2021-08-17T09:53:49.200013Z","shell.execute_reply":"2021-08-17T09:53:54.692205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_tokenized_values(text, tokenizer, batch_size):\n    input_ids = []\n    attention_mask = []\n    for i in tqdm(range(0, len(text), batch_size)):\n        tokenized_batch = tokenizer(text[i: i+batch_size])\n        input_ids.append(tokenized_batch[\"input_ids\"])\n        attention_mask.append(tokenized_batch[\"attention_mask\"])\n        \n    return tf.concat(input_ids, axis=0), tf.concat(attention_mask, axis=0)\n\ntrain_input_ids, train_attention_mask = get_tokenized_values(train_df[\"comment_text\"].values.tolist(), tokenizer, BATCH_SIZE * 4)\nvalid_input_ids, valid_attention_mask = get_tokenized_values(valid_df[\"comment_text\"].values.tolist(), tokenizer, BATCH_SIZE * 4)\ntest_input_ids, test_attention_mask = get_tokenized_values(test_df[\"comment_text\"].values.tolist(), tokenizer, BATCH_SIZE * 4)\n\ny_train = train_df.toxic.values\ny_valid = valid_df.toxic.values","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:53:54.694558Z","iopub.execute_input":"2021-08-17T09:53:54.694909Z","iopub.status.idle":"2021-08-17T09:55:17.211302Z","shell.execute_reply.started":"2021-08-17T09:53:54.694874Z","shell.execute_reply":"2021-08-17T09:55:17.210347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(((train_input_ids, train_attention_mask), y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(BATCH_SIZE * 2)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(((valid_input_ids, valid_attention_mask), y_valid))\n    .batch(BATCH_SIZE)\n    .prefetch(BATCH_SIZE * 2)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(((test_input_ids, test_attention_mask), np.ones(len(test_input_ids))))\n    .batch(BATCH_SIZE)\n    .prefetch(BATCH_SIZE * 2)\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:55:17.213406Z","iopub.execute_input":"2021-08-17T09:55:17.213834Z","iopub.status.idle":"2021-08-17T09:55:17.241951Z","shell.execute_reply.started":"2021-08-17T09:55:17.213792Z","shell.execute_reply":"2021-08-17T09:55:17.241214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = next(iter(train_dataset))\nx[0].shape, x[1].shape","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:55:17.243460Z","iopub.execute_input":"2021-08-17T09:55:17.243808Z","iopub.status.idle":"2021-08-17T09:55:17.631615Z","shell.execute_reply.started":"2021-08-17T09:55:17.243774Z","shell.execute_reply":"2021-08-17T09:55:17.630695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n\n- Transformers: https://jalammar.github.io/illustrated-transformer/\n- BERT: https://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/","metadata":{}},{"cell_type":"code","source":"bert_model = TFAutoModel.from_pretrained(MODEL)\n\ninput_ids = keras.layers.Input(shape=(MAX_DOC_LENGTH,), dtype=tf.int32)\nattention_mask = keras.layers.Input(shape=(MAX_DOC_LENGTH,), dtype=tf.int32)\nsequence_output = bert_model(input_ids, attention_mask)[0]\ncls_token = sequence_output[:, 0, :]\nout = keras.layers.Dense(1, activation=\"sigmoid\")(cls_token)\n\nmodel = keras.models.Model(inputs=(input_ids, attention_mask), outputs=out)\nmodel.compile(keras.optimizers.Adam(lr=1e-5), loss=\"binary_crossentropy\", metrics=[\"accuracy\", keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:55:17.633294Z","iopub.execute_input":"2021-08-17T09:55:17.633923Z","iopub.status.idle":"2021-08-17T09:55:48.159677Z","shell.execute_reply.started":"2021-08-17T09:55:17.633880Z","shell.execute_reply":"2021-08-17T09:55:48.158850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:55:48.160886Z","iopub.execute_input":"2021-08-17T09:55:48.161267Z","iopub.status.idle":"2021-08-17T09:55:48.176587Z","shell.execute_reply.started":"2021-08-17T09:55:48.161219Z","shell.execute_reply":"2021-08-17T09:55:48.175617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training\nAlso worth trying: https://datascience.stackexchange.com/a/13496/32796","metadata":{}},{"cell_type":"code","source":"n_steps = train_input_ids.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:55:48.177918Z","iopub.execute_input":"2021-08-17T09:55:48.178435Z","iopub.status.idle":"2021-08-17T10:00:59.946444Z","shell.execute_reply.started":"2021-08-17T09:55:48.178396Z","shell.execute_reply":"2021-08-17T10:00:59.943899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(valid_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:09:59.331290Z","iopub.execute_input":"2021-08-17T10:09:59.331606Z","iopub.status.idle":"2021-08-17T10:10:40.035831Z","shell.execute_reply.started":"2021-08-17T10:09:59.331577Z","shell.execute_reply":"2021-08-17T10:10:40.035015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(ROOT_PATH / \"sample_submission.csv\")\nsub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:03:43.452384Z","iopub.execute_input":"2021-08-17T10:03:43.452867Z","iopub.status.idle":"2021-08-17T10:08:51.950015Z","shell.execute_reply.started":"2021-08-17T10:03:43.452820Z","shell.execute_reply":"2021-08-17T10:08:51.945694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}