{"cells":[{"metadata":{},"cell_type":"markdown","source":"# XLM-Roberta Large tokenize dataset\n\nThis kernel tokenizes the whole dataset ahead of time and saves it in npy file format for later loading in order to save time during training.\n\nBased on [abhishek's](https://www.kaggle.com/abhishek/bert-multi-lingual-tpu-training-8-cores-w-valid) and [xhlulu's](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta) kernels.","execution_count":null},{"metadata":{"_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\nimport sys\nfrom sklearn import metrics, model_selection\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-large', do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train2.toxic = df_train2.toxic.round().astype(int)\n\ndf_valid = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv', \n                       usecols=[\"comment_text\", \"toxic\"])\n\n\n# Combine train1 with a subset of train2\ndf_train = pd.concat([\n    df_train1[['comment_text', 'toxic']],\n    df_train2[['comment_text', 'toxic']].query('toxic==1'),\n    df_train2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])\n\ndf_test = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nx_train = regular_encode(df_train.comment_text.values, tokenizer, maxlen=192)\nx_valid = regular_encode(df_valid.comment_text.values, tokenizer, maxlen=192)\nx_test  = regular_encode(df_test.content.values,       tokenizer, maxlen=192)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('x_train',x_train)\nnp.save('x_valid',x_valid)\nnp.save('x_test',  x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('df_train_toxic',df_train.toxic.values)\nnp.save('df_valid_toxic',df_valid.toxic.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('test_df_ids',df_test.id.values)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}