{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TRAIN1_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\"\nTRAIN2_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv\"\nVALID_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\"\nTEST_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\"\n\ntrain1_df = pd.read_csv(TRAIN1_PATH, usecols=['comment_text', 'toxic']).fillna('none')\ntrain2_df = pd.read_csv(TRAIN2_PATH, usecols=['comment_text', 'toxic']).fillna('none')\ntrain_full_df = pd.concat([train1_df, train2_df], axis=0).reset_index(drop=True)\ntrain_df = train_full_df.sample(frac=1).reset_index(drop=True)\nvalid_df = pd.read_csv(VALID_PATH, usecols=['comment_text', 'toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.head(400000)\ntrain_df.comment_text.values.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import XLMRobertaTokenizer\n\nxlm_roberta_tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(text):\n    text = str(text).strip().lower()\n    return \" \".join(text.split())\n\n\ndef regular_encode(text, tokenizer, max_length):\n    output = tokenizer.batch_encode_plus(\n        text,\n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        add_special_tokens=False,\n        max_length=max_length)\n    \n    return np.array(output['input_ids']), np.array(output['attention_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_encodings = regular_encode(\n    text=train_df.comment_text.values, \n    tokenizer=xlm_roberta_tokenizer,\n    max_length=192)\n\ntrain_input_ids, train_attention_mask = train_encodings\n\nvalid_encodings = regular_encode(\n    text=valid_df.comment_text.values,\n    tokenizer=xlm_roberta_tokenizer,\n    max_length=192)\n\nvalid_input_ids, valid_attention_mask = valid_encodings\n\n\nnp.save('train_input_ids', train_input_ids)\nnp.save('train_attention_mask', train_attention_mask)\nnp.save('train_targets', train_df.toxic.values)\n\nnp.save('valid_input_ids', valid_input_ids)\nnp.save('valid_attention_mask', valid_attention_mask)\nnp.save('valid_targets', valid_df.toxic.values)\n\nprint(f\"Complete!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input_ids.shape, train_attention_mask.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}