{"cells":[{"metadata":{},"cell_type":"markdown","source":"# imports"},{"metadata":{"_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nimport numpy as np\nimport joblib\nimport logging\nimport transformers\nimport sys\nimport torch.nn as nn\nimport gc;\n\nfrom scipy import stats\nfrom collections import OrderedDict, namedtuple\nfrom torch.optim import lr_scheduler\nfrom transformers import (\n    AdamW, get_linear_schedule_with_warmup, get_constant_schedule, \n    XLMRobertaTokenizer, XLMRobertaModel, XLMRobertaConfig,\n)\nfrom sklearn import metrics, model_selection\nfrom tqdm.autonotebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\n# load the data\n\ntrain1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"])\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"])\ntrain2.toxic = train2.toxic.round().astype(int)\n\ndf_valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\ndf_train = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=99937, random_state=0), # hacked to make train_data size divisible by bs;\n])\n\ndel train1, train2\ngc.collect(); gc.collect();\n\n# model = CustomRoberta();\n\nprint(df_train.shape, df_valid.shape)\ngc.collect(); gc.collect(); gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom joblib import Parallel, delayed\n\ndef regular_encode(texts, tokenizer=tokenizer, maxlen=128):\n    enc_di = tokenizer.encode_plus(\n        str(texts[0]),\n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids']), texts[1]\n\nrows = zip(df_train['comment_text'].values.tolist(), df_train.toxic.values.tolist())\nx_train = Parallel(n_jobs=4, backend='multiprocessing')(delayed(regular_encode)(row) for row in tqdm(rows))\n\nrows = zip(df_valid['comment_text'].values.tolist(), df_valid.toxic.values.tolist())\nx_valid = Parallel(n_jobs=4, backend='multiprocessing')(delayed(regular_encode)(row) for row in tqdm(rows))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(x_train).shape, np.array(x_valid).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(\"x_train_tokenized\", x_train)\nnp.save(\"x_valid_tokenized\", x_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(x_valid)[:,1]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}