{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tqdm import tqdm\nfrom multiprocessing.pool import Pool\n\nimport os\nimport sys\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from transformers import BertTokenizer\nfrom transformers import XLMRobertaTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN1_PATH = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv'\nTRAIN2_PATH = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv'\nVALID_PATH = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv'\nTEST_PATH = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv'\n\ntrain1_df = pd.read_csv(TRAIN1_PATH, usecols=['comment_text', 'toxic']).fillna('none')\ntrain2_df = pd.read_csv(TRAIN2_PATH, usecols=['comment_text', 'toxic']).fillna('none')\ntrainfull_df = pd.concat([train1_df, train2_df], axis=0).reset_index(drop=True)\n\ntrain_df = trainfull_df.sample(frac=1, random_state=42)\nvalid_df = pd.read_csv(VALID_PATH, usecols=['comment_text', 'toxic'])\ntest_df = pd.read_csv(TEST_PATH, usecols=['content'])\n\ntrain1_df.shape, train2_df.shape, valid_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n#xlm_roberta_tokenizer = XLMRobertaTokenizer.from_pretrained('')\n\nMAX_LENGTH = 192\n#num_cores = 4\ntrain_N = 800000\ntest_N = test_df.shape[0]\nvalid_N = valid_df.shape[0]\n\n\n# get dataframes\ntrain_df = train_df.head(train_N)\ntest_df = test_df\nvalid_df = valid_df\n\n\n# remove wierd spaces and convert to lower case\ndef preprocessing(text):\n    text = str(text).strip().lower()\n    return \" \".join(text.split())\n\n\n# encode string for each subprocess\ndef token_encoding(t, target, tokenizer, max_length):\n    # there is no target for the test case\n    if target is True:\n        texts = t[0]\n        targets = t[1]\n    else:\n        texts = t\n    \n    input_ids = []\n    token_type_ids = []\n    attention_mask = []\n    \n    for i in tqdm(range(0, len(texts))):\n        text = preprocessing(texts[i])\n        inputs = tokenizer.encode_plus(text,\n                                       None,\n                                       pad_to_max_length=True, \n                                       max_length=max_length)\n        \n        input_ids.append(inputs['input_ids'])\n        token_type_ids.append(inputs['token_type_ids'])\n        attention_mask.append(inputs['attention_mask'])\n    \n    if target is True:\n        return np.array(input_ids), np.array(token_type_ids), np.array(attention_mask), np.array(targets)\n    \n    else:\n        return np.array(input_ids), np.array(token_type_ids), np.array(attention_mask)\n\n\ndef encoding_process(df, N, num_cores, tokenizer, max_length):\n    \n    text1 = df.comment_text.values[:int(N/4)]\n    target1 = df.toxic.values[:int(N/4)]\n    \n    text2 = df.comment_text.values[int(N/4):int(N/2)]\n    target2 = df.toxic.values[int(N/4):int(N/2)]\n    \n    text3 = df.comment_text.values[int(N/2):int(0.75 * N)]\n    target3 = df.toxic.values[int(N/2):int(0.75 * N)]\n    \n    text4 = df.comment_text.values[int(0.75 * N):]\n    target4 = df.toxic.values[int(0.75 * N):]\n    \n    process_pool = Pool(num_cores)\n    \n    chunk1 = ((text1, target1), True, tokenizer, max_length)\n    chunk2 = ((text2, target2), True, tokenizer, max_length)\n    chunk3 = ((text3, target3), True, tokenizer, max_length)\n    chunk4 = ((text4, target4), True, tokenizer, max_length)\n    \n    chunks = [chunk1, chunk2, chunk3, chunk4]\n    \n    output = process_pool.starmap(token_encoding, chunks)\n    \n    input_ids = np.concatenate([out[0] for out in output], axis=0)\n    token_type_ids = np.concatenate([out[1] for out in output], axis=0)\n    attention_mask = np.concatenate([out[2] for out in output], axis=0)\n    targets = np.concatenate([out[3] for out in output], axis=0)\n    \n    assert input_ids.shape[0] == token_type_ids.shape[0] \\\n            == attention_mask.shape[0] == targets.shape[0] == N\n    \n    return input_ids, token_type_ids, attention_mask, targets\n    \n\ndef test_encoding_process(df, N, num_cores, tokenizer, max_length):\n    \n    text1 = df.content.values[:int(N/4)]\n    \n    text2 = df.content.values[int(N/4):int(N/2)]\n    \n    text3 = df.content.values[int(N/2):int(0.75 * N)]\n    \n    text4 = df.content.values[int(0.75 * N):]\n    \n    process_pool = Pool(num_cores)\n    \n    chunk1 = (text1, False, tokenizer, max_length)\n    chunk2 = (text2, False, tokenizer, max_length)\n    chunk3 = (text3, False, tokenizer, max_length)\n    chunk4 = (text4, False, tokenizer, max_length)\n    \n    chunks = [chunk1, chunk2, chunk3, chunk4]\n    \n    output = process_pool.starmap(token_encoding, chunks)\n    \n    input_ids = np.concatenate([out[0] for out in output], axis=0)\n    token_type_ids = np.concatenate([out[1] for out in output], axis=0)\n    attention_mask = np.concatenate([out[2] for out in output], axis=0)\n    \n    assert input_ids.shape[0] == token_type_ids.shape[0] == attention_mask.shape[0] == N\n    \n    return input_ids, token_type_ids, attention_mask\n\n\ndef save_data(compressed=False):\n    if compressed is True:\n        np.savez_compressed('train-df-compressed-input-ids.npz', train_input_ids)\n        np.savez_compressed('train-df-compressed-attention-mask.npz', train_attention_mask)\n        np.savez_compressed('train-df-compressed-token-type-ids.npz', train_token_type_ids)\n        np.savez_compressed('train-df-compressed-targets.npz', train_targets)\n        \n        # valid\n        np.savez_compressed('valid-df-compressed-input-ids.npz', valid_input_ids)\n        np.savez_compressed('valid-df-compressed-attention-mask.npz', valid_attention_mask)\n        np.savez_compressed('valid-df-compressed-token-type-ids.npz', valid_token_type_ids)\n        np.savez_compressed('valid-df-compressed-targets.npz', valid_targets)\n        \n        # test\n        np.savez_compressed('test-df-compressed-input-ids.npz', test_input_ids)\n        np.savez_compressed('test-df-compressed-attention-mask.npz', test_attention_mask)\n        np.savez_compressed('test-df-compressed-token-type-ids.npz', test_token_type_ids)\n\n    else:\n        np.save('train-df-input-ids.npy', input_ids)\n        np.save('train-df-attention-mask.npy', attention_mask)\n        np.save('train-df-token-type-ids', token_type_ids)\n        np.save('train-df-targets', targets)\n        \n        \ntrain_input_ids, train_token_type_ids, train_attention_mask, train_targets \\\n                        = encoding_process(\n                                    df=train_df,\n                                    N=train_N,\n                                    num_cores=4,\n                                    tokenizer=bert_tokenizer,\n                                    max_length=MAX_LENGTH)\n\nprint(\"Training data encoding complete.\")\n\n\nvalid_input_ids, valid_token_type_ids, valid_attention_mask, valid_targets \\\n                        = encoding_process(\n                                    df=valid_df,\n                                    N=valid_N,\n                                    num_cores=4,\n                                    tokenizer=bert_tokenizer,\n                                    max_length=MAX_LENGTH)\n\nprint(\"Validation data encoding complete.\")\n\n\ntest_input_ids, test_token_type_ids, test_attention_mask \\\n                        = test_encoding_process(\n                                    df=test_df,\n                                    N=test_N,\n                                    num_cores=4,\n                                    tokenizer=bert_tokenizer,\n                                    max_length=MAX_LENGTH)\n\nprint(\"Test data encoding complete.\")\n\nsave_data(compressed=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = np.load('train-df-compressed-input-ids.npz', mmap_mode='r')\nx_train.f.arr_0.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from multiprocessing import Pool\n\nnum_cores = 4\n\ndef fx(t):\n    t1, t2 = t\n    for i in tqdm(range(t1, t2)):\n        z = i * i\n        #print(f\"{z}\")\n\n\ndef process():\n        \n    n1 = (0, int(N/4))\n    n2 = (int(N/4), int(N/2))\n    n3 = (int(N/2), int(0.75 * N))\n    n4 = (int(0.75 * N), N)\n    \n    n = [n1, n2, n3, n4]\n    \n    pool = Pool(num_cores)\n    result = pool.map(fx, n)\n    return result\n    \nps = process()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_batch_encoding(texts, tokenizer, batch_size, max_len=192):\n    input_ids = []\n    for i in tqdm(range(0, len(texts), batch_size)):\n        text = texts[i: i + batch_size].tolist()\n        inputs = tokenizer.batch_encode_plus(text, \n                                             pad_to_max_length=True, \n                                             max_length=max_len)\n        input_ids.extend(inputs['input_ids'])\n    return np.array(input_ids)\n\ninput_ids = fast_batch_encoding(train_df.comment_text.values, tokenizer, batch_size=256)\ninput_ids.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.decode(list(input_ids[0]))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}