{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n#!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n#!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev --version nightly\n#20200420\n#!pip install transformers==2.5.1\n!pip install transformers==2.9.0\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pandarallel > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler\n\nimport time\nimport random\nfrom datetime import datetime\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport re\n\n# !pip install nltk > /dev/null\nimport nltk\nnltk.download('punkt')\n\nfrom nltk import sent_tokenize\n\nfrom pandarallel import pandarallel\n\npandarallel.initialize(nb_workers=2, progress_bar=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\nimport re\n\nimport logging\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\nimport sys\nfrom sklearn import metrics, model_selection\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n#SEED=528491\nSEED=42\nMAX_LENGTH=224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=True, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'])\n\nLANGS = {\n    'en': 'english',\n    'it': 'italian', \n    'fr': 'french', \n    'es': 'spanish',\n    'tr': 'turkish', \n    'ru': 'russian',\n    'pt': 'portuguese'\n}\n\ndef get_sentences(text, lang='en'):\n    return sent_tokenize(text, LANGS.get(lang, 'english'))\n\ndef exclude_duplicate_sentences(text, lang='en'):\n    sentences = []\n    for sentence in get_sentences(text, lang):\n        sentence = sentence.strip()\n        if sentence not in sentences:\n            sentences.append(sentence)\n        #else :\n        #    print(\"-------------------------------------------\")\n        #    print(sentence)\n        #    print(\"-------------------------------------------\")\n    return ' '.join(sentences)\n\ndef clean_text(text, lang='en'):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text)\n    text = re.sub(r'#[\\S]+\\b', '', text)\n    text = re.sub(r'@[\\S]+\\b', '', text)\n    text = re.sub(r'https?\\S+', '', text)\n    text = re.sub(r'\\s+', ' ', text)\n    text = exclude_duplicate_sentences(text, lang)\n    return text.strip()\n\ndef clean_text(text, lang='en'):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text)\n    text = re.sub(r'#[\\S]+\\b', '', text)\n    text = re.sub(r'@[\\S]+\\b', '', text)\n    text = re.sub(r'https?\\S+', '', text)\n    text = re.sub(r'\\s+', ' ', text)\n    #text = exclude_duplicate_sentences(text, lang)\n    return text.strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"langs=['tr','it','es','ru','fr','pt']\n#langs=['ru','fr','pt']\n#langs=['tr','it','es']\nD_TRANS='/kaggle/input/jigsaw-train-multilingual-coments-google-api/'\n\nfor i in range(len(langs)):\n    print(langs[i])\n    fn = D_TRANS+'jigsaw-toxic-comment-train-google-%s-cleaned.csv'%langs[i]\n    \n    df_train = pd.read_csv(fn, usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\n    n_samples_1 = df_train[df_train['toxic'] == 1].shape[0]\n    \n    # Combine train1 with a subset of train2\n    df_train = pd.concat([\n        df_train[['comment_text', 'toxic']].query('toxic==1'),\n        df_train[['comment_text', 'toxic']].query('toxic==0').sample(n=n_samples_1, random_state=SEED)\n        ])\n\n    df_train = df_train.sample(frac=1).reset_index(drop=True)\n    print(df_train.shape)\n    print(\"-------------------------\")\n    print(\"n_samples_1 : {}\".format(n_samples_1))\n    \n    df_train[\"comment_text\"] = df_train.parallel_apply(lambda x: clean_text(x['comment_text']), axis=1)\n    x_train_id, x_train_mask = regular_encode(df_train.comment_text.values, tokenizer, maxlen=MAX_LENGTH)\n    y_train                  = df_train.toxic.values\n    \n    np.save('x_train_id_%s'%langs[i], x_train_id)\n    np.save('x_train_mask_%s'%langs[i], x_train_mask)\n    np.save('y_train_%s'%langs[i], y_train)\n    \n    del(df_train)\n    del(x_train_id)\n    del(x_train_mask)\n    del(y_train)\n    gc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}