{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, warnings, pickle, gc, re, string\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# HYPERPARAMETERS\n\nMAX_LEN = 220\nMAX_FEATURES = 100000\nEMBED_SIZE = 600\n\nBATCH_SIZE = 128\nN_EPOCHS = 5\n\nLEARNING_RATE = 8e-4\n\n# We will concatenate Crawl and GloVe embeddings\n\nCRAWL_EMB_PATH = '../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'\nGLOVE_EMB_PATH = '../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Loading train sets...')\ntrain1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n\ntrain = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])\n\ndel train1, train2\n\nprint('Loading validation sets...')\nvalid = pd.read_csv('/kaggle/input/val-en-df/validation_en.csv')\n\nprint('Loading test sets...')\ntest = pd.read_csv('/kaggle/input/test-en-df/test_en.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"misspell_dict = {\"aren't\": \"are not\", \"can't\": \"cannot\", \"couldn't\": \"could not\",\n                 \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\",\n                 \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n                 \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\",\n                 \"i'd\": \"I had\", \"i'll\": \"I will\", \"i'm\": \"I am\", \"isn't\": \"is not\",\n                 \"it's\": \"it is\", \"it'll\": \"it will\", \"i've\": \"I have\", \"let's\": \"let us\",\n                 \"mightn't\": \"might not\", \"mustn't\": \"must not\", \"shan't\": \"shall not\",\n                 \"she'd\": \"she would\", \"she'll\": \"she will\", \"she's\": \"she is\",\n                 \"shouldn't\": \"should not\", \"that's\": \"that is\", \"there's\": \"there is\",\n                 \"they'd\": \"they would\", \"they'll\": \"they will\", \"they're\": \"they are\",\n                 \"they've\": \"they have\", \"we'd\": \"we would\", \"we're\": \"we are\",\n                 \"weren't\": \"were not\", \"we've\": \"we have\", \"what'll\": \"what will\",\n                 \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\",\n                 \"where's\": \"where is\", \"who'd\": \"who would\", \"who'll\": \"who will\",\n                 \"who're\": \"who are\", \"who's\": \"who is\", \"who've\": \"who have\",\n                 \"won't\": \"will not\", \"wouldn't\": \"would not\", \"you'd\": \"you would\",\n                 \"you'll\": \"you will\", \"you're\": \"you are\", \"you've\": \"you have\",\n                 \"'re\": \" are\", \"wasn't\": \"was not\", \"we'll\": \" will\", \"tryin'\": \"trying\"}\n\ndef _get_misspell(misspell_dict):\n    misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))\n    return misspell_dict, misspell_re\n\ndef replace_typical_misspell(text):\n    misspellings, misspellings_re = _get_misspell(misspell_dict)\n\n    def replace(match):\n        return misspellings[match.group(0)]\n\n    return misspellings_re.sub(replace, text)\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']',\n          '>', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£', '·', '_', '{', '}', '©', '^',\n          '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',\n          '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',\n          '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',\n          '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',\n          'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',\n          '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts + list(string.punctuation):\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    return re.sub(r'\\d+', ' ', x)\n\ndef preprocess(train, valid, test, tfms):\n    for tfm in tfms:\n        print(tfm.__name__)\n        train['comment_text'] = train['comment_text'].progress_apply(tfm)\n        valid['comment_text_en'] = valid['comment_text_en'].progress_apply(tfm)\n        test['content'] = test['content'].progress_apply(tfm)\n    \n    return train, valid, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = [replace_typical_misspell, clean_text, clean_numbers]\ntrain, valid, test = preprocess(train, valid, test, tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comments = list(train['comment_text']) + list(valid['comment_text_en']) + list(test['content_en'])\n\nprint(comments[0])\nprint(len(comments))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install laserembeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!python -m laserembeddings download-models","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from laserembeddings import Laser\n\nlaser = Laser()\n\nembedding_matrix = laser.embed_sentences(comments, lang='en')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(\"jigsaw-ml_laser.npy\", embedding_matrix)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}