{"cells":[{"metadata":{"_uuid":"710ed17d0c57bd287be0ee3b2782a53a54510561"},"cell_type":"markdown","source":"## IMPORTS "},{"metadata":{"_uuid":"585eec8b444e194eda5a141d2a2316547235ffdf"},"cell_type":"markdown","source":"Solving a problem with pca"},{"metadata":{"_uuid":"abb7e3c30b8a412a50c6b451c49939e3cf4bc11b","scrolled":true,"trusted":true},"cell_type":"code","source":"import random\nimport copy\nimport time\nimport pandas as pd\nimport numpy as np\nimport gc\nimport re\nimport torch\nfrom torchtext import data\n#import spacy\nfrom tqdm import tqdm_notebook, tnrange\nfrom tqdm.auto import tqdm\n\ntqdm.pandas(desc='Progress')\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk import word_tokenize\n\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nfrom torchtext.data import Example\nfrom sklearn.metrics import f1_score\nimport torchtext\nimport os \n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# cross validation and metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom torch.optim.optimizer import Optimizer\nfrom unidecode import unidecode\n\nfrom sklearn.preprocessing import StandardScaler\nfrom textblob import TextBlob\nfrom multiprocessing import  Pool\nfrom functools import partial\nimport numpy as np\nfrom sklearn.decomposition import PCA\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a4ff5590a6f152dc1bec5aeca79aef10218f7de"},"cell_type":"markdown","source":"### Basic Parameters"},{"metadata":{"_uuid":"deee49df5ca1c4413f71677939e26aa1ff784e44","scrolled":true,"trusted":true},"cell_type":"code","source":"concat =0\nif concat:\n    embed_size = 604 # how big is each word vector\nelse:\n    embed_size = 304 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\nbatch_size = 512 # how many samples to process at once\nn_epochs = 5# how many times to iterate over all samples\nn_splits = 5 # Number of K-fold Splits\n\nSEED = 1029\n\ndebug = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b53b0ebd37575ab31361099f0538cbb0457db5b6","trusted":true},"cell_type":"code","source":"loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"654cbe3c8a1f2a618a2441afe00df3b4a89e0a58"},"cell_type":"markdown","source":"### Ensure determinism in the results\n\nA common headache in this competition is the lack of determinism in the results due to cudnn. The following Kernel has a solution in Pytorch.\n\nSee https://www.kaggle.com/hengzheng/pytorch-starter. "},{"metadata":{"_uuid":"58bbf87335799247586aaed16531f4d28d10ed4a","scrolled":true,"trusted":true},"cell_type":"code","source":"def seed_everything(seed=10):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c890692644acce2dc4f6e2f929d6d294faca4ad2"},"cell_type":"markdown","source":"### Code for Loading Embeddings\n\nFunctions taken from the kernel:https://www.kaggle.com/gmhost/gru-capsule\n"},{"metadata":{"_uuid":"7026ee1d913f54dd4b560f654efdb9f833581cd3","scrolled":true,"trusted":true},"cell_type":"code","source":"## FUNCTIONS TAKEN FROM https://www.kaggle.com/gmhost/gru-capsule\n\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        #ALLmight\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n        else:\n            embedding_vector = embeddings_index.get(word.capitalize())\n            if embedding_vector is not None: \n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix \n    \n            \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db250776a4bf6d253182506a225a14fb3c636a73"},"cell_type":"code","source":"## FUNCTIONS TAKEN FROM https://www.kaggle.com/gmhost/gru-capsule\n## Load indexes only\n\ndef load_glove_index():\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    return embeddings_index\n    \n            \ndef load_fasttext_index():    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    return embeddings_index\n\ndef load_para_index():\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    return embeddings_index\n\ndef create_glove(word_index,embeddings_index):\n    emb_mean,emb_std = -0.005838499,0.48782197\n    all_embs = np.stack(embeddings_index.values())\n    embed_size = all_embs.shape[1]\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size+4))\n    \n    count_found = nb_words\n    for word, i in tqdm(word_index.items()):\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        extra_embed =np.array([0,0,0,0])\n        if word.istitle():\n            extra_embed[0]=1\n        elif word.isupper():\n            extra_embed[1]=1\n        word_sent = TextBlob(word).sentiment\n        extra_embed[2:] = [word_sent.polarity,word_sent.subjectivity]\n        #ALLmight\n        if embedding_vector is not None: \n            embedding_matrix[i] =  np.append(embedding_vector,extra_embed)\n        else:\n            if word.istitle():\n                embedding_vector = embeddings_index.get(word.lower())\n                if embedding_vector is not None: \n                    embedding_matrix[i] = np.append(embedding_vector,extra_embed)\n                else:\n                    embedding_matrix[i,300:] = extra_embed\n                    count_found-=1\n            elif word.islower():\n                embedding_vector = embeddings_index.get(word.capitalize())\n                if embedding_vector is not None: \n                    embedding_matrix[i] = np.append(embedding_vector,extra_embed)\n                else:\n                    embedding_matrix[i,300:] = extra_embed\n                    count_found-=1\n            else:\n                embedding_matrix[i,300:] = extra_embed\n                count_found-=1\n    print(\"Got embedding for \",count_found,\" words.\")\n    return embedding_matrix\n\ndef create_para(word_index,embeddings_index):\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    all_embs = np.stack(embeddings_index.values())\n    embed_size = all_embs.shape[1]\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size+4))\n    count_found = nb_words\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word.lower())\n        extra_embed =np.array([0,0,0,0])\n        if word.istitle():\n            extra_embed[0]=1\n        elif word.isupper():\n            extra_embed[1]=1\n        word_sent = TextBlob(word).sentiment\n        extra_embed[2:] = [word_sent.polarity,word_sent.subjectivity]\n        #ALLmight\n        if embedding_vector is not None: \n            embedding_matrix[i] = np.append(embedding_vector,extra_embed)\n        else:\n            embedding_matrix[i,300:] = extra_embed\n            count_found -=1\n    print(\"Got embedding for \",count_found,\" words.\")\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea10c8e218a1280faa9802bcb7f1117c89ec96f9"},"cell_type":"markdown","source":"## LOAD PROCESSED TRAINING DATA FROM DISK"},{"metadata":{"_uuid":"07e9890ec0b490cef57565f7dff953aa56ebd3dc"},"cell_type":"markdown","source":"## Normalization\n\nBorrowed from:\n* How to: Preprocessing when using embeddings\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\n* Improve your Score with some Text Preprocessing https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing"},{"metadata":{"_uuid":"28ebb28ba78972bb8d4fee9b53437045542d20fb","scrolled":true,"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known\n\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text\n\ndef correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x\n\ndef unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text\n\n''' remeber paragram is already lowercased. For Glove we have different meanings \nfor upper and lower words. We won't be using Lowercase for glove embeddings unless \nabsolutely essential'''\n\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:\n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")    \n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abeab4c80d6829cf2eae706bfa7929e2871af81f","scrolled":true,"trusted":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';',  '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\n\ndef clean_numbers(x):\n    if bool(re.search(r'\\d', x)):\n        x = re.sub('[0-9]{5,}', '#####', x)\n        x = re.sub('[0-9]{4}', '####', x)\n        x = re.sub('[0-9]{3}', '###', x)\n        x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\ndef mid_padder(sequence,maxlen):\n    for i, data in enumerate(sequence):\n        if len(data)>=maxlen:\n            sequence[i] = data[:maxlen]\n        elif len(data)<maxlen:\n            mid_point = int(len(data)/2)\n            num_zeros = maxlen - len(data)\n            n_seq = data[:mid_point] + [0]*num_zeros + data[mid_point:]\n            sequence[i] = n_seq\n    return np.array(sequence,dtype='int32')\n\ndef fraction_padder(sequence,maxlen):\n    for i, data in enumerate(sequence):\n        if len(data)>=maxlen:\n            mid_value = int(maxlen*2/3)\n            sequence[i] = data[:mid_value]+data[-(maxlen-mid_value) :]\n        elif len(data)<maxlen:\n            mid_point = int(len(data)/2)\n            num_zeros = maxlen - len(data)\n            n_seq = data[:mid_point] + [0]*num_zeros + data[mid_point:]\n            sequence[i] = n_seq\n    return np.array(sequence,dtype='int32')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c09d981ae674e6a373189a04dba8d0932b0765b"},"cell_type":"markdown","source":"Extra feature part taken from https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb"},{"metadata":{"trusted":true,"_uuid":"aeff3d9d31a8c7ee9dc76f9c1b9e0fa5546beaac"},"cell_type":"code","source":"token_dict_manageable = {'GPE': ['india', 'us', 'china', 'usa', 'canada', 'america', 'uk', 'pakistan', 'russia', 'germany', 'australia', 'the united states', 'israel', 'delhi', 'japan', 'mumbai', 'instagram', 'u.s.', 'hyderabad', 'north korea', 'singapore', 'california', 'france', 'iran', 'london', 'dubai', 'turkey', 'england', 'quora', 'indonesia', 'mexico', 'philippines', 'malaysia', 'italy', 'syria', 'android', 'britain', 'spain', 'hollywood', 'nigeria', 'vietnam', 'new york', 'texas', 'nepal', 'saudi arabia', 'sweden', 'bangladesh', 'bollywood', 'new zealand', 'brazil', 'florida', 'egypt', 'chennai', 'greece', 'south korea', 'ireland', 'dc', 'korea', 'kerala', 'south africa', 'thailand', 'gujarat', 'jio', 'hong kong', 'poland', 'twitter', 'tamil', 'iraq', 'taiwan', 'afghanistan', 'paris', 'san francisco', 'ai', 'switzerland', 'uae', 'gate', 'chicago', 'toronto', 'u.s', 'ukraine', 'rome', 'netherlands', 'palestine', 'bitsat', 'norway', 'los angeles', 'united states', 'jerusalem', 'du', 'noida', 'bihar', 'karnataka', 'jaipur', 'scotland', 'portugal', 'west bengal', 'sri lanka', 'new york city', 'ussr', 'argentina', 'georgia', 'denmark', 'ahmedabad', 'gmat', 'the soviet union', 'las vegas', 'kenya', 'ib', 'qatar', 'hawaii', 'mangalore', 'austria', 'boston', 'sydney', 'sanskrit', 'finland', 'washington', 'melbourne', 'venezuela', 'barcelona', 'newton', 'seattle', 'galaxy', 'h1b', 'new delhi', 'belgium', 'hadoop', 'myanmar', 'naruto', 'berlin', 'macbook', 'neet', 'sc', 'cambridge', 'ayurveda', 'ohio', 'vr', 'bhutan', 'puerto rico', 'miami', 'tibet', 'u.k.', 'cuba', 'morocco', 'ontario', 'houston', 'tokyo', 'arizona', 'melania', 'atlanta', 'chandigarh', 'vancouver', 'romania', 'illuminati', 'colorado', 'rajasthan', 'michigan', 'new jersey', 'iceland', 'montreal', 'bengal', 'telangana', 'indore', 'the united kingdom', 'ghana', 'the united states of america', 'amsterdam', 'manhattan', 'dallas', 'alaska', 'urdu', 'patna', 'rs', 'hungary', 'la', 'tn', 'comedk', 'oregon', 'viteee', 'mahabharata', 'arjuna', 'ny', 'kashmiri', 'illinois', 'md', 'photoshop', 'trump', 'bhubaneswar', 'serbia', 'tennessee', 'beijing', 'colombia', 'bluetooth', 'great britain', 'physics', 'wikipedia', 'ethereum', 'jammu', 'cet', 'gurgaon', 'lebanon', 'aiims', 'odisha', 'kuwait', 'bulgaria', 'cisco', 'quebec', 'bali', 'shanghai', 'whom', 'haryana', 'massachusetts', 'phoenix', 'chile', 'bangkok', 'canon', 'moscow', 'cm', 'wwi', 'iphones', 'jakarta', 'alabama', 'catalonia', 'san diego', 'redmi', 'm.sc', 'pennsylvania', 'maryland', 'ca', 'istanbul', '°', 'toefl', 'jordan', 'ethiopia', 'modi', 'virginia', 'aiims delhi', 'manchester', 'mongolia', 'nc', 'the roman empire', 'django', 'indiana', 'va', 'honduras', 'mysore', 'alibaba', 'guwahati', 'zimbabwe', 'ubuntu', 'cs', 'philadelphia', 'youtuber', 'oman', 'colonies', 'bangalore', 'cambodia', 'yemen', 'aadhaar', 'libya', 'rings', 'algeria', 'punjab', 'kuala lumpur', 'mecca', 'minnesota', 'north carolina', 'wales', 'manaphy', 'oklahoma', 'saturn', 'kanpur', 'gaza', 'karachi', 'ww2', 'capgemini', 'portland', 'gb', 'seoul', 'narendra modi', 'northern ireland', 'uber', 'karna', \"north korea 's\", 'detroit', 'kota', 'croatia', 'ug', 'dublin', 'orlando', 'iim', 'jamaica', 'macedonia', 'columbus', 'albania', 'mobile', 'ranchi', 'walmart', 'peru', 'wisconsin', 'denver', 'vedas', 'north', 'jon', 'the british empire', 'titanic', 'austin', 'azerbaijan', 'bharat', 'munich', 'ola', 'hermione', 'xbox', 'utah', 'costa rica', 'connecticut', 'empire', 'uttarakhand', 'armenia', 'louisiana', 'jee', 'panama', 'sri lankan', 'kosovo', 'missouri', 'santa', 'athens', 'madrid', 'prague', 'nevada', 'uc', 'kashmiris', 'raipur', 'ashoka', 'visakhapatnam', 'upsc', 'madurai', \"qur'an\", 'san jose', 'united kingdom', 'venice', 'seo', 'vienna', 'indo', 'estonia', 'kindle', 'greenland', 'bhopal', 'somalia', 'romanian', 'greeks', 'delaware', 'kazakhstan', 'vadodara', 'new orleans', 'nyc', 'itunes', 'south carolina', 'north india', 'iiit', 'faridabad', 'san antonio', 'brooklyn', 'kochi', 'berkeley', 'yugoslavia', 'kathmandu', 'prc', 'hiroshima', 'firefox', 'soviet union', 'the ottoman empire', 'tyrion', 'edinburgh', 'mississippi', 'haiti', 'srinagar', 'gk', 'bombay', 'lithuania', 'ww1', 'washington dc', 'welsh', 'legends', 'kannada', 'xi', 'b.a.', 'trivandrum', 'emirates', 'uganda', 'manipur', 'ktm', 'manila', 'maine', 'persia', 'sf', 'cognizant', 'thane', 'kentucky', 'laos', 'liverpool', 'the west bank', 'u.s.a.', 'kcet', 'abu dhabi', 'u.s.a', 'sudan', 'cleveland', 'alberta', 'stockholm', 'uzbekistan', 'arkansas', 'tanzania', 'states', 'hindi', 'congo', 'kansas', 'pittsburgh', 'ghaziabad', 'tunisia', 'ronaldo', 'turkic', 'monaco', 'xrp', 'tolkien', 'bahrain', 'kotlin', 'vizag', 'obama', 'charlotte', 'oxford', 'kurdistan', 'logan', 'scala', 'guam', 'brunei', 'draupadi', 'baltimore', 'charlottesville', 'new mexico', 'british columbia', 'telugu', 'b12', 'f1', 'ba', 'mckinsey', 'luxembourg', 'angular', 'angularjs', 'ecuador', 'modiji', 'ladakh', 'manali', 'karma', 'balochistan', 'idaho', 'lagos', 'zurich', 'holland', 'titan', 'eu', 'calgary', 'imei', 'mbbs', 'brussels', 'bolivia', 'sacramento', 'amrita', 'burma', 'slovenia', 'gay', 'jodhpur', 'gujarati', 'iowa', 'surat', 'milan', 'psychopaths', 'victoria', 'shenzhen', 'ggsipu', 'slavs', 'nagasaki', 'outlook', 'nagaland', 'caucasian', 'budapest', 'cochin', 'voldemort', 'south india', 'nashville', 'quota', 'belarus', 'maldives', 'vlsi', 'allahabad', 'tampa', 'columbia', 'avenger', 'namibia', 'madagascar', 'aristotle', 'hindustan', 'jharkhand', 'barack obama', 'bosnia', 'gtx', 'guatemala', 'millennials', 'rishikesh', 'uruguay', 'nmat', 'perth', 'montana', 'tehran', 'cgl', 'cape town', 'north africa', 'latvia', 'white', 'vatican', 'madara', 'islamabad', 'roman empire', 'facebook', 'coursera', 'zambia', 'mexico city', 'hulu', 'kolkata', 'intp', 'malta', 'hero', 'frankfurt', 'n. korea', 'formula', 'kumar', 'palo alto', 'phuket', 'assassin', 'us', 'lisbon', 'paypal', 'vermont', 'h1', 'oakland', 'tel aviv', 'maths', 'aurangzeb', 'd.c.', 'nmims', 'amish', 'nicaragua', 'the united states', 'warangal', 'b2b', 'chernobyl', 'auckland', 'minneapolis', 'xinjiang', 'ravana', 'zebpay', 'zomato', 'ucl', 'a.i.', 'slovakia', 'etl', 'nacl', 'odyssey', 'papua new guinea', 'pa', 'new hampshire', 'indianapolis', 'aurangabad', 'macau', 'wyoming', 'oslo', 'cairo', 'khalistan', 'cricket', 'm.s.', 'gmail', 'memphis', 'd&d', 'mesra', 'i5', 'uptu', 'mhcet', 'ottawa', 'buffalo', 'dmt', 'duolingo', 'birmingham', 'meerut', 'kashmir', 'cincinnati', 'kathua', 'bannon', 'or', 'st. louis', 'vijayawada', 'florence', 'ubisoft', 'vellore', 'orange county', 'tinder', 'akbar', 'oh', 'mensa', 'narendra', 'geneva', \"5'5\", 'trinidad', 'jacksonville', 'manga', 'nk', \"5'3\", 'icloud', 'bangladeshi', 'golang', 'montenegro', 'paytm', 'taipei', 'silchar', 'hl', 'kansas city', 'honours', 'fedex', 'rwanda', 'wipro', 'alexandria', 'el salvador', 'up', 'verizon', 'west africa', 'orochimaru', 'princeton', 'irma', 'banglore', 'dp', 'labrador', 'baba', 'fiji', 'newark', 'adderall', 'cersei', 'zealand', 'truecaller', 'cloud', 'warsaw', 'bengaluru', 'tds', 'p.s.', 'geography', 'b.s.', 'halo', 'banasthali', 'guangzhou', 'asifa', 'hela', 'angola', 'the republic of ireland', 'op', 'sicily', 'b2c', 'anatolia', 'laravel', 'league', 'johannesburg', 'ssd', 'czech republic', 'nosql', 'cu', 'tum', 'kyrgyzstan', 'turkmenistan', 'vodafone', 'lasik', 'manchester city', 'springfield', 'osaka', 'himachal', 'west virginia', 'snape', 'nebraska', 'meghan', 'doklam', 'mi', 'roc', 'naples', 'newcastle', 'richmond', 'bahamas', 'narayana', 'queensland', 'kingdom', 'dprk', 'navajo', 'east germany', \"st. xavier 's\", 'b.a', 'pirates', 'washington , dc', 'waht', 'kalyan', 'xi', 'agile', 'hercules', 'bronx', 'eminem', 'kanye west', 'joker', 'charleston', 'manafort', 'morgoth', 'chechnya', 'keralites', 'punisher', 'guangdong', 'nairobi', 'salem', 'middleton', 'ipl', 'ca ipcc', 'laplace', 'wakanda', 'whitefield', 'milwaukee', 'goa', 'salisbury', 'westerner', 'hanoi', 'manitoba', 'cyprus', 'rhode island', 'netflix', \"south korea 's\", 'santa monica', 'mathura', 'tmc', 'spjimr', 'tucson', 'queens', 'barbados', 'black', 'sao paulo', 'honolulu', 'y^2', 'llp', 'glasgow', 'jiit', 'maggi', 'dalit', 'dominican republic', 'west germany', 'ascii', 'arabia', 'hannibal', 'kkk', 'eq', 'lahore', 'cologne', 'sunnyvale', 'informatica', 'the czech republic', 'ahmadabad', 'mayweather', 'azeris', 'musk', 'h1b', 'baghdad', 'arijit singh', 'oppo', 'santa clara', 'tasmania', 'the netherlands', 'b.tech', 'komodo', 'czechoslovakia', 'ayodhya', 'east timor', 'meghalaya', 'kerela', 'jalandhar', 'ooty', 'reiki', 'nuremberg', 'qa', 'indochina', 'the byzantine empire', 'jabalpur', 'wicca', 'mphil', 'belfast', 'heisenberg', 'east africa', 'reno', 'uranus', 'nit delhi', 'kiev', 'santa barbara', 'pk', 'm.s', 'ripple', 'youtube', 'slytherin', 'dhaka', 'hokage', 'andriod', 'bpo', 'tacoma', 'affiliate', 'santa cruz', 'ahemdabad', 'h-1b', 'r2', 'pyongyang', 'vegas', 'canberra', 'blockchain', 'georgetown', 'the dominican republic', 'indigo', 'the old testament', 'hindu', 'quantico', 'fms delhi', 'babylon', 'kremlin', 'krishna', 'kylo', 'blue whale', 'winnipeg', 'myrtle beach', 'joffrey', 'chrome', 'warren', 'rio de janeiro', 'ts', 'mozambique', 'belize', 'hogwarts', 'kepler', 'saskatchewan', 'hulk', 'jiren', 'revit', 'belgrade', 'washington state', 'south park', 'aisha', 'snapdragon', 'independence', 'obito', 'isreal', 'ci', 'prozac', 'india', 'iit delhi', \"the people 's republic of china\", 'the netherlands', 'hand', 'l.a.', 'josaa', 'gibraltar', 'tumblr', 'kaaba', 'doha', 'surya', 'gospel', 'shivaji', 'calculus', 'zte', 'washington d.c.', 'maga', 'manual', 'the republic of china', 'bet365', 'gemini', 'macbook', 'aruba', 'vedanta', 'nepali', 'p.s', 'matlab', 'yellowstone', 'netaji', 'ds', 'adelaide', 'aryan', 'senegal', 'finance', 'nsep', 'south delhi', 'ganga', 'tejas', 'kyoto', 'riyadh', 'uchiha', 'albuquerque', 'nanjing', 'amritsar', 'baja', 'ho chi minh', 'mountain view', 'samoa', 'diploma', 'damascus', 'litecoin', 'the roman republic', 'ottoman empire', 'kingston', 'oreo', 'beirut', 'eritrea', 'indra', 'chad', 'baluchistan', 'xl', 'muscat', 'bleach', 'bain', 'rd', 'djibouti', 'olympiad', 'manchuria', 'anushka', 'fermi', 'north korea ’s', 'tancet', 'versailles', 'san juan', 'al', 'arlington', 'analytics', 'salt lake city', 'gate', 'filipina', 'tanakh', 'pok', 'islamophobia', 'the united arab emirates', 'diwali', 'guinea', 'edx', 'natsu', 'josaa', 'beverly hills', 'boulder', 'mmmut', 'manipal', 'new york state', 'the united states of america', 'sun', 'friends', 'fort', 'vt', 'new zealander', 'swamy', 'edmonton', 'putin', 'dharma', 'omaha', 'duryodhana', 'nice', 'bc', 'jiraiya', 'reading', 'koramangala', 'hc', 'm.sc', 'anchorage', \"new zealand 's\", 'ambani', 'b.c.', 'd3', 'são paulo', 'dos', 'osmania', 'kd', \"5'7\", '%', 'gorkhaland', 'pasadena', 'hq', 'durham', 'galactic empire', 'malibu', 'agra', 'bolsheviks', 'nifty', 'ftre', 'westminster', 'swami vivekananda', 'i.e.', 'capricorn', 'erlang', 'long beach', 'hollister', 'dresden', 'louisville', 'ips', 'haskell', 'chromium', 'west bank', 'wonderland', \"saudi arabia 's\", 'chaitanya', 'udaipur', 'munnar', 'bristol', 'shandong', 'albany', 'bavaria', 'videos', 'saigon', 'malayalis', 'gis', 'bhagat singh', 'guyana', 'nj', 'tollywood', 'ey', 'niger', 'programming', 'cg', 'vivo', 'everton', 'panjab', 'menlo park', 'sri chaitanya', 'pharma', 'maersk', 'kabul', 'psycho', 'puritan', 'fury', 'puranas', 'railway', 'stellar', 'be', 'nicobar', 'davos', 'po', 'm.a', 'medina', 'ece', 'odia', 'hyperloop', 'mains', 'messi', 'gi', 'rochester', 'vishnu', 'kohli', 'nitro', 'mount rushmore', 'sioux', 'new south wales', 'gender', 'hurricane irma', 'sa', 'charlemagne', 'ro', 'unnao', 'moldova', \"south africa 's\", 'uyghur', 'pf', 'uan', 'fremont', 'riga', 'la liga', 'uoft', 'mandir', 'asus', 'the russian empire', 'xxxtentacion', 'wonder woman', 'riddler', 'lucknow', 'quebec city', 'afrin', 'hamburg', 'university', 'botswana', 'inc.', 'cornwall', 'ct', 'monero', 'asgard', 'kanji', 'the russian federation', 'undertale', 'hashirama', 'xml', 'halifax', 'cozumel', 'penang', 'antartica', 'hawkeye', 'long island', 'imovie', 'domino', 'cr', 'btech', \"the united states '\", 'sonia', 'adobe', 's. korea', 'ishmael', 'ue', 'narnia', 'mosul', 'kannur', 'natasha', 'concerta', 'bogota', 'invento', 'burlington', 'siliguri', 'st. petersburg', 'sridevi', 'gsoc', 'uwc', 'aiims rishikesh', 'oop', 'iniesta', 'antigua', 'esfp', 'ne', 'bugatti', 'havana', 'king kong', \"san francisco 's\", 'kubernetes', 'guntur', 'marathas', 'blood', 'futurama', 'movie', 'ibps', 'bhilai', \"hong kong 's\", 'sri city', 'bj', 'bastille', 'wellington', 'kakashi', \"new york 's\", 'b.tech', 'i.t', 'srs', 'fort lauderdale', 'sigmund freud', 'unity', 'nit jalandhar', 'brighton', 'pitampura', 'sofia', 'tijuana', 'vatican city', 'dune', 'united states of america', 'netherland', 'nainital', 'stuttgart', 'aiims pg', 'asoiaf', 'chaos', 'spring', 'pattaya', 'des moines', 'south florida', 'jaisalmer', 'united arab emirates', 'sichuan', 'wikitribune', 'j7', 'busan', 'jcpoa', 'oracle', 'kunti', 'corona', 'srilanka', 'terraria', 'new vegas', 'lima', 'providence', 'wellbutrin', 'b.sc', 'sharingan', 'wayanad', 'rp', 'quicksilver', 'kong', 'colorado springs', 'upes', 'fe', 'il', 'iskcon'], 'PERSON': ['quora', 'donald trump', 'trump', 'obama', 'jesus', 'hitler', 'jee mains', 'bitcoin', 'harry potter', 'jee', 'ca', 'ai', 'hillary clinton', 'putin', 'kolkata', 'ww2', 'tamil nadu', 'hillary', 'netflix', 'chennai', 'tamil', 'harry', 'hindi', 'goa', 'mac', 'muhammad', 'modi', 'greeks', 'elon musk', 'paytm', 'rahul gandhi', 'gmail', 'stalin', 'java', 'gordon miller', 'sc', 'karnataka', \"donald trump 's\", 'mm', 'goku', 'jee advanced', 'batman', 'clinton', 'god', 'jesus christ', 'punjabi', 'shiva', 'krishna', 'barack obama', 'allen', 'gandhi', 'marathi', 'spotify', 'adolf hitler', 'moon', 'mark zuckerberg', 'mangalore', 'beatles', 'andhra pradesh', 'narendra modi', 'wikipedia', 'moto', 'darth vader', 'kim jong un', 'shakespeare', 'voldemort', 'will trump', 'bernie sanders', 'steve jobs', 'twitter', 'michael jackson', 'hulk', 'mao', 'bachelor', 'bill gates', 'kota', 'xi jinping', 'justin trudeau', 'aakash', 'cfa', 'john', 'adam', 'arvind kejriwal', 'stephen hawking', 'bill clinton', 'jee mains 2017', 'hunter', 'bpo', 'kim jong - un', 'luke', 'mahatma gandhi', 'erdogan', 'buddha', 'donald', 'ps4', 'android', 'joker', 'mary', 'nehru', 'salman khan', 'mains', 'anakin', 'law', 'vladimir putin', 'f(x', 'jfk', 'dhanbad', 'navi mumbai', 'trumps', 'msc', 'jain', 'kohli', 'kim', 'wonder woman', 'icloud', 'sat', 'hogwarts', 'tesla', 'ruby', 'jee main', 'albert einstein', 'kali linux', 'sikkim', 'jon snow', 'xanax', 'bush', 'brahmins', 'queen elizabeth', 'neet', 'james', 'mike pence', 'hyderabad', 'rick', 'fiitjee', 'oscar', 'rbi', 'bangladesh', 'taylor swift', 'linkedin', 'mohammed', 'robert mueller', 'superman', 'jordan peterson', 'harvey weinstein', 'obamacare', 'einstein', 'karl marx', 'balaji viswanathan', 'will quora', 'neo', 'nri', 'george washington', 'abraham', 'machedo', 'bill', 'paul', 'himachal pradesh', 'vegeta', 'wolverine', 'justin bieber', 'roman', 'comey', 'nixon', 'luke skywalker', 'winston churchill', 'ola', 'michelle obama', 'jio', 'kannadigas', 'coursera', 'kennedy', 'will donald trump', 'william', 'eve', 'iron man', 'david', 'flash', 'pondicherry', 'daenerys', 'alex jones', 'kim jong - un', 'shias', 'jack', 'wi - fi', 'hamilton', 'malayalam', 'charles', 'roy moore', 'lok sabha', 'torah', 'angela merkel', 'balaji vishwanathan', 'jeff bezos', 'ambedkar', 'ben shapiro', 'surat', 'ron', 'tony stark', 'lenin', 'jennifer lawrence', 'dunkirk', 'itunes', 'rajya sabha', 'prophet muhammad', 'loki', 'raspberry pi', 'churchill', 'kik', 'ex', 'qoura', 'darwin', 'myntra', 'sachin tendulkar', 'valentine', 'sherlock holmes', 'virat kohli', 'a+', 'jack', 'abraham lincoln', 'jimmy wales', 'thanos', 'indira gandhi', 'allen kota', 'llc', 'peter', 'mozart', 'zeus', 'michael', 'reagan', 'can quora', 'bjp', 'kate', 'logan paul', 'stephen king', 'seo', 'lamborghini', 'alexander the great', 'bangalore', 'james bond', 'yogi adityanath', 'mussolini', 'netanyahu', 'richard muller', 'bruce wayne', 'percy jackson', 'banglore', 'gujrat', 'kkk', 'i5', 'robert frost', 'godzilla', 'strange', 'itachi', 'george soros', 'james comey', 'markle', 'jee advanced 2017', 'austin', 'jackie chan', 'anakin skywalker', 'jabba', 'beethoven', 'maruti suzuki', 'drake', 'abdul kalam', 'michael jordan', 'william shakespeare', 'punjabis', 'shepherd', 'warren buffett', 'jee', 'jared kushner', 'george', 'asus', 'thomas jefferson', 'mauritius', 'saitama', 'gotham', 'h4', 'george w. bush', 'clintons', 'messi', 'john lennon', 'arduino', 'bermuda', 'bruce lee', 'lily', 'ronald reagan', 'jawaharlal nehru', 'paul mccartney', 'flynn', 'tom', 'tripura', 'sam', 'ali', 'shopify', 'overwatch', 'donald j. trump', 'google adsense', 'macbeth', 'black', 'steve bannon', 'john mccain', 'rama', 'quoran', 'elizabeth', \"adam d'angelo\", 'capgemini', 'lewis', 'fdr', 'selenium', 'kakashi', 'haryana', 'black panther', 'jeff sessions', 'noah', 'jedi', 'joseph stalin', 'gandhiji', 'robert', 'britishers', \"gopal kavalireddi 's\", 'manmohan singh', 'trudeau', 'dalit', 'palpatine', 'brown', 'lebron james', 'jeremy corbyn', 'casio', 'aspergers', 'ferrari', 'chhattisgarh', 'peter parker', 'rey', 'cmat', 'cgpa', \"hillary clinton 's\", 'srm', 'sun', 'steve', 'milky', 'paul ryan', 'breaking bad', 'cristiano ronaldo', 'hawking', 'bloomberg', 'rothschild', 'saddam hussein', 'zoho', 'genghis khan', 'joseph smith', 'ali alshamsi', 'joseph', 'macron', 'sonia gandhi', 'xiaomi', 'khan', 'amitabh bachchan', 'arnab goswami', 'snoke', 'neymar', 'richard dawkins', 'athena walker', 'nancy pelosi', 'aquarius', 'clash', 'peter thiel', 'queen elizabeth ii', 'salman', 'taylor', 'zuckerberg', 'amit shah', 'obi - wan', 'germanic', 'diploma', 'ios', 'chris langan', 'mike tyson', 'barry allen', 'ravi', 'darth sidious', 'shahrukh khan', 'constantinople', '√', 'should quora', 'asperger', 'mao zedong', 'bill cosby', 'gopro', 'han solo', 'shivaji maharaj', 'spider - man', 'bing', 'nelson mandela', 'whar', 'html', 'libra', 'alexander hamilton', 'dc', 'h2so4', 'qualcomm', 'leh', 'cohen', 'wbjee', 'barry', 'awdhesh singh', 'jains', 'clash royale', 'isaac newton', 'youtubers', 'michael cohen', 'lee', 'max', 'diana', 'oprah', 'nibiru', 'whites', 'jee mains', 'shabbat', 'prince william', 'barack hussein obama', 'emmanuel macron', 'bran', 'holi', 'bach', 'darth maul', 'taj mahal', 'tony', 'edgar allan poe', 'hanuman', 'frankenstein', 'elvis presley', 'data', 'achilles', 'james potter', 'tom cruise', 'rick riordan', 'charles darwin', 'sachin', 'roger federer', 'darkseid', 'calcutta', 'johnson', 'habib fanny', 'barca', 'udaipur', 'ww1', 'chopin', 'elizabeth warren', 'skrill', 'eminem', 'franklin veaux', 'deepika padukone', 'george lucas', 'zakir naik', 'uranus', 'theresa may', 'jaime', 'ludhiana', 'ernest w. adams', 'edward snowden', 'elon', 'david hogg', 'robin', 'will america', 'sean kernan', 'oracle', 'kafka', 'wifi', 'kali', 'ali alshamsi', 'andaman', 'himalayas', 'arthur', 'blm', 'tywin', 'windows xp', 'leonardo da vinci', 'adsense', 'jake paul', 'battle', 'mark', 'neil armstrong', 'latino', 'jinnah', 'b.com', 'katy perry', 'john cena', 'matthew bates', 'martin luther king', 'mitch mcconnell', 'math', 'madonna', 'barbie', 'tommy robinson', 'doklam', 'dooku', 'laptop', 'eisenhower', 'jackson', \"elon musk 's\", 'varanasi', 'cancun', \"stephen hawking 's\", 'ghost rider', 'john wick', 'balaji', 'richard nixon', 'travis kalanick', 'test', 'gordon ramsay', 'minecraft', 'bob', 'assad', 'bleach', 'mds', 'ned stark', 'mccain', 'dracula', 'royal', 'jobs', 'ghost', 'ayn rand', 'hilary clinton', 'mukesh ambani', 'roseanne', 'joe', 'metallica', \"michael jackson 's\", 'g(x', 'nodejs', 'redmi 4', 'vader', 'queen victoria', 'selena gomez', 'cognizant', 'marx', 'skyrim', 'ganesh', 'godaddy', \"gordon miller 's\", 'noam chomsky', 'sindhi', 'arsene wenger', 'rohit sharma', 'kant', 'o+', 'floyd mayweather', 'ben', 'mandela', 'leo', 'dobby', 'shivaji', 'jon', 'dumbledore', 'oet', 'normandy', 'princess diana', 'akshay kumar', 'roman saini', 'xi', 'confucius', 'gemini', 'rahu', 'prussia', 'al franken', 'christopher hitchens', 'milo yiannopoulos', 'penn', 'carbon', 'matthew', 'bosch', 'padme', 'christopher nolan', 'ned', 'lebron james', 'santa claus', 'wechat', 'ritalin', 'gurugram', 'simon', 'spock', 'kanye west', \"harry potter 's\", 'cricket', 'boris johnson', 'jimmy carter', 'jk rowling', 'stevens', 'nikki haley', 'mike', 'caesar', 'prince harry', 'pablo escobar', 'thomas', 'ravish kumar', 'ed sheeran', 'tom brady', 'victoria', 'survivor', 'yoda', 'andrew jackson', 'bhagat singh', 'chewbacca', 'chuck norris', 'sasuke', 'smith', 'boba fett', 'il', 'ram', 'george orwell', 'thomas edison', 'perl', 'quara', 'john f. kennedy', 'dave consiglio', 'green lantern', 'chandler', 'meghan markle', 'ragnarok', 'comedk', 'robert baratheon', 'kevin spacey', 'ted cruz', 'isaiah', 'stan lee', 'michelangelo', 'tito', 'moses', 'hegel', 'reliance jio', 'youtuber', 'koch', 'james altucher', 'kellyanne conway', 'jimmy trivedi', 'jabalpur', 'krav maga', 'okcupid', 'margaret thatcher', 'gorakhpur', 'emma watson', 'cameroon', 'jp morgan', 'pence', 'stephenie meyer', 'shillong', 'antichrist', 'kim kardashian', 'turk', 'kochi', 'tony blair', 'byju', 'fidel castro', 'che guevara', 'osama bin laden', 'sam harris', 'neanderthals', 'tolkien', 'gaddafi', 'vinay kumaran', 'mohammad', 'rajiv gandhi', 'fibonacci', 'charlie', 'snap', 'hobbes', 'kodaikanal', 'ed', 'franco', 'asp.net', 'will bitcoin', 'david bowie', \"taylor swift 's\", 'ahmedabad', 'yuvraj singh', 'maxine waters', 'jake williams', 'al gore', 'avatar', 'siri', 'george bush', 'deadpool', 'katrina kaif', 'roosevelt', 'jon davis', 'kashmiris', 'raghuram rajan', 'adam smith', 'elvis', 'bob dylan', 'hilary', 'planck', 'rvce', 'ross', 'ripple', 'romance', 'ctmu', 'warren buffet', 'henry viii', 'ashes', 'aamir khan', 'christopher columbus', 'akatsuki', 'dante', 'euros', 'corbyn', 'daenerys targaryen', 'dennis rodman', 'ise', 'mockingbird', 'ayurveda', 'adam and eve', 'ambien', 'steve smith', 'telangana', 'islamophobic', 'marvel', 'radha', 'matter', 'rachel', 'solomon', 'kaplan', 'prince charles', 'sarah huckabee sanders', 'jar jar binks', 'stephen', 'muhammad ali', 'chris', 'android studio', 'malcolm x', 'the nintendo switch', 'frost', 'zara', 'ranbir kapoor', 'sindri', 'will modi', 'robert e. lee', 'riemann', 'jordan', 'x-2', 'li', 'ravana', 'jr.', 'rahul dravid', 'kushner', 'idea', 'kendriya vidyalaya', 'leonardo da vinci', 'bernie', 'snap', 'harley davidson', 'kratos', 'mace windu', 'rihanna', 'quran', 'jerry', 'usain bolt', 'wattpad', 'jason', 'jacob', 'osama bin laden', 'npat', 'sean spicer', 'atlas', 'gujaratis', 'aramaic', 'jack ma', 'goku black', 'jim', 'ranveer singh', 'ravan', 'kurt cobain', 'fox', '2d', 'litecoin', 'sarah sanders', 'theresa', 'google translate', 'kim jung un', 'andrew', 'ed', 'watson', 'doom', 'brad pitt', 'luke cage', 'aishwarya rai', \"justin trudeau 's\", 'kamal hassan', 'spectre', 'rajinikanth', 'celsius', 'yojana', 'jio 4 g', 'yogi', 'ryan', 'pope', 'sheldon', \"will trump 's\", 'andheri', 'mgtow', 'arun jaitley', 'joe biden', 'linkin park', 'julia', 'catherine', 'ajmer', 'angel', 'bean', 'amazon fba', 'mario', 'jimi hendrix', 'bhagavad gita', 'mali', 'anthony bourdain', 'alia bhatt', 'christopher', 'martin luther king jr.', 'pega', 'will obama', 'roy', 'canon', 'logan', 'marilyn monroe', 'vicodin', 'xamarin', 'hajj', 'bill maher', 'cloud', 'jamshedpur', 'judas', 'beerus', 'francis', 'sennheiser', 'frieza', 'ramaiah', 'judaic', 'sirius', 'coimbatore', 'stephen curry', 'lex luthor', 'hashimoto', 'jee 2019', 'williams', 'edge', 'b.des', 'b2b', 'martin luther', 'sessions', 'truman', 'immanuel kant', 'merlin', 'nepalis', 'priyanka chopra', 'hubli', 'rex tillerson', 'kevin', 'kendrick lamar', 'srm chennai', 'abcd', 'pascal', 'trumpcare', 'dettol', \"bill o'reilly\", 'samsung', 'albus dumbledore', 'halo', 'lois lowry', 'sardar patel', 'dprk', 'dan', 'lucy', 'phil', 'ben affleck', 'kanhaiya kumar', 'irodov', 'rinnegan', 'ash ketchum', 'wolf', 'chandler bing', \"balaji vishwanathan 's\", 'dravid', 'lowe', 'alien', 'obi wan', 'hafiz saeed', 'prophet', 'diesel', 'mark cuban', 'kejriwal', 'jungkook', 'henry', 'macbook pro', 'jessica jones', 'jason bourne', 'levi', 'stds', 'jim crow', 'thor ragnarok', 'james patterson', 'talmud', \"barack obama 's\", 'kyrie irving', 'xbox', 'zerodha', 'richard feynman', 'hp', 'stephen colbert', 'malawi', 'windows', 'sita', 'hindustani', 'ajit pai', 'upwork', 'queen', 'gds', 'bahasa', 'catalan', \"rahul gandhi 's\", 'captain america', 'gaara', 'zidane', 'robert mugabe', 'obi - wan kenobi', 'brain', 'johnny depp', 'xp', 'sandy hook', 'graham', 'carter', 'saddam', 'kirk', 'cleopatra', 'suzuki gixxer', 'ben 10', 'akbar', 'luftwaffe', 'heroes', 'chelsea', 'lady gaga', 'sheldon cooper', 'michelle', \"stephen king 's\", 'nazareth', 'jaipur', 'clark', 'chelsea clinton', 'ambani', 'narendra', 'oprah winfrey', 'kundalini', 'tony robbins', 'rice', 'ketu', 'john cate', 'shark tank', 'olympus', 'hassan', 'gauri lankesh', 'wittgenstein', 'ann coulter', 'bitcoin cash', \"jon snow 's\", 'jenkins', 'magnus carlsen', 'maratha', 'yahweh', 'flies', 'tim cook', 'marcus aurelius', 'apartheid', 'hokkien', 'grey', 'tom petty', 'picasso', 'word', 'vitamin d', 'christ', 'danzo', 'mvc', 'colt', '\\\\int', 'alice', 'stormy daniels', 'louis vuitton', 'robb stark', 'monica', 'sandeep maheshwari', 'elke weiss', 'tom riddle', 'anonymous', 'iphone x', 'sanjay gandhi', 'buenos aires', 'al', 'tywin lannister', 'ralph waldo emerson', 'billy', 'paul manafort', 'rosa parks', 'bane', 'dick', 'amazon alexa', 'roblox', 'meryl streep', 'will indians', 'redmi', 'hermione granger', \"balaji viswanathan 's\", 'doug jones', 'weinstein', 'trotsky', 'himalayan', 'the lok sabha', 'brahman', 'main', 'helen keller', 'sp jain', 'hades', 'roth ira', 'homo sapiens', 'spurs', 'kevin durant', 'higgs', 'wright', 'common', 'ximb', 'led zeppelin', 'rupee', 'joffrey', 'john adams', 'grindelwald', 'x^2', 'maya angelou', 'jamie', 'manit bhopal', 'prophet mohammed', 'anne frank', 'dima vorobiev', 'brock lesnar', 'studio', 'leonardo dicaprio', 'frank', 'theodore roosevelt', 'ramsay', 'cinderella', 'thrissur', 'allen dlp', 'nashik', 'chechens', 'michael flynn', 'will russia', 'neo nazis', 'dwayne johnson', 'ram mandir', 'clans', 'kalpit veerwal', 'linux mint', 'juggernaut', 'btech', 'hydra', 'ellen', 'shazam', 'gorbachev', 'docker', 'ra', 'sarah', 'laravel', 'jones', 'calvin', 'rutherford', 'swiggy', 'leon trotsky', 'daredevil', 'martin', 'will google', 'roseanne barr', 'john oliver', 'pain', 'pol pot', 'thrawn', 'princeton', 'wimbledon', 'montessori', 'hc verma', 'chester bennington', 'lol', 'ramayana', 'norse', 'mohali', 'mach', 'jquery', 'hrithik roshan', 'b1/b2', 'melania trump', 'belkin', 'baba ramdev', 'jiren', 'donald trumps', 'blender', 'descartes', 'steele', 'mahathir', 'majin buu', 'judy', 'benjamin franklin', 'richard', 'ron weasley', 'hyperloop', 'lakers', 'oliver cromwell', 'html5', 'rajdhani', 'aquaman', 'jose mourinho', 'jaypee noida', 'stephen paddock', 'adidas', 'lightroom', 'bruce', 'martin shkreli', 'kylo ren', 'brahmin', 'x', 'wiki', 'asim qureshi', 'isaac', 'rush limbaugh', 'harry styles', 'tableau', 'crohn', \"jared kushner 's\", 'h4 ead', 'patna', 'osama', 'james damore', 'lil wayne', 'indra', 'young', 'rick sanchez', 'berserk', 'easy', 'henry ford', 'donald trump ’s', 'raven', 'nintendo switch', 'manu', 'qin', 'arya', 'sc', 'google adsense', 'shel silverstein', 'benjamin netanyahu', \"bill clinton 's\", 'would quora', 'which quora', 'guru granth sahib', 'padmaavat', 'will muslims', 'luna', 'baloch', 'bajaj', 'patel', 'arthur miller', 'kim jong', 'linux', 'prophet mohammad', 'a^2', 'punjab', \"george orwell 's\", 'aegon', 'vipassana', 'payoneer', 'amir khan', 'tulu', 'giza', 'matt damon', 'subramanian swamy', 'bill nye', 'dragon ball', 'lannisters', 'diamond', 'parks', 'android app', 'will apple', 'chetan bhagat', 'muller', 'romani', 'euro', 'kashmiri', 'poseidon', 'se', 'obamas', 'linus torvalds', 'rudy giuliani', 'ariana grande', 'guru', 'rajnikanth', \"peter thiel 's\"], 'ORG': ['quora', 'facebook', 'google', 'youtube', 'iit', 'islam', 'amazon', 'jee', 'cbse', 'bangalore', 'apple', 'upsc', 'trump', 'iphone', 'whatsapp', 'hindi', 'android', 'ias', 'gst', 'eu', 'java', 'cs', \"trump 's\", 'microsoft', 'mit', 'gpa', 'mbbs', 'ca', 'wordpress', 'cat', 'obc', 'linux', 'sbi', 'ac', 'nit', 'ssc', 'gre', 'bba', 'congress', 'ece', 'seo', 'vit', 'harvard', 'sat', 'samsung', 'nasa', 'fbi', 'maharashtra', 'ssc cgl', 'ba', 'tcs', 'uber', 'ielts', 'nda', 'nba', 'christianity', 'aiims', 'pcm', 'iim', 'neet', 'sap', 'linkedin', 'un', 'stanford', 'nfl', 'pdf', 'ncert', 'ap', 'isro', 'infosys', 'computer science', 'disney', 'ipl', 'aws', 'bca', 'usb', 'psu', 'bpd', 'paypal', 'ebay', 'kolkata', 'mcu', 'pokemon', 'flipkart', 'judaism', 'rss', 'cia', 'mca', 'clat', 'bhu', 'ibm', 'usd', 'isc', 'b.tech', 'bds', 'cnn', 'brexit', 'honda', 'srm', 'game of thrones', 'gop', 'sony', 'ocd', 'nato', 'ignou', 'bts', 'ww2', 'css', 'lgbt', 'wwe', 'rbi', 'm.tech', 'javascript', 'mnc', 'npd', 'vit vellore', 'nra', 'airtel', 'thanos', 'intel', 'kiit', 'fifa', 'ipad', 'gate', 'cv', 'karnataka', 'pcb', 'iiser', 'bengaluru', 'dtu', 'irs', 'ifs', 'llb', 'navy', 'uttar pradesh', 'kannada', 'yahoo', 'nift', 'skype', 'bits', 'ipcc', 'atm', 'up', 'bits pilani', 'iit bombay', 'starbucks', 'ux', 'minecraft', 'co2', 'ies', 'naruto', 'kvpy', 'ico', 'ese', 'eee', 'aap', 'coimbatore', 'ips', 'bmw', 'assam', 'toyota', 'acca', 'samsung galaxy', 'tx', 'house', 'stem', 'btech', 'vishnu', 'sql', 'nike', 'bs', 'nios', 'ml', 'accenture', 'army', 'ongc', 'airbnb', 'cpt', 'trump ’s', 'vc', 'hsc', 'intj', 'the white house', 'os', 'infj', 'yale', 'dell', 'fox news', 'air', 'university', 'digital', 'deloitte', 'senate', 'physics', 'mtech', 'inr', 'isi', 'ford', 'lsd', 'boeing', 'ipu', 'delhi university', 'obama', 'machine learning', 'net', 'po', 'ec', 'state', 'vpn', 'sip', 'gpu', '&', 'sas', 'sun', 'visa', 'iiit', 'columbia', 'fiverr', 'the democratic party', 'bbc', 'ucla', 'hdfc', 'srm university', 'abc', 'mueller', 'cpec', 'iot', 'real madrid', 'bms', 'alexa', 'intelligence', 'universe', 'ccna', 'the european union', 'b.tech', 'reddit', 'mlb', 'comedk', 'cfa', 'mercury', 'the republican party', 'malayalam', 'mcdonald', 'bams', 'wifi', 'dehradun', 'ww1', 'roman', 'lic', 'bc', 'dnc', 'kcet', 'data science', 'uc berkeley', 'amity', 'cbi', 'infp', 'hcl', 'maths', 'tata', 'du', 'science', 'nlp', 'renaissance', 'napoleon', 'afcat', 'diy', 'einstein', 'drdo', 'the supreme court', 'ctc', 'punjab', 'sauron', 'bsc', 'emi', 'nintendo', 'ssd', 'indonesians', 'goldman sachs', 'ar', 'mtech', 'nokia', 'ict', 'ncc', 'supreme court', 'madhya pradesh', 'bnbr', 'lenovo', 'icici', 'rc', 'iit delhi', 'apollo', 'the united nations', 'assad', 'wipro', 'usps', 'bitsat', 'ga', 'nit trichy', 'instagram', 'hamas', 'std', 'act', 'cad', 'ipod', 'bsc', 'lgbtq', 'united', 'cctv', 'mercedes', 'asd', 'caltech', 'ct', 'bangladeshi', 'rna', 'the usa', 'lincoln', 'thapar university', 'pa', 'nse', 'intp', 'the catholic church', 'iphone 7', 'nus', 'ramayana', 'taliban', 'mughal', 'udemy', 'fox', 'ss', 'messi', 'iit madras', 'manchester united', 'finance', 'quran', 'csir', 'arsenal', 'isp', 'costco', 'google maps', 'powerpoint', 'time', 'fms', 'nlu', 'nhl', 'iocl', 'sasuke', 'indian army', 'iit kharagpur', 'allies', 'xii', 'lpg', 'ufc', 'amd', 'nid', 'odi', 'iit kanpur', 'obamacare', 'justice league', 'cnc', 'ssc cgl 2017', 'evm', 'white house', 'airbus', 'cbd', 'cornell', 'jadavpur university', 'isb', 'eastern', 'mongols', 'mit manipal', 'civil engineering', 'potus', 'js', 'iitians', 'rv', 'nikon', 'google analytics', 'bse', 'vit chennai', 'qr', 'mlm', 'alexander', 'superman', 'christ', 'cms', 'mba', 'ppc', 'ccp', 'ab', 'pokémon', 'sjw', 'schengen', 'iaf', 'b.com', 'rohingya', 'pulsar', 'pcos', 'lpu', 'skyrim', 'rrb', 'aca', 'dj', 'iist', 'education', 'pec', 'acer', 'electrical engineering', 'opt', 'nj', 'iit guwahati', 'vtu', 'btech', 'vedic', 'windows', 'upsc mains', 'world', 'ecommerce', 'nietzsche', 'vjti', 'banglore', 'hyundai', 'cgl', 'ipm', 'the us army', 'wordpress', 'nhs', 'ashkenazi', 'm.tech', 'ic', 'magento', 'motorola', 'latinos', 'delhi ncr', 'ppf', 'si', 'pu', 'doj', 'gui', 'mpc', 'pune', 'the indian army', 'jrf', 'ide', 'mughals', 'mvp', 'medicaid', 'bmi', 'iitjee', 'ibs', 'edm', 'dns', 'nsa', 'mla', 'app', 'medicare', 'mains', 'uceed', 'idf', 'b.tech', 'imo', 'eamcet', 'iift', 'antifa', 'pepsi', 'social security', 'xlri', 'nsit', 'vat', 'cersei', \"mcdonald 's\", 'romeo', 'plc', 'rti', 'omr', 's8', 'mun', 'sa', 'nyu', 'telugu', 'ged', 'juliet', 'iot', 'eth', 'rcc', 'icc', 'nata', 'labour', 'amazon india', 'jee mains', 'verizon', 'niser', 'morty', 'steam', 'manipal university', 'rama', 'cgi', 'vikings', 'psg', 'm.tech', 'rs', 'xat', 'kfc', 'lte', 'porsche', 'fcc', 'gs', 'asean', 'acl', 'noc', 'iphone 5s', 'rmo', 'vajiram', 'coke', 'hbo', 'nz', 'pm modi', 'ncr', 'apple watch', 'ntu', 'julius caesar', 'icai', 'symbiosis', 'e.g', 'social media', 'big data', 'msc', 'sanskrit', 'bit', 'ca ipcc', 'uscis', 'vmc', 'cng', 'mph', 'toshiba', 'hdmi', 'dob', 'panasonic', 'hezbollah', 'gm', 'ge', 'fmcg', 'emp', 'chemistry', 'gmail', 'data', 'az', 'pte', 'christ university', 'uae', 'cpc', 'vodafone', 'bp', 'rpm', 'fda', 'mpsc', 'falcon', 'columbia university', 'cts', 'rpa', 'germanic', 'mathematics', 'devops', 'yamaha', 'dropbox', 'bank', 'aa', 'cmu', 'nissan', 'pla', 'dui', 'aries', 'pi', 'mcdonalds', 'nitk', 'pe', 'ice', 'quora moderation', 'visa', 'md', 'amity university', 'commerce', 'vp', 'iupac', 'usaf', 'kpop', 'fc barcelona', 'trump for', 'ntpc', 'acm', 'indian navy', 'barc', 'telegram', 'nit durgapur', 'delta', 'rpg', 'volvo', 'kpmg', 'gurgaon', 'georgia tech', 'rcb', 'zoloft', 'bengalis', 'parliament', 'cbs', 'cmi', 'sop', 'lahore', 'crs', 'amity noida', 'york', 'unacademy', 'cusat', 'kremlin', 'abap', 'upwork', 'oxford', 'dmv', 'chelsea', 'got', 'enfp', 'audi', 'llc', 'sec', 'tsa', 'gf', 'calcutta university', 'mumbai university', 'iso', 'nbc', 'lan', 'volkswagen', 'thc', 'kyc', 'gsm', 'wattpad', 'uiuc', 'brisbane', 'soundcloud', 'iet', 'padmavati', 'galaxy', 'post', 'pubg', 'urdu', 'media', 'k', 'ib', 'h1-b', 'django', 'force', 'epf', 'sde', 'msrit', 'star trek', 'target', 'church', 'parkland', 'a1', 'crimea', 'shell', 'ita', 'amazon.com', 'copenhagen', 'at&t', 'commonwealth', 'apple music', 'uptu', 'mtv', 'harvard university', 'usmc', 'jmi', 'pnb', 'itunes', 'ur', 'l&t', 'safari', 'azure', 'imu', 'international', 'catia', 'qa', 'amu', 'ui', 'autism', 'rf', 'ultron', 'got', 'hrm', 'whites', 'hackerrank', 'psat', 'json', 'raspberry', 'cvv', 'andhra', 'elitmus', 'kylo ren', 'ntse', 'air force', 'mac', 'upes', 'wwiii', 'united nations', 'xavier', 'sakura', 'woocommerce', 'chevrolet', 'ir', 'oracle', 'sbi po', 'sp', 'sansa', 'otg', 'huawei', 'dps', 'jadavpur', 'cfo', 'imperial', 'shimla', 'rfid', 'xbox one', 'allahabad', 'tableau', 'waterloo', 'xrp', 'ucsd', 'reliance', 'u.k', 'asl', 'anna university', 'obc ncl', 'pandora', 'rajasthan', 'aicte', 'khan academy', 'nvidia', 'duterte', 'computer engineering', 'f-35', 'baroda', 'jeep', 'lse', 'osho', 'oneplus', 'adobe', 'iiest shibpur', 'atc', 'ima', 'nepali', 'hydrogen', 'freud', 'otp', 'html5', 'cern', 'msnbc', 'jamia millia islamia', 'rs', 'bbq', 'wharton', 'cmc vellore', 'fd', 'maxwell', 'the quora world meetup', 'lpa', 'h1b', 'pentagon', 'mamata banerjee', 'dceu', 'morgan stanley', 'fdi', 'united airlines', 'chandigarh university', 'lasik', 'jee mains 2017', 'jaguar', 'tsh', 'aaa', 'ig', 'iim indore', 'csc', 'youtubers', 'the new york times', 'dc', 'upi', 'narendra modi', 'trump administration', 'set', 'pan', 'the air force', 'information technology', 'htc', 'neo', 'b.arch', 'crispr', 'jharkhand', 'guardian', 'sibm', 'wes', 'nmims', 'caucasian', 'itc', 'premier league', 'tech', 'vba', 'als', 'cc', 'csr', 'entp', 'ivf', 'delhi / ncr', 'jyp', 'snapdeal', 'ims', 'air india', 'ipu cet', 'dsp', 'galactus', 'sharepoint', 'us army', 'wto', 'srcc', 'ldr', 'roi', 'mrp', 'amc', 'lexus', 'vit bhopal', 'oc', 'theresa', 'rto', 'siemens', 'marvel comics', 'pms', 'kkr', 'wa', 'iit bhu', 'bmm', 'cdc', 'american', 'msw', 'quantum', 'iima', 'walmart', 'rohingyas', 'cto', 'gate', 'dd', 'macintosh', 'ramayan', 'nazis', 'kgb', 'indian air force', 'ui / ux', 'dlp', 'jquery', 'civil', 'scotus', 'bihari', 'np', 'odin', 'nikola tesla', 'iiser pune', 'pin', 'peta', 'espn', 'ltte', 'nirav modi', 'cgpa', 'paraguay', 'biharis', 'wb', 'aso', 'hk', 'institute', 'micromax', 'aerospace engineering', 'tu delft', 'ssc cgl 2018', 'forbes', 'dasa', 'disney world', 'epa', 'apple tv', 'jsp', 'upenn', 'discord', 'the us navy', 'house of cards', 'rubik', 'bcg', 'business', 'iit kgp', 'hdi', 'mf', 'hdd', 'allied', 'enfield', 'data analytics', 'bank of america', 'swat', 'cristiano ronaldo', 'axis', 'svm', 'mailchimp', 'kvpy sa', 'me', 'wells fargo', 'nest', 'xbox 360', 'union', 'google docs', 'cfd', 'quickbooks', 'eca', 'healthcare', 'iiit allahabad', 'mim', 'kvpy sx', 'unesco', 'ta', 'the trump administration', 'mohammad', 'google adwords', 'maya', 'fir', 'haskell', 'esl', 'agi', 'jbl', 'pes', \"5'6\", 'ux / ui', 'nit kurukshetra', 'msm', 'gujarati', 'mbb', 'rbc', 'ipe', 'pandavas', 'pizza hut', 'bi', 'cpm', 'amway', 'vps', 'pd', 'hrc', 'jabba', 'msit', 'bipc', 'trump and', 'indian government', 'vga', 'b.e.', 'ktm', 'ubc', 'amie', 'bengal', 'stanford university', 'jvm', 'nascar', 'bf', 'b.com', 'mi', 'anna', 'lds', 'subway', 'nfc', 'apk', 'png', 'berkeley', 'hpcl', 'nexus', 'redmi note', 'vmware', 'hotmail', 'ea', 'naukri.com', 'a4', 'mercedes benz', 'muoet', 'tensorflow', 'quora.com', 'arunachal pradesh', 'nicki minaj', 'rotc', 'uc', 'ncaa', 'architecture', 'ashoka university', 'xcode', 'shashi tharoor', 'indian foreign service', 'burger king', 'trump ,', 'hpv', 'the federal reserve', 'the marine corps', 'time', 'deathstroke', 'comcast', 'f-22', 'imt', 'sadhguru', 'ltc', 'g7', 'sap hana', 'kickstarter', 'jbims', 'pixar', 'naoh', 'ts eamcet', 'lotr', 'chevy', 'moses', 'robinhood', 'oxbridge', 'xbox', 'creed', 'jee advance', 'pnp', 'craigslist', 'pgp', 'vellore', 'jnu', 'rhaegar', 'poc', 'mossad', 'coorg', 'nri', 'cet', 'sprint', 'startup', 'vlsi', 'as', 'tvf', 'ftp', 'solidworks', 'adobe photoshop', 'european union', 'dungeons', 'raf', 'itr', 'lbsnaa', 'aao', 'isl', 'upa', 'brandeis university', 'spirit', 'vijay', 'harvey', 'bmsce', 'marine corps', 'brock lesnar', 'agra', 'kmc', 'msi', 'mooc', 'jiit noida', 'jss noida', 'co', 'sanders', 'upes dehradun', 'dba', 'ksa', 'iphone 8', 'salesforce', \"elon musk 's\", 'quora content review', 'u.n.', 'patreon', 'walt disney', 'srk', 'the walking dead', 'the us government', 'aiq', 'pwd', 'paypal', 'img', 'who', 'kota', 'trinity', 'university of toronto', 'fx', 'the roman catholic church', 'jedi', 'the secret service', 'central', 'aristotle', 'equifax', 'oxford university', 'bcom', 'ca cpt', 'asu', 'congressional', 'iim ahmedabad', 'air', 'bit mesra', 'iskcon', 'snu', 'phds', 'iim a', 'eps', 'eeg', 'csk', 'gc', 'the justice league', 'juventus', 'hongkong', 'carnegie mellon', 'lapd', 'al - qaeda', 'fema', 'mazda', 'cfp', 'unsc', 'chase', 'mait', 'justice', 'coep', 'arm', 'nestle', 'iiit delhi', 'pf', 'cod', 'mvc', 'olx', 'ait', 'sundar pichai', 'sem', 'esperanto', 'medium', 'winterfell', 'mht - cet', 'xamarin', 'college', 'wich', 'imperial college london', 'abu dhabi', 'baidu', 'ibps', 'ssc je', 'american express', 'quoran', 'hcf', 'hsbc', 'mb', 'ibps po', 'plato', 'google adwords', 'new york times', 'tvs', 'supreme', 'mafia', 'uti', 'cibil', 'nit rourkela', 'itil', 'iit gandhinagar', 'nmat', 'microsoft office', 'iimc', 'cell', 'ip', 'secret', 'tefl', 'jntu', 'cps', 'atv', 'western union', 'saas', 'hra', 'atp', 'chanakya', 'fwb', 'lstm', 'marathi', 'iron', 'fed', 'daiict', 'bajaj', 'wal - mart', 'ed', 'falcon heavy', 'benadryl', 'the app store', 'monsanto', 'pov', 'yoda', 'kerala', 'pkk', 'matrix', 'fc', 'bcom', 'uno', 'resonance', 'nitie', 'imu cet', 'white', 'emt'], 'NORP': ['indian', 'american', 'chinese', 'americans', 'muslims', 'indians', 'muslim', 'british', 'japanese', 'jews', 'german', 'christians', 'russian', 'european', 'christian', 'french', 'hindu', 'hindus', 'asian', 'democrats', 'jewish', 'trump', 'canadian', 'african', 'korean', 'spanish', 'republicans', 'english', 'europeans', 'pakistani', 'islamic', 'greek', 'quorans', 'russians', 'turkish', 'australian', 'italian', 'nazi', 'arab', 'israeli', 'palestinians', 'germans', 'arabs', 'asians', 'pakistanis', 'conservatives', 'republican', 'java', 'mexican', 'vietnamese', 'indonesian', 'nazis', 'irish', 'turks', 'dutch', 'arabic', 'africans', 'canadians', 'bengali', 'indian muslims', 'western', 'iranian', 'democrat', 'democrats', 'israelis', 'soviet', 'swedish', 'syrian', 'portuguese', 'persian', 'catholic', 'c++', 'sikhs', 'iranians', 'palestinian', 'mexicans', 'jew', 'koreans', 'australians', 'malaysian', 'christianity', 'quran', 'italians', 'egyptian', 'north indians', 'catholics', 'brazilian', 'buddhist', 'romans', 'north korean', 'democratic', 'liberals', 'african americans', 'kerala', 'polish', 'thai', 'roman', 'south indians', 'nigerian', 'tamilnadu', 'african american', 'scottish', 'islam', 'filipinos', 'saudi', 'republicans', 'quoran', 'native americans', 'kurds', 'singaporean', 'egyptians', 'south indian', 'latin', 'north indian', 'brits', 'indonesians', 'tamilians', 'conservatives', 'native american', 'taiwanese', 'singaporeans', 'batman', 'westerners', 'kurdish', 'swiss', 'mormon', 'hispanic', 'malaysians', 'mma', 'dhoni', 'slavic', 'east asian', 'jehovah', 'han', 'norwegian', 'nepalese', 'shia', 'armenian', 'persians', 'thapar', 'finnish', 'brazilians', 'south korean', 'scandinavian', 'zionist', 'marvel', 'sunni', 'south african', 'communist', 'republican', 'armenians', 'fortnite', 'spaniards', 'danish', 'bangladeshis', 'malay', 'nigerians', 'east asians', 'iitian', 'albanians', 'buddhists', 'youtube', 'serbs', 'hispanics', 'czech', 'democrat', 'iocl', 'mongolian', 'swedes', 'soviets', 'ukrainian', 'latin american', 'krishna', 'south koreans', 'mahabharat', 'mueller', 'zionists', 'north american', 'nordic', 'hungarian', 'lebanese', 'indian muslim', 'philippine', 'cuban', 'arabian', 'marxist', 'afghan', 'arya', 'caucasians', 'patanjali', 'nazism', 'ronaldo', 'bengalis', 'salesforce', 'serbian', 'ukrainians', 'south africans', 'middle eastern', 'south asian', 'zionism', 'diwali', 'african - american', 'victorian', 'cantonese', 'arduino', 'romanians', 'new zealanders', 'hawaiian', 'north koreans', 'iraqi', 'hindutva', 'marines', 'filipino', 'african - americans', 'iitians', 'eastern european', 'poles', 'southern', 'syrians', 'tibetan', 'gdpr', 'albanian', 'islamists', 'south american', 'communist', 'white', 'protestants', 'manali', 'orthodox', 'southeast asian', 'anglo', 'atheists', 'clans', 'sunnis', 'afghans', 'celtic', 'nepali', 'austrian', 'ottomans', 'catalan', 'saudis', 'moroccan', 'belgian', 'byzantine', 'schengen', 'marine', 'bahubali', 'martian', 'oriental', 'communists', 'ottoman', 'colombian', 'mongolians', 'new yorkers', 'islamist', 'dems', 'bulgarians', 'tennesseans', 't20', 'guardians', 'kashmiri muslims', 'maharashtra', 'balkan', 'venezuelan', 'scandinavians', 'bulgarian', 'aztec', 'kenyan', 'nicmar', 'macedonians', 'sri lankans', 'romanian', 'dravidian', 'burmese', 'hungarians', 'ethiopian', 'tirupati', 'rohingya muslims', 'baltic', 'serbians', 'b.tech', 'georgian', 'jamaican', 'californians', 'tamilian', 'puerto rican', 'western european', 'finns', 'tibetans', 'hindi', 'btech', 'latin americans', 'norwegians', 'spartans', 'aryans', 'neanderthal', 'protestant', 'kardashians', 'kaggle', 'mongol', 'white americans', 'iraqis', 'alabamians', 'mayans', 'native', 'puerto ricans', 'tajikistan', 'ips', 'south asians', 'croatian', 'tories', 'central american', 'southeast asians', 'honduran', 'black americans', 'argentine', 'libertarians', 'pakistani muslims', 'assamese', 'texans', 'indian quorans', 'algerian', 'shia muslims', 'feminists', 'roman catholic', 'caucasian', 'benghazi', 'austrians', 'sunni muslims', 'javanese', 'bayesian', 'libertarian', 'amish', 'native americans', 'aryan', 'somali', 'cambodian', 'semitic', 'sinhalese', 'malayali', 'd.', 'north americans', 'southerners', 'spartan', 'yiddish', 'saudi arabian', 'moroccans', 'catalans', 'north african', 'assyrians', 'kasol', 'azerbaijani', 'eastern europeans', 'mayan', 'argentinian', 'cartesian', 'bangladeshi', 'us marines', 'ethiopians', 'basque', 'walmart', 'swift', 'alaskan', 'leftists', 'eastern', 'northern chinese', 'trans', 'bosnian', 'newtonian', 'pre', 'sagittarius', 'siberian', 'bosniaks', 'krabi', 'western europeans', 'abrahamic', 'bhutanese', 'craigslist', 'tumblr', 'indian american', 'ba', 'b.ed', 'farsi', 'asap', 'sufism', 'afrikaans', 'g5', 'dravidians', 'protestantism', 'exodus', 'can muslims', 'titans', 'middle easterners', 'roman catholics', 'gaussian', 'manchus', 'kkk', 'californian', 'lithuanian', 'maharashtrians', 'howcan', 'jamaicans', 'dem', 'sindh', 'somalis', 'sri lankan', 'cubans', 'mamc', 'zoroastrians', 'teen titans', 'wish', 'ct', 'shinto', 'eurasian', 'estonian', 'haitians', 'belgians', 'pathans', 'north africans', 'transformers', 'semitism', 'sikhism', 'hawaiians', 'leo', 'x^2', 'nabard', 'haitian', 'elite', 'hebrew', 'asian american', 'feynman', 'american indians', 'indian christians', 'social', 'sudanese', 'hellenistic', 'afrikaners', 'hamiltonian', 'stannis', 'evangelical christians', 'prussian', 'da', 'debian', 'gujrat', 'zoroastrian', 'venezuelans', 'jordanians', 'yugoslav', 'central asian', 'qidian', 'madras', 'arabism', 'vans', 'scan', '°', 'spiderman', 'south east asian', 'caribbean', 'kardashian', 'sufi', 'inuit', 'bosnians', 'athenian', 'asian americans', 'ultra', 'moslem', 'spaniard', 'indian americans', 'shirdi', 'orwellian', 'iberian', '4d', 'flemish', 'estonians', 'tennessean', 'jordanian', 'chilean', 'tunisian', 'quranic', 'bigfoot', 'cme', 'georgians', 'pythagorean', 'arab muslims', 'indian hindus', 'algerians', 'bangladeshi muslims', 'conservative', 'croatians', 'hmong', 'yamaha', 'north vietnamese', 'mech', 'mesopotamian', 'mobile', 'southern european', 'parisians', 'olympian', 'lucknow', 'sundanese', 'libyans', 'akshay', 'congolese', '|x|', 'northern', 'sebastian', 'jats', 'peruvian', 'semites', 'machiavellian', '&', 'harvey', 'gurgaon', 'hindu muslim', 'sunni muslim', 'czechs', 'cambodians', 'orthodox christians', 'east german', 'south americans', 'saharan african', 'tamilans', 'northern european', 'keras', 'darwinian', 'bohemian', 'afsb', 'qatari', 'sumerians', 'soviets', 'jains', 'libyan', 'nationalist', 'bath', 'american muslims', 'polynesians', 'speech', 'griffin', '\\\\in', 'b.tech', 'texan', 'bhakts', 'west european', 'seiko', 't.v.', 'linguistics', 'babylonian', 'asgardians', 'saudi arabians', 'kenyans', 'argentinians', 'palestinian arabs', '^3', 'minoans', 'lightning', 'mandarin chinese', 'shia muslim', 'can indians', 'physician', 'northern europeans', 'colombians', 'cognizant', 'dominican', 'iphone x', 'saiyans', 'macedonian', 'west african', 'arab muslim', 'kryptonians', 'alchemist', 'khalistani', 'saiyan', 'hindu - muslim', 'socrates', 'olympians', 'evangelical', 'magi', 'han chinese', 'ionic', 'uzbek', 'bootstrap', 'yemeni', 'latvian', 'hasidic', 'tupac', 'adolf', 'moslems', 'alive', 'swede', 'neo - nazis', 'kurukshetra', 'parthians', 'puritans', 'european americans', 'anatolian', 'asian - americans', 'southern chinese', 'galician', 'southern europeans', 'banksy', 'rwandan', 'lutherans', 'wiccans', 'brahmin', 'sentinelese', 'icelandic', 'slovenians', 'antarctic', 'confucian', 'maltese', 'balinese', 'the north koreans', 'lithuanians', 'orthodox christian', 'freudian', 'normans', 'sino - soviet', 'rajasthani', 'cali', 'indus', 'mysql', 'satanists', 'ecuadorian', 'baja', 'atheism', 'tirunelveli', 'galatians', 'nicaraguan', 'bordeaux', 'tasmanian', 'authoritarian', 'israelites', 'brasil', 'non - muslims', 'ww2 german', 'gazans', 'shiites', 'verilog', 'logistics', 'branch', 'andaman', 'humans', 'deccan', 'socialists', 'dravid', 'maharashtrian', 'pharisees', 'south vietnamese', 'sgpt', 'non', 'yugoslavian', 'catholicism', 'sulawesi', 'anglicans', 'bacardi', 'latvians', 'east african', 'darwinism', 'girls’', 'serb', 'baelish', 'non - muslims', 'hindu indians', 'magoosh', 'azerbaijanis', 'ukranian', 'christian republicans', 'psychiatrist', 'hackintosh', 'siamese', 'travel', 'tory', 'spring', 'ferb', 'stockfish', 'madras christian', 'palantir', 'borivali', 'buddhism', 'austronesian', 'yemenis', 'hainanese', 'satanist', 'mainland chinese', 'ashkenazi', 'manchurian', 'thoothukudi', 'eastern orthodox', 'intuitive', 'politicians', 'kryptonian', 'ponzi', 'polynesian', 'accenture', 'punch', 'uruguayans', 'amur', 'presbyterian', 'oxiclean', 'eritrean', 'anabaptists', 'sicilians', 'u.k', 'anglican', 'sino - indian', 'bolivian', '\\\\lim', 'irishman', 'svalbard', 'baptists', 'khalistan', 'quoras', 'pomeranian', 'hyderabadi muslims', 'boolean', 'iberia', 'kargil', 'confederate', 'traditional chinese', 'judeo', 'punisher', 'nationalists', 'black', 'catalonian', 'slovenian', 'kinetic', 'israeli arabs', 'logitech', 'mandarin', 'white american', 'phoenician', 'gulf arabs', 'east indian', 'us', 'bangaloreans', 'federalism', 'portugese', 'costa rican', 'east european', 'islamism', 'rohingya', 'red', 'r.', 'judeo - christian', 'superboy', 'shanghainese', 'lcm', 'islanders', 'spanglish', 'hvac', 'chechen', 'black american', 'kantian', 'chinese indonesian', 'west africans', 'zimbabwean', 'overwatch', 'romantic', 'narcissists', 'tuskegee', 'shiite', 'naukri', 'midwestern', 'klout', 'brittish', 'nitish', 'ashish', 'snapfish', 'romance', 'democratic', 'stark', 'quorians', 'northeast asians', 'thanos', 'hakka', 'east indians', 'mains', 'uruguayan', 'bangladesh', 'samaritans', 'khans', 'northerners', 'blinkist', 'browning', 'gemini', 'belarusian', 'maharatna', 'east - asian', 'racist', 'banaras hindu', 'olympiad', 'jungian', 'carthaginians', 'south east asians', 'aramco', 'sicilian', 'cancerian', 'ayurvedic', 'bactrian', 'plains indians', 'taoist', 'popular', 'tunisians', 'mediterraneans', 'ottoman jews', 'separatist', 'marxian', 'southern indians', 'lmrc', 'north eastern', 'caste', 'retrograde', 'infowars', 'asgardian', 'gandhian', 'eurasians', 'jinping', 'west asian', 'wolverine', 'experian', 'attic', 'assyrian', 'valyrian', 'black africans', 'hindustani', 'sino - tibetan', 'pakistans', 'bengali muslims', 'konkani', 'brahmans', 'melanesian', 'a sunni muslim', 'amhara muslims', 'saudi arab', 'tyrion', 'nubians', 'enceladus', 'hindu indian', 'satish', 'hindu bengalis', 'an african american', 'permian', 'shotokan', 'x=2', 'qurans', 'korean americans', 'theranos', 'flanders', 'humpty', 'unsullied', 'spartacus', 'outcasts', 'narayan', 'exorcist', 'seq2seq', 'neo - nazi', 'scientologist', 'julian', 'asian - american', 'n^2', 'crank', 'gods', 'adonis', 'dystopian', 'herman', 'vulcans', 'mac', 'sushi', 'te', 'russo', 'japanese - americans', 'southern italians', 'kharagpur', 'vegetarians', 'circassian', 'northeast indians', 'sac', 'sumerian', 'suite', 'stack', 'mesopotamians', 'namibian', 'native indians', 'hans', 'inhumans', 'athenians', 'bellatrix', 'iranic', 'jihadists', 'the african american', 'jeffersonian', 'babylonians', 'sino - japanese', 'mycenaean', 'pokémon', 'ltd', 'wwii german', 'jewish americans', 'fortran', 'gulf arab', 'kubernetes', 'intelligent', 'non - muslim', 'nephilim', 'elastic', 'non muslims', 'wahhabism', 'stalinism', 'ichange', 'east africans', 'kuwaitis', 'indian hindu', 'messianic jews', 'french canadians', 'afro - americans', 'trinidadian', 'the american', 'the ottoman turks', 'yankee', 'ancient greek', 'facial', 'east germans', 'atlassian', 'karan', 'etruscans', 'british muslims', '4-digit', 'sociopath', 's400', 'magicians', 'white europeans', 'alaskans', 'inuyasha', 'sjw', 'blowfish', 'thermodynamics', 'punish', '100ml', 'zoroastrianism', 'japanese americans', 'crystal', 'chinese americans', 'b.tech', 'sinitic', 'dalmatian', 'animals', 'roseanne', 'zimbabweans', 'caribbeans', 'hindu bengali', 'marshal', 'liberian', 'dharamshala', 'arabians', 'aerospace', 'eucharist', 'easterners', 'the native americans', 'tsipras', 'singlish', 'galactic', 'saharan africans', 'west europeans', 'ptu', 'history', 'parisian', 'fiverr', 'luxembourgish', 'brahminism', 'rajahmundry', 'sardinian', 'thracian', 'appian', 'guatemalan', 'bosnian muslims', 'laotians', 'bruneians', 'thessalonians', 'tongans', 'british indian', 'sino - russian', 'waters', 'pelicans', 'jesuit', 'ottoman turkish', 'mesoamerican', 'central americans', 'scythian', 'the african americans', 'elon', 'peaks', 'bitsat', 'ethnic', 'ssb', 'bilbo', 'native german', 'cheese', 'fetish', 'russian orthodox', 'titan', 'dish', 'peruvians', 'kazakh', 'tanzanian', 'pagani', 'hortonworks', 'ssid', 'sans', 'jacksonian', 'vashikaran', 'laxman', 'robotics', 'automatic', 'netsuite', 'scythians', 'monsta x', 'rishi', 'indian malaysians', 'humanism', 'romulans', 'vegans', 'native indians', 'cavendish', 'hindu nationalists', 'polytechnique', 'irish - americans', 'eritreans', 'mauritian', 'hindians', 'librarians', 'selfish', 'american muslim', 'civil', 'hootsuite', 'netbeans', 'anglophones', 'eroupian', 'syeds', 'teutonic', 'himalayan', 'daoist', 'sunpower', 'coolpad', 'है', 'phoenicians', 'nusrat', 'panamanian', 'jovian', 'acadians', 'hutterites', 'ugandans', 'red indians', 'frenchman', 'pa', 'northern han', 'middle - easterners', 'malaysian indians', 'lesbians', 'siberians', 'chinese malaysians', 'europian', 'asoiaf', 'northern indians', 'maoist', 'hessian', 'crow', 'ghanaian', 'nightwish', 'gnostic', 'pacifica', 'western muslims', 'dabolim', 'ramadhan', 'north european', 'reptilian', 'biafrans', 'nabard', 'martians', 'northeast indian', 'tolani', 'agnostics', 'israelite', 'irish catholics', 'orthodox jews', 'nuyoricans', 'superwoman', 'conservative', 'sat english', 'acropolis', 'northern african', 'native indonesian', 'strat', 'n’', 'napolean', 'pattern', 'german - americans', 'argentines', 'ashish kedia', 'parbhani', 'fish', 'shiite muslims', 'pakistani muslim', 'hamming', 'existentialism', 'quranists', 'insert', 'floridians', 'bangladeshi hindus', 'the north indians', 'panchkula', 'mahomet', '250w', 'calvinists', 'partners', 'proactiv', 'skynet', '√4', 'todoist', 'huracan', 'tagalog', 'pusa', 'polimi', 'irani', 'barbarians', 'comic sans', 'ukranians', 'south indian muslims', 'seinfeld', 'lgbt', 'pf', 'merovingian', 'pinot', 'northern irish', 'indian christian', 'niantic', 'european muslims', 'trump americans', 'kosovar', 'ambani', 'chicagoans', 'veterans', \"iit'ians\", 'macbook', 'etruscan', 'beef', 'northeast asian', 'weavesilk', 'snape', 'dystopian', 'namibians', 'momentum', 'interviews', 'latin - american', 'contra', 'south slavic', 'castilian', 'indian bengalis', 'white european', 'samrat', 'adwords', 'amazonian', 'japanese quorans', 'pagans', 'antibiotic', 'chileans', 'carribean', 'kurds', 'yellowknife', 'gladiator', 'mathematicians', 'andamans', 'aromanians', 'symantec', 'left', 'shanghaiese', 'voip', 'european american', 'vietnamese americans', 'irishmen', 'malaysian muslim', 'hunters', 'malay muslim', 'latinos', 'celtics', 'glofish', 'anarchists', 'upvotes', 'lite', 'advani', 'european christian', 'amerindian', 'muslims', 'frisian', 'incan', 'ordovician', 'levant arabs', 'prussians', 'chinese muslims', 'ba german', 'brownian'], 'WORK_OF_ART': ['mba', 'phd', 'bible', 'star wars', 'ma', 'batman', 'star trek', 'love', 'marvel', 'nobel', 'ph.d.', '&', 'me', 'koran', 'harry potter', 'stranger things', 'deep learning', 'quoran', \"a master 's degree\", 'the nobel peace prize', 'old testament', 'quora', 'nobel prize', 'ruby on rails', 'sharia', 'phd', 'brahmin', 'beauty and the beast', 'prove', 'the declaration of independence', 'the simpsons', 'goa', 'the night king', 'metoo', 'the walking dead', 'a nobel prize', 'the nobel prize', 'friends', 'the infinity gauntlet', 'the outsiders', 'new testament', 'soul eater', 'the scarlet letter', 'gandhinagar', 'a song of ice and fire', 'macos', 'hamlet', 'punch man', 'death note', '1984', 'nobel peace prize', 'the mona lisa', 'gd', 'the queen of england', 'emba', 'sagittarius', 'the grapes of wrath', 'game of thrones', 'the great gatsby', 'the pledge of allegiance', 'pretty little liars', \"alice 's adventures in wonderland\", 'the kite runner', 'black lives matter', 'oscar', 'the vampire diaries', \"master 's degree\", 'the giver', 'a new hope', 'hannah montana', 'the jungle book', 'breaking bad', 'the hunger games', 'star trek discovery', \"my master 's degree\", 'the book of mormon', 'diary of a wimpy kid', 'pokémon', 'treasure island', 'machine learning', 'terminator', 'sesame street', 'nature', 'holocaust', 'fake news', 'the art of war', 'the wire', 'the hunchback of notre dame', 'roseanne', 'the big bang theory', 'fairy tail', 'breaking dawn', 'animal farm', 'soul', 'fight club', 'fifty shades of grey', 'amber heard', 'a space odyssey', 'marvel or dc', 'playboy', 'ca final', 'star wars story', 'the treaty of versailles', 'the andy griffith show', \"the handmaid 's tale\", 'the legend of zelda', 'the goblet of fire', 'genesis', 'stranger', 'dreamers', 'nobel prizes', 'dragon ball z', 'the son of neptune', 'hurricane', 'mains', 'google', 'nobel prizes', 'black butler', 'tender is the night', 'the book of revelation', 'the secret', 'tide pods', 'this is america', 'the crucible', 'chennai', 'the holy bible', 'bud , not buddy', 'lily potter', 'summer of the monkeys', 'the dark knight', 'i love lucy', 'the shining', 'spider - man', 'family guy', 'the secret life of bees', 'a rose for emily', 'black panther', 'yu - gi - oh ! gx', 'star wars : the last jedi', 'science', 'the mad king', 'the legend of bagger vance', 'the federalist papers', 'ready player one', 'days of our lives', 'marvel and dc', 'walker', 'stormfront', 'rich dad poor dad', 'new moon', 'the black cat', 'the promise', 'the prince of tennis', 'the necklace', 'the legend of korra', 'teen titans', 'the play store', 'the rogue', 'goa', 'sonic the hedgehog', 'thirteen reasons why', 'saved by the bell', 'the road not taken', 'the little mermaid', 'the happy prince and other tales', 'the fairly oddparents', 'infinity wars', 'the fall of the house of usher', 'a nobel peace prize', 'the merchant of venice', 'skulduggery pleasant', 'the nobel prize in physics', 'revelation', 'the tesla semi', 'the way of the world', 'the maze runner', 'the chronicles of narnia', 'star wars episode', 'the kane chronicles', 'fortune 500', 'i++', 'the things they carried', 'the young and the restless', 'the next generation', 'wonder woman', 'the cosby show', 'a wrinkle in time', 'saturday night live', 'ghost hunters', 'batman beyond', 'seinfeld', 'the originals', 'the bell jar', 'the catcher in the rye', 'venus', 'danes', 'economic and philosophic manuscripts of 1844', \"star trek 's\", 'trump', 'the wizard of oz', \"do n't die , my love\", 'love', 'sword art online', 'sirius', 'genius', 'the lovely bones', 'the waltons', 'star trek : the next generation', 'the lion king', 't20', 'finance', 'wuthering heights', 'avatar', 'a brief history of time', 'the stepford wives', 'dragons', 'the coldest winter ever', 'jurisprudence', 'the picture of dorian gray', 'toys r us', 'family ties', 'spider - man : homecoming', 'one love', 'chicken', 'the deerslayer', 'star trek beyond', 'grammy', 'punch', 'into the wild', 'insta', 'perfect', 'network', 'civil peace', 'already taken 2', 'adventure', 'sister , sister', 'yu - gi - oh', 'enduring love', 'loser', 'a clockwork orange', 'game', 'sunny leone', 'the chamber of secrets', 'the metamorphosis', 'natural law and natural rights', 'dexter', 'rurouni kenshin', 'wizards of waverly place', 'the twilight zone', 'end of the spear', 'starship troopers', 'primer', 'the secret garden', '3000', 'the game : penetrating the secret society of pickup artists', 'return of the jedi', 'star wars legends', \"' the prince '\", \"5'3\", 'teen wolf', 'redwall', 'sargon of akkad', 'the gruffalo', 'the contender', 'the fog horn', 'wolf quest', 'blood on the river', 'the law of life', 'where the wild things are', 'angel eyes', 'ecstasy of saint teresa', 'the red pill', 'i am legend', \"no man 's sky\", 'road to nowhere', 'tesla', 'the island', 'the little prince', 'before i go to sleep', 'house of night', \"' the dark knight '\", 'the untouchables', 'harry potter and the half - blood prince', 'the blood of olympus', 'le havre', 'dear white people', 'the brothers karamazov', 'the voice', 'the red badge of courage', 'depression', 'hansel and gretel', 'tuck everlasting', 'miracle at st. anna', 'dolphin tale', 'a christmas carol', 'heart of the ocean', 'ca', 'the brook', 'the light in the forest', 'harry potter and the deathly hallows – part 1', 'the solitary reaper', 'lazytown', 'life', 'the neverending story', 'star trek : discovery', 'hacker', 'the nobel prizes', 'the hobbit', 'eat , pray , love', 'a faraway home', 'heat', 'the phantom of the opera', 'gods', 'the gathering', 'winnie - the - pooh', 'gentlemen of the road', 'cse', 'the mandela effect', 'deadpool', 'hetalia : axis powers', 'html and css', 'hatchet', 'fallen angels', 'the dust bowl', '\" the catcher in the rye \"', 'jacket', 'catcher in the rye', \"cat 's cradle\", 'dance moms', 'the matrix', 'a walk to remember', 'the city of ember', 'the jungle', 'alone', 'the master in management', 'out of the flames', 'the canterbury tales', 'cengage', 'the green mile', 'what is ancient philosophy', 'the real housewives of atlanta', 'the mill and the cross', \"look up daughter : ca n't you see i love you\", 'days of our lives \"', 'mobile suit gundam wing', 'i can only imagine', 'requiem for a dream', 'crime and the american dream', 'hope for the flowers', 'style', 'durgapur', 'black', 'the alchemist', 'spring boot', \"the night king 's\", 'the invisible man', 'the last jedi', 'the flintstones', 'the tell - tale heart', 'gnu', 'amazing grace', 'the view', 'the melancholy of haruhi suzumiya', 'the rocking - horse winner', 'the notebook', 'a modest', 'the 10th kingdom', 'a series of unfortunate events', 'art of war', 'pulitzer prize', 'gone with the wind', \"a master 's\", 'apocalypse now', 'fahrenheit 451', 'star wars : the force awakens', 'human', 'miracle on 34th street', 'harry potter and the prisoner of azkaban', \"a master 's in computer science\", 'breitbart', 'the book thief', 'the devil and tom walker', 'what is the what', 'the fifth element', 'the methods of rationality', '\" the dark tower \"', 'massacre at myall creek', 'the interpretation of dreams', 'thriller', 'me too', 'cheers', 'weeds', 'sugar land', 'gold medal', 'prodigal son', 'ww 2', 'in light of india', 'the rain came', 'the subtle knife', 'the long dark night', 'lion', 'star trek the next generation', 'quran', 'the god delusion', 'woodbury , ny', 'poverty is where the money is', \"theodore ' the beaver ' cleaver\", 'leave it to beaver', 'the black panther', 'the adventures of precious the dog', 'star wars : the old republic', 'the furnished room', '\" the golden child \"', 'the silver lotus', 'les liaisons dangereuses', 'the lottery', 'gaffer', 'ghost whisperer', 'sympathy for mr. vengeance', 'the charmer', 'the tenant of wildfell hall', 'the bell curve', 'xenomorph queen', 'star trek : enterprise', 'bates motel', 'arjun reddy', \"the guv'nor\", '\" the book thief \"', 'a raisin in the sun', 'the big bang', 'the end of love', 'soul surfer', \"5'9.5\", 'when the scorpion stings : the history of the 3rd cavalry regiment , south vietnam', 'sicario', 'the good fairies of new york', 'the bluest eye', 'break a leg : a memoir', 'by night in chile', 'more than you know', 'thirst', 'digging', 'a corner of the universe', 'the story of my experiments with truth', 'the mortal immortal', '“ family guy ”', 'the adventures of pinocchio', 'seven deadly sins', 'b17', 'why earls fall in love', 'the original six', 'brandi love', 'beowulf', 'zorba the greek', 'israel’', 'glowstone', 'an inspector calls', 'the curious case of benjamin button', '\" falling up \"', 'the prince', 'the power trip', 'i know you got soul', 'the egypt game', 'tales from the crypt', 'intelligent', 'the gift of the magi', 'howard the duck', 'la duchesse de langeais', 'nobody knows but me', 'the soul of a new machine', 'first light', 'a prayer for owen meany', 'canterwood crest', 'person of interest', 'freedom or death', 'the twilight saga', 'stig of the dump', 'forrest gump', 'home', 'where the red fern grows', 'battle royale', 'srm main campus', 'absalom and achitophel', 'the land of silver rain', 'in search of deep throat', 'el chavo del ocho', 'this is america', 'i want to be a police', 'the garden of forking paths', 'the temple of master hotei', 'afternoon of a faun', 'the gun', 'bosniaks', 'ambition', 'the wellness solution', 'raspberry pi', 'out of the ordinary', \"peter 's lullaby\", 'people you may know', 'the sleeper awakes', 'the influence of sea power upon history', 'a long way down', 'the true story of christopher emmanuel balestrero', 'the jerry springer show', 'pokémon : diamond and pearl', 'the pearl', '\" love waves \"', 'phora', 'the mindy project', 'cryptonomicon', 'the return of the king', 'lovelorn', 'where good ideas come from', 'blood brothers', 'the wars of the roses', 'somewhere over the rainbow', 'the night listener', 'libra moon', 'his needs , her needs : building an affair - proof marriage', 'the secret of the great red spot : the jupiter chronicles - book 1', 'the late show', 'the king of queens', 'the post', 'the great', 'poughkeepsie', 'pride and prejudice', '1408', 'the wreck of the hesperus', 'clear light of day', '2/2/2', 'raising the bar', 'star trek conventions', 'anne of green gables', 'october sky', 'dancing with the stars', 'the power of now', 'the rock : exodus', 'a mystery of heroism', 'spring', 'touched by an angel', 'angels and demons', 'buffy the vampire slayer', 'the lincoln hunters', 'the scooby - doo show', \"black 's law dictionary\", 'black and white', 'gun shy', 'a wonderful life', 'cover up', 'hubble', 'the word of god for the people of god', 'the ten commandments', 'the diary of anne frank', 'on the sidewalk bleeding', 'soul to soul : a black russian american family 1865 - 1992', 'the jee mains', 'the second sin', 'lie to me', 'poison', 'world wars', 'runner', 'something borrowed', 'the edge of evolution', 'uptown funk', 'the holy land', 'the winds of winter', 'the sign of the beaver', 'ai', 'a game of thrones', 'the portrait of a lady', 'two to the fifth', 'sweat', 'where lies the land', 'vande matram', 'the land of blood and honey : the rise of modern israel', 'mathematical universe hypothesis', 'a new earth : awakening to your life ’s purpose', 'down through the years', 'conservative', 'bones', 'switch', \"gulliver 's travels\", 'the alamo', 'gates of hades', 'close your eyes', 'the c programming language', 'the secrets of the immortal nicholas flamel', 'the fall', 'you are old', 'a new china : an intermediate reader of modern chinese', 'cat and mouse', 'bright', 'a little cloud', 'barn burning', 'the usual suspects', 'the neverending story', 'the \" hitler myth \"', 'barking up the wrong tree', 'the dawn patrol', 'the rum diary', 'listography journal : your life in lists', 'the law of the sea', 'the avengers', 'traction', 'the baby in the bottle', 'yu - gi - oh gx', 'messages to the world', 'the wheel of time', 'doctor who', 'the mars project', 'i love you', 'star trek tos', \"' the room '\", 'the 13th floor', 'touch blue', 'where grace abounds : an amish widow romance', 'the american crisis', 'dear john', 'the golden twine', 'remember me', 'the sea of monsters', \"olive 's ocean\", 'dl', 'faces in the crowd', 'the gods of the copybook headings', 'age of empires', '2lakhs', 'city of bones', 'gentle rogue', 'where treetops glisten : three stories of heartwarming courage', 'get away with murder', 'surrender', 'pavitra rishta', 'ghost hunt', 'kafka on the shore', 'trouble river', 'killing pablo', 'demon hunters : trinity', 'the star child', 'bayonet charge', 'book', 'tapas on the ramblas', 'god in me', 'maggie', 'the federalist papers', 'the battle of the labyrinth', 'the southern vampire mysteries', \"where 's wally\", 'beyonce', 'personal narrative of a pilgrimage to al - madinah and meccah', 'we have met the enemy : self - control', 'the terminator', 'sushi pack', 'the missing gospels', 'the god of small things', 'big words for little people', 'where mercy flows', 'what dreams may come', 'star wars rebels', 'black cat', 'point blanc', 'rich dad , poor dad', 'true blue', 'the black pearl', 'facebook', 'the valley of the shadow of death : a tale of tragedy and redemption', 'i see you', 'jagua nana', 'noah', 'refugee blues', 'swades', 'cs', 'the crown', 'home to harlem', \"let 's stay together\", \"' to sleep with the angels ''\", 'phasma ( star wars ) : journey to star wars : the last jedi', 'boot camp', 'imagine me and you : a novel \"', 'smallville', 'marvel ’s black panther', '6s', '\" water for elephants \"', 'boost your bust', 'the book of five rings', 'the moon is a harsh mistress', 'looking for alaska', 'the man from earth', 'the messenger', 'the last laugh', 'i love you , beth cooper', 'thorn in my pride : a novel of the l.a. riots of 1992', '\" the vampire diaries \"', 'every last one', 'amber addison', 'fullmetal alchemist', 'the trip', 'the edge of seventeen', 'remember this titan : the bill yoast story', 'allah hu akbar', 'wounded', 'the purge', 'spring breakers', 'the day of the triffids', 'a master', 'bahadur shah', 'star trek voyager', 'pi', 'zodiac', 'the lucifer effect', 'm.d.', 'chain saw confidential', 'the townshend acts', \"le morte d'arthur\", 'all lives matter', 'how do i love thee', 'the thin man', 'avatar : the last airbender', 'star trek : voyager', 'lipstick jungle', 'the greatest generation', 'the summer day', 'the bungalow mystery', 'glee', 'billions', 'dceu', 'the rockford files', 'secret angels', 'maiden rose', 'profit from your losses', 'boardwalk empire', 'bigg boss', 'dharma & greg', 'the adventures of tom sawyer', \"the international jew '\", 'times now', 'the loved one', 'muppet treasure island : sailing for adventure', 'grudge match', 'the lightning thief', 'the fault in our stars', 'timothy of the cay', \"the millionaire next door : the surprising secrets of america 's wealthy\", 'conflict and compromise', 'daytime emmy award', 'the flowers of may', 'android to android', 'an afrocentric manifesto', 'ultra moon', 'star wars vii', 'safe', 'star wars : episode ix', 'the l word', 'a light in the attic', 'a week in the woods', 'ergo proxy', 'the call of the wild', 'dear mr. henshaw', 'the game', 'chasing redbird', 'pink ribbons , inc : breast cancer', 'deus ex machina', 'the gifted', 'californication', 'heavily suppressed )', 'war and peace', 'picasso', \"spring 's sprung\", 'roll of thunder , hear my cry', 'rockstar', '1 and', 'the soul stone', 'islamophobia', 'zatch bell', 'the diary of a young girl', 'down these mean streets', 'tangerine', 'the man who invented christmas', 'gotti', 'the name of the rose', 'the fun they had', 'the candymakers', 'the bean trees', 'for the love of money : a memoir', 'cat person', 'alice 19th', 'jedi master', 'barnes', 'the seventh tower', 'witch hunt', 'tuxedo mask', 'cut the deck', 'life is good : the book', 'all about the benjamins', 'after ever after', 'sanctuary', 'the way things ought to be', 'the world is flat', 'alice in the country of hearts', 'mistborn', 'wish you were here', 'snow crash', 'shooter', 'gnss', 'sweet home alabama', 'sjw', 'the power of habit', 'independence', 'gypsies', 'the red king', 'fish in the trap', 'star wars and marvel', 'black people', 'the unfoldment : the organic path to clarity , power , and transformation', 'the good earth', 'the smell of war', 'inspector george gently', 'the amnesiac', 'the mathematics of life', 'panama papers', 'picnic', 'look what you made me do', 'the grey lady', 'all quiet on the western front', 'recovery of your inner child : the highly acclaimed method for liberating your inner self', 'big nate : flips out', 'death by chocolate', 'the odyssey of kp2', 'dynasty', 'the freedom writers diary', 'the rescuers down under', 'the adventures of huckleberry finn', 'calcutta', 'the forsaken', 'computer science', 'harry potter and the cursed child', 'inkdeath', 'lock and key', 'the seussification of romeo and juliet', 'mickey mouse clubhouse', 'infinity bee pollen', 'gorgias : encomium of helen', 'the sleeping gypsy', 'sentinel event', 'twilight', 'internal revenue code', 'the art of exceptional living', 'city of glass', 'the little match girl', 'the age of innocence', 'runescape', 'x - men', 'donmeh', 'zimmerman', 'midnight for charlie bone', 'the best birth : your guide', 'do you believe in miracles', 'star wars : rebels', 'death to america', 'the official preppy handbook', 'the great dictator', 'the wolf of wall street', 'where the lost things go', 'the phantom tollbooth', 'the great khali', 'james and the giant peach', 'the sea of trolls', 'ender', 'rightmove', 'the birds', 'the black knight', \"irresistible : why we ca n't stop checking , scrolling , clicking and watching\", 'tales of the frog princess', 'notification', 'a master of science', \"a master 's or\", 'the burning bridge', 'the sisters grimm', 'furio giunta', 'the nightmare before christmas', 'saturday night', 'playing the game : a novel', 'the open window', 'mischief night', 'star', \"when nothing else matters : michael jordan 's last comeback\", 'dark side of the moon', 'naruto', 'young goodman brown', 'bit of rough', 'the art of noises', 'origin', 'going too far', 'jeopardy', 'king of new york', 'the fly', 'watchmen', 'sixteen candles', 'consumed by you', 'mcdougal littell biology', 'the closing of the american mind', 'only cows allowed', 'i dream of jeannie', 'in living color', \"a midsummer night 's dream\", \"my master 's\", 'kaun banega crorepati', 'the white giraffe', '\" love across the salt desert \"', 'star trek v : the final frontier', 'their eyes were watching god', 'a few good men', 'the addams family', 'dragon ball', \"the doll 's house\", 'nobel laureates', 'saw', 'the book of fred', 'mummy', 'batman & robin', 'flowers for algernon', 'the book of eli', 'fight or flight', 'sons of anarchy', 'star trek enterprise', 'angels & demons', \"raven 's lucky numbers dream book : the only lottery book you 'll ever need\", 'the pacific', \"cook 's country\", 'the deathly hallows', 'revelations', 'the children of the forest', 'deep is the night : dark fire', 'bengaluru in karnataka', 'the giver quartet', 'princess hours', 'little house on the prairie', 'commissary kitchen', 'the egg', 'goi', 'the metamorphosis of prime intellect', 'the language of dance', 'jon snow', 'sunrise over fallujah', 'football genius', 'i light a candle', 'thought architectures', 'the wild swans at coole', 'taboo', 'the hunchback of notre - dame', 'boa constrictor', 'the black cassock', 'the princess bride', 'ga - rei', 'final cut pro x', 'torah - observant', 'the wealth of nations', 'the first day', 'i just want to be loved by you and you and you', 'the deer hunter', 'shipping business', 'boogie nights', 'the hollow kingdom trilogy', 'steel magnolias', 'whose line is it anyway', 'sing down the moon', 'the incredibles 2', 'the turn of the screw', 'my little pony : friendship', 'huswifery', 'scrum master', 'build', 'stark', 'hellboy', 'bits hd', 'wide sargasso sea', 'the sand reckoner', 'shut the door', 'foreigner', 'the royal tenenbaums', 'graph theory', 'once upon a time', 'a book of opposites', 'the doll people', 'world of warcraft', 'tragedy in tin can holler', 'the vantage point', 'lethal', 'a long way gone', 'comedk', 'unbornness : human pre - existence and the journey toward birth', 'the truman show', 'political science', 'fear factor', 'the moon', 'super mario run', 'the vampire huntress legend series', 'the rainbow bridge : a visit to pet paradise', 'goosebumps', 'war', '\" true blood \"', 'sense and sensibility', 'wild at heart', 'marvel cinematic universe', '\" the outsiders \"', 'the lord of the rings', 'nougat bits', 'e.t.', 'hemlock grove', 'the ted knight show', 'liberalism', 'rocket man', 'the last airbender', 'the fall of troy', 'the sunil gavaskar omnibus', 'they made me do', 'the poisonwood bible', 'the dark knight rises', 'bored to death', 'think out of the box', 'goonda', 'the western tradition : from the ancient world to louis xiv', 'flint', 'confessions of a dangerous mind', 'sex and the city', 'a child called it', 'notes on a scandal', 'midnight sun', 'attack on titan', 'mev', 'the tale of despereaux', 'my life in ruins', 'ai no kusabi', 'othello', 'last shot', 'pluralsight', \"grey 's anatomy\", 'the missing', 'casablanca', 'when the wind blows', 'the nanny', 'toddlers & tiaras', \"night 's kiss\", 'dark shadows', 'stargate atlantis', 'napster', 'the wind in the willows', 'the manchurian candidate', 'pulitzer', 'the old testament', 'quora user', 'on the take \"', 'the scholar gipsy', 'b.c.', 'mein kampf', 'danny phantom', 'sufi vision and initiation : meetings with remarkable', 'green light', 'when autumn falls', 'the white queen', \"' love '\", 'the tyger', 'hazing meri sugarman', \"blu 's hanging\", 'this american life', 'hoops', 'sick', 'labyrinth of memories : a child in the spanish civil war', 'the hills have eyes : the beginning', 'toy story', 'the story of tracy beaker', 'the last of the really great whangdoodles', 'the open boat', 'a bitter brew : faith , power , and poison in a small new england town', 'maple skate', 'elements of art', 'dunkirk', 'demons', 'rules of the road', 'the young turks', 'manchild in the promised land', 'malice', 'the red pyramid', 'the restless gun', 'roll the bones : the history of gambling \"', 'the fountainhead', 'american pie', \"rennie 's way\", 'usa', 'the last supper', 'op', 'bloomability', 'from the dark tower', 'memoirs of a geisha', 'the inner world of jimi hendrix', 'mark', 'neighbours', 'emily windsnap', 'gambit', 'nagpur', 'the killer angels : a novel of the civil war', 'p90', 's.t.o.r.m. - the black sphere', 'm.sc mathematics', 'the circuit : stories from the life of a migrant child', \"ca n't take my eyes off of you\", 'under a war - torn sky', 'a tesla semi', 'the paul and young ron show', 'don quixote', 'artemis fowl', \"lucifer 's hammer\", 'impossible', 'the belly of paris', \"i 'll see you again\", 'amulet', 'the cask of amontillado', 'bronx masquerade', 'forest gump', \"constructor theory '\", 'nothing succeeds like success', 'beetle', 'fort of chittorgarh', 'aladdin', 'beyond the hills : the journey of waite phillips', 'six cherry blossoms : and other stories', 'dwarf fortress', 'the count of monte cristo', 'mea culpa', 'the \" maximum ride : school \\'s out forever \"', 'individual vs society', 'star trek :', 'friendly fire', 'an evolutionary view of women at work', 'installed applications', 'trump and the attack of the bowling ball', 'the house of the spirits \" by isabel allende', 'the love of my life', 'the oedipus plays', 'what i know for sure', 'proofs', 'the king in yellow', 'oliver nocturne', 'pawn stars', 'parallel lives', 'the manhood of the master', 'soy un socialista , soy un comunista , soy chicano', 'the doomsday book', 'daily stormer or', 'kool - aid', 'la haine', 'joe \" brown', 'quoran q&a', 'the great khali \"', \"the parson 's tale\", 'promoting the common good : bringing economics and theology together again', 'weinstein , moore and franken', 'currency manipulator', 'grand sponsors', 'heavy rain', 'dirty money', '\" confidence man \"', 'how we roll', 'euro', 'the c++ programming language', 'plan', 'graylog', 'american dragon : jake long', 'a \" gun ban \"', 'satellite c655-s5512 laptop be improved ?', '“ chopper of republic of anzakistan ”', 'ethnic', 'madame butterfly', 'the white tiger', '\" master class \"', 'edible bones', \"their holy book as quran rather than '\", 'engineering , sweden', 'the american dream', 'secret society', '\" the game : penetrating the secret society of pickup artists \"', '“ now more than ever', 'k.j.h.p.h\"at the top of king xi punjab', \"harry and the wrinkles '\", 'the force awakens', 'food science and quality', 'smile guide', \"' yeti obhijaan '\", 'the world god only knows', 'bakrichods', 'turkish delight', 'to do list \"', \"a master 's degree from\", 'war for planet of the apes', 'differenz', 'schellingschen systems', 'kiss me', 'two whatever', 'everyday use', 'master of the game']}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63cb21525251b060aeb309e7be4b48772f8720f5","scrolled":true,"trusted":true},"cell_type":"code","source":"def parallelize_apply(df,func,colname,num_process,newcolnames):\n    # takes as input a df and a function for one of the columns in df\n    pool =Pool(processes=num_process)\n    arraydata = pool.map(func,tqdm(df[colname].values))\n    pool.close()\n    \n    newdf = pd.DataFrame(arraydata,columns = newcolnames)\n    df = pd.concat([df,newdf],axis=1)\n    return df\n\ndef parallelize_dataframe(df, func):\n    df_split = np.array_split(df, 4)\n    pool = Pool(4)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n\ndef count_regexp_occ(regexp=\"\", text=None):\n    \"\"\" Simple way to get the number of occurence of a regex\"\"\"\n    return len(re.findall(regexp, text))\n\ndef sentiment(comment):\n    testimonial = TextBlob(comment)\n    a = testimonial.sentiment\n    return [a.polarity,a.subjectivity]\n\ndef fword(x,word):\n    if (x.split(\" \")[0].lower() ==word):\n        return 1\n    else: return 0\n    \n# some fetures \n# ['','','india/n','','quora','','sex','','','country/countries','china','','','chinese','','']\ndef add_features(df):\n    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n    df[\"lower_question_text\"] = df[\"question_text\"].apply(lambda x: x.lower())\n    # df = parallelize_apply(df,sentiment,'question_text',4,['sentiment','subjectivity']) \n    # df['sentiment'] = df['question_text'].progress_apply(lambda x:sentiment(x))\n    df['total_length'] = df['question_text'].progress_apply(len)\n    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words'] \n    # Adding new feats from meta kernel - already .697 will it improve#\n    df['num_exclamation_marks'] = df['question_text'].progress_apply(lambda comment: comment.count('!'))\n    df['num_question_marks'] = df['question_text'].progress_apply(lambda comment: comment.count('?'))\n    df['num_punctuation'] = df['question_text'].progress_apply(lambda comment: sum(comment.count(w) for w in '.,;:'))\n    df['num_symbols'] = df['question_text'].progress_apply(lambda comment: sum(comment.count(w) for w in '*&$%'))\n    df['num_smilies'] = df['question_text'].progress_apply(lambda comment: sum(comment.count(w) for w in (':-)', ':)', ';-)', ';)')))\n    df['num_sad'] = df['question_text'].progress_apply(lambda comment: sum(comment.count(w) for w in (':-<', ':()', ';-()', ';(')))\n    df[\"mean_word_len\"] = df[\"question_text\"].progress_apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n    # Number of F words - f..k contains folk, fork,\n    df[\"nb_fk\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"[Ff]\\S{2}[Kk]\", x))\n    # Number of S word\n    df[\"nb_sk\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"[Ss]\\S{2}[Kk]\", x))\n    # Number of D words\n    df[\"nb_dk\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"[dD]ick\", x))\n    # Number of occurence of You, insulting someone usually needs someone called : you\n    df[\"nb_you\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"\\W[Yy]ou\\W\", x))\n    # Just to check you really refered to my mother ;-)\n    df[\"nb_mother\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"\\W[Mm]other\\W\", x))\n    # Just checking for toxic 19th century vocabulary\n    df[\"nb_ng\"] = df[\"question_text\"].progress_apply(lambda x: count_regexp_occ(r\"\\W[Nn]igger\\W\", x))\n    df['sum_expletive'] = df[\"nb_fk\"] + df[\"nb_sk\"] + df[\"nb_dk\"] + df[\"nb_ng\"]\n    df['people_person'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('people', 'person')))\n    df['trump'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('trump','donald','president')))\n    df['women'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('woman','women', 'girl', 'girls','womens')))\n    df['men'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('man','men','mens')))\n    df['race'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('white','black','whites','blacks')))\n    df['china'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('china','chinese')))\n    df['india'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('india','indian','indians')))\n    df['country'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('country','countries')))\n    df['quora'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in (['quora'])))\n    df['america'] = df[\"lower_question_text\"].progress_apply(lambda comment: sum(comment.count(w) for w in ('america','american','americans')))\n    df['sum_feat'] = df[\"nb_you\"]+df[\"nb_mother\"]+df['sum_expletive']+df['people_person']+df['trump']+df['women']+df['men']+df['race']+df['china']+df['india']+df['country']+df['quora']+df['america']\n    df['why'] = df[\"lower_question_text\"].progress_apply(lambda comment: fword(comment,'why'))\n    df['howwhat'] = df[\"lower_question_text\"].progress_apply(lambda comment: fword(comment,'how')+ fword(comment,'what'))\n    df['doare'] = df[\"lower_question_text\"].progress_apply(lambda comment: fword(comment,'do')+ fword(comment,'are'))\n    return df\n\n# some experimentation with qids\ndef add_qid_features(train,test):\n    full=pd.concat([train[['qid','question_text']],test])\n    full['qid_base_ten'] = full['qid'].apply(lambda x : int(x, 16))\n    min_qid = full['qid_base_ten'].min()\n    max_qid = full['qid_base_ten'].max()\n    full['qid_base_ten_normalized'] = full['qid_base_ten'].apply(lambda x : (x - min_qid)/(max_qid-min_qid))\n    train = full[0:len(train)]['qid_base_ten_normalized']\n    test = full[len(train):]['qid_base_ten_normalized']\n    return train,test\n\ndef create_spacy_feats(x):\n    x = x.lower()\n    features = [0]*5\n    for i, k in enumerate(token_dict_manageable.keys()):\n        for word in token_dict_manageable[k]:\n            if word in x:\n                features[i]+=1\n    return features\n\ndef load_and_prec():\n    if debug:\n        train_df = pd.read_csv(\"../input/train.csv\")[:80000]\n        test_df = pd.read_csv(\"../input/test.csv\")[:20000]\n    else:\n        train_df = pd.read_csv(\"../input/train.csv\")\n        test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    ######TO REMOVE\n    #test_df = pd.concat([test_df]*7).reset_index()\n    \n    ###################### Add Features ###############################\n    #  https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb\n    '''\n    train = add_features(train_df)\n    test = add_features(test_df)\n    '''\n    #POS spacy features\n    train_df = parallelize_apply(train_df,create_spacy_feats,\"question_text\",4,token_dict_manageable.keys())\n    test_df = parallelize_apply(test_df,create_spacy_feats,\"question_text\",4,token_dict_manageable.keys())\n    \n    #qid \n    #train_qid,test_qid = add_qid_features(train_df,test_df)\n    \n    train = parallelize_dataframe(train_df, add_features)\n    test = parallelize_dataframe(test_df, add_features)\n    \n    #train = pd.concat([train,train_qid],axis=1)\n    #test = pd.concat([test,test_qid],axis=1)\n    \n    # lower\n    #train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    #test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n\n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    #train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    #test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    # Remove remaninig apostrophes\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.replace(\"'\",''))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x:  x.replace(\"'\",''))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n\n    '''\n    ###################### Add Features ###############################\n    #  https://github.com/wongchunghang/toxic-comment-challenge-lstm/blob/master/toxic_comment_9872_model.ipynb\n    train = add_features(train_df)\n    test = add_features(test_df)\n    '''\n    features = train[['total_length','capitals','words_vs_unique','sum_feat','why','howwhat','doare','GPE', 'PERSON', 'ORG', 'NORP', 'WORK_OF_ART']].fillna(0)\n    test_features = test[['total_length','capitals','words_vs_unique','sum_feat','why','howwhat','doare','GPE', 'PERSON', 'ORG', 'NORP', 'WORK_OF_ART']].fillna(0)\n    \n    # doing PCA to reduce network run times\n    ss = StandardScaler()\n    #pc = PCA(n_components=5)\n    ss.fit(np.vstack((features, test_features)))\n    features = ss.transform(features)\n    test_features = ss.transform(test_features)\n    #pc.fit(np.vstack((features, test_features)))\n    #features = pc.transform(features)\n    #test_features = pc.transform(test_features)\n    \n    ###########################################################################\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X)+list(test_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n#     # Splitting to training and a final test set    \n#     train_X, x_test_f, train_y, y_test_f = train_test_split(list(zip(train_X,features)), train_y, test_size=0.2, random_state=SEED)    \n#     train_X, features = zip(*train_X)\n#     x_test_f, features_t = zip(*x_test_f)    \n    \n    #shuffling the data\n \n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    features = features[trn_idx]\n    \n    return train_X, test_X, train_y, features, test_features, tokenizer.word_index\n#     return train_X, test_X, train_y, x_test_f,y_test_f,features, test_features, features_t, tokenizer.word_index\n#     return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c72fcddb4f680879e231c3dbfc0c71e27fc424c","scrolled":true,"trusted":true},"cell_type":"code","source":"start = time.time()\n# fill up the missing values\n# x_train, x_test, y_train, word_index = load_and_prec()\nx_train, x_test, y_train, features, test_features, word_index = load_and_prec() \n# x_train, x_test, y_train, x_test_f,y_test_f,features, test_features,features_t, word_index = load_and_prec() \nprint(time.time()-start)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5c51a8329d569d13b9f0369ebb98ca8e2e55440"},"cell_type":"markdown","source":"### Load Embeddings\n\nChanging method: first load index and then load embedding\nTwo embedding matrices have been used. Glove, and paragram. The mean of the two is used as the final embedding matrix"},{"metadata":{"_uuid":"6a5f4502324d369ff6faa3692accee4f8a233005","scrolled":true,"trusted":true},"cell_type":"code","source":"# missing entries in the embedding are set using np.random.normal so we have to seed here too\nseed_everything()\nif debug:\n    paragram_embeddings = np.random.randn(120000,304)\n    glove_embeddings = np.random.randn(120000,304)\n    if concat:\n        embedding_matrix = np.concatenate([glove_embeddings, paragram_embeddings], axis=1)\n        print(np.shape(embedding_matrix))\n        embedding_matrix = embedding_matrix[:,:embed_size]\n    else:\n        embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)        \nelse:\n    glove_embedding_index = load_glove_index()\n    # add_lower(glove_embedding_index,word_index.keys())\n    glove_embeddings = create_glove(word_index,glove_embedding_index)\n    del glove_embedding_index\n    \n    para_embedding_index = load_para_index()\n    paragram_embeddings = create_para(word_index,para_embedding_index)\n    del para_embedding_index\n    if concat:\n        embedding_matrix = np.concatenate([glove_embeddings, paragram_embeddings], axis=1)\n        print(np.shape(embedding_matrix))\n        embedding_matrix = embedding_matrix[:,:embed_size]\n    else:\n        embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa6a41607b804d76a2ddc530c912b5673bcd2423"},"cell_type":"code","source":"np.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8098bea0cee9117ff9dc4e11feba53e49b80cb55"},"cell_type":"markdown","source":"### Cyclic CLR\nCode taken from https://www.kaggle.com/dannykliu/lstm-with-attention-clr-in-pytorch"},{"metadata":{"_uuid":"9d531a7454923f90d0e7443b1ed1373d008c2e88","scrolled":true,"trusted":true},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0da30e2afce23b753796f3045b44ce91a07e4303"},"cell_type":"markdown","source":"### Pytorch Run Functions"},{"metadata":{"trusted":true,"_uuid":"6a5afb54f70a29808af19946ba08ef971d194e46"},"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self,dataset):\n        self.dataset = dataset\n    def __getitem__(self,index):\n        data,target = self.dataset[index]\n        return data,target,index\n    def __len__(self):\n        return len(self.dataset)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b711f30cbb4b0297fec816b8102c477d9a546c80","trusted":true},"cell_type":"code","source":"def pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):\n    seed_everything()\n    avg_losses_f = []\n    avg_val_losses_f = []\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(x_train)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(x_test)))\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        seed_everything(i*1000+i)\n        x_train = np.array(x_train)\n        y_train = np.array(y_train)\n        if feats:\n            features = np.array(features)\n        x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        if feats:\n            kfold_X_features = features[train_idx.astype(int)]\n            kfold_X_valid_features = features[valid_idx.astype(int)]\n        x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        \n        model = copy.deepcopy(model_obj)\n\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n        step_size = 5000\n        base_lr, max_lr = 0.0001, 0.003   \n        optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                                 lr=max_lr)\n        \n        ################################################################################################\n        scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n                   step_size=step_size, mode='exp_range',\n                   gamma=0.99994)\n        ###############################################################################################\n\n        train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))\n        valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))\n        \n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n        for epoch in range(n_epochs):\n            start_time = time.time()\n            model.train()\n\n            avg_loss = 0.  \n            for i, (x_batch, y_batch, index) in enumerate(train_loader):\n                if feats:       \n                    f = kfold_X_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n                \n            model.eval()\n            \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                \n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n        avg_losses_f.append(avg_loss)\n        avg_val_losses_f.append(avg_val_loss) \n        # predict all samples in the test set batch per batch\n        for i, (x_batch,) in enumerate(test_loader):\n            if feats:\n                f = test_features[i * batch_size:(i+1) * batch_size]\n                y_pred = model([x_batch,f]).detach()\n            else:\n                y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n\n    print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n    return train_preds, test_preds\n\n\ndef pytorch_model_run_full(x_train,y_train,features,x_test,model_obj, feats = False,clip=True):\n    seed_everything()\n    test_preds = np.zeros((len(x_test)))\n    x_train = np.array(x_train)\n    y_train = np.array(y_train)\n    if feats:\n        features = np.array(features)\n    x_train_fold = torch.tensor(x_train, dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train[:, np.newaxis], dtype=torch.float32).cuda()\n\n    if feats:\n        kfold_X_features = features\n    \n    model = model_obj\n    model.cuda()\n\n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n    step_size = 5000\n    base_lr, max_lr = 0.001, 0.003   \n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                             lr=max_lr)\n    \n    ################################################################################################\n    scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n               step_size=step_size, mode='exp_range',\n               gamma=0.99994)\n    ###############################################################################################\n\n    train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))\n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    \n\n    \n    for epoch in range(n_epochs):\n        # set train mode of the model. This enables operations which are only applied during training like dropout\n        start_time = time.time()\n        model.train()\n\n        avg_loss = 0.  \n        for i, (x_batch, y_batch, index)  in enumerate(train_loader):\n            # Forward pass: compute predicted y by passing x to the model.\n            ################################################################################################\n            if feats:       \n                f = kfold_X_features[index]\n                y_pred = model([x_batch,f])\n            else:\n                y_pred = model(x_batch)\n\n            if scheduler:\n                scheduler.batch_step()\n            \n            # Compute and print loss.\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            if clip:\n                nn.utils.clip_grad_norm_(model.parameters(),1)\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t  time={:.2f}s'.format(\n            epoch + 1, n_epochs, avg_loss, elapsed_time))\n\n    model.eval()\n    # predict all samples in the test set batch per batch\n    for i, (x_batch,) in enumerate(test_loader):\n        if feats:\n            f = test_features[i * batch_size:(i+1) * batch_size]\n            y_pred = model([x_batch,f]).detach()\n        else:\n            y_pred = model(x_batch).detach()\n\n        test_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n    return test_preds\n\ndef bestThresshold(y_train,train_preds):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01),disable=True):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n    return delta , tmp[2]\n# delta = bestThresshold(y_train,train_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"600e07a2d1ce3942d0eece0f660c6d3b3ef46c2a"},"cell_type":"code","source":"def scoring(y_true, y_proba, verbose=True):\n    from sklearn.metrics import roc_curve, precision_recall_curve, f1_score\n    from sklearn.model_selection import RepeatedStratifiedKFold\n\n    def threshold_search(y_true, y_proba):\n        precision , recall, thresholds = precision_recall_curve(y_true, y_proba)\n        thresholds = np.append(thresholds, 1.001) \n        F = 2 / (1/precision + 1/recall)\n        best_score = np.max(F)\n        best_th = thresholds[np.argmax(F)]\n        return best_th \n\n\n    rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10)\n\n    scores = []\n    ths = []\n    for train_index, test_index in rkf.split(y_true, y_true):\n        y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]\n        y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n\n        # determine best threshold on 'train' part \n        best_threshold = threshold_search(y_true_train, y_prob_train)\n\n        # use this threshold on 'test' part for score \n        sc = f1_score(y_true_test, (y_prob_test >= best_threshold).astype(int))\n        scores.append(sc)\n        ths.append(best_threshold)\n\n    best_th = np.mean(ths)\n    score = np.mean(scores)\n\n    if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')\n\n    return best_th, score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"548b5d7402220304f65441945e21827ee6827cec"},"cell_type":"code","source":"def boosted_pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):\n    seed_everything()\n    avg_losses_f = []\n    avg_val_losses_f = []\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(x_train)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(x_test)))\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        x_train = np.array(x_train)\n        y_train = np.array(y_train)\n        if feats:\n            features = np.array(features)\n        x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        if feats:\n            kfold_X_features = features[train_idx.astype(int)]\n            kfold_X_valid_features = features[valid_idx.astype(int)]\n        x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        \n        model = copy.deepcopy(model_obj)\n\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n        step_size = 5000\n        base_lr, max_lr = 0.001, 0.003   \n        optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                                 lr=max_lr)\n        \n        ################################################################################################\n        scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n                   step_size=step_size, mode='exp_range',\n                   gamma=0.99994)\n        ###############################################################################################\n\n        train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))\n        valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))\n        \n        # did shuffle ==false to handle boost\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        # store train predictions\n        train_preds_fold = np.zeros((x_train_fold.size(0)))\n        print(f'Fold {i + 1}')\n        for epoch in range(n_epochs):\n            start_time = time.time()\n            model.train()\n\n            avg_loss = 0.  \n            for i, (x_batch, y_batch, index) in enumerate(train_loader):\n                if feats:       \n                    f = kfold_X_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n                # store train predictions\n                train_preds_fold[index] = sigmoid(y_pred.detach().cpu().numpy())[:, 0]\n                \n            model.eval()\n            \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                # print('ybatch' , y_batch.size())\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n        \n        # need to train model some more on bad predictions. Here is where I am boosting this model\n        train_preds_fold = (train_preds_fold > 0.33).astype(int)\n        print(len(train_preds_fold))\n        # need to get index where true labels is 1 and pred label is false \n        boosting_index = []\n        for i in range(len(y_train_fold)):\n            if train_preds_fold[i] != y_train_fold[i] and train_preds_fold[i] == 1:\n                boosting_index.append(i)\n        boosting_index=np.array(boosting_index)\n        boost_x_fold = torch.tensor(x_train_fold.cpu().numpy()[boosting_index.astype(int)], dtype=torch.long).cuda()\n        boost_y_fold = torch.tensor(y_train_fold.cpu().numpy()[boosting_index.astype(int)], dtype=torch.float32).cuda()\n        kfold_boost_features = kfold_X_features[boosting_index.astype(int)]\n        print(boost_x_fold.size())\n        boost = MyDataset(torch.utils.data.TensorDataset(boost_x_fold, boost_y_fold))\n        boost_loader = torch.utils.data.DataLoader(boost, batch_size=batch_size, shuffle=False)\n        # BOOSTING:\n        for epoch in range(2):\n            start_time = time.time()\n            model.train()\n            for i, (x_batch, y_batch,index) in enumerate(boost_loader):\n                if feats:       \n                    f = kfold_boost_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n                # print('ybatch' , y_batch.size())\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n                \n                \n            model.eval()\n            \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                \n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n\n\n        avg_losses_f.append(avg_loss)\n        avg_val_losses_f.append(avg_val_loss) \n        # predict all samples in the test set batch per batch\n        for i, (x_batch,) in enumerate(test_loader):\n            if feats:\n                f = test_features[i * batch_size:(i+1) * batch_size]\n                y_pred = model([x_batch,f]).detach()\n            else:\n                y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n\n    print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c4cf6c35c0b0712e210e5b92ac1f31f0bd017f4"},"cell_type":"code","source":"'''\nIdea for Pseudo-Labelling\n1. Run n-1 epochs on the train fold\n2. Predict whole test set\n3. Concatenated data = concat train fold with test set\n4. Run 1 epoch on the concatenated data\n5. get OOF train preds\n6. get test preds\n'''\ndef pl_pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True,pl_mode=1):\n    seed_everything()\n    avg_losses_f = []\n    avg_val_losses_f = []\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(x_train)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(x_test)))\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\n    for foldnum, (train_idx, valid_idx) in enumerate(splits):\n        x_train = np.array(x_train)\n        y_train = np.array(y_train)\n        if feats:\n            features = np.array(features)\n        x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        if feats:\n            kfold_X_features = features[train_idx.astype(int)]\n            kfold_X_valid_features = features[valid_idx.astype(int)]\n        x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        \n        model = copy.deepcopy(model_obj)\n\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n        step_size = 300\n        base_lr, max_lr = 0.001, 0.003   \n        optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                                 lr=max_lr)\n        \n        ################################################################################################\n        scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n                   step_size=step_size, mode='exp_range',\n                   gamma=0.99994)\n        ###############################################################################################\n\n        train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))\n        valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))\n        \n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {foldnum + 1}')\n        for epoch in range(n_epochs-1):\n            start_time = time.time()\n            model.train()\n\n            avg_loss = 0.  \n            for i, (x_batch, y_batch, index) in enumerate(train_loader):\n                if feats:       \n                    f = kfold_X_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n                \n            model.eval()\n            \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                \n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n\n\n        # PL START\n        avg_losses_f.append(avg_loss)\n        avg_val_losses_f.append(avg_val_loss) \n        # predict all samples in the test set batch per batch\n        model.eval()\n        for i, (x_batch,) in enumerate(test_loader):\n            if feats:\n                f = test_features[i * batch_size:(i+1) * batch_size]\n                y_pred = model([x_batch,f]).detach()\n            else:\n                y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        \n        if pl_mode==1: # pseudo = train+test\n            num_pl_epoch = 1\n            pl_y = torch.cat((torch.tensor(test_preds_fold[:, np.newaxis], dtype=torch.float32).cuda() ,y_train_fold),0)\n            pl_X = torch.cat((x_test_cuda,x_train_fold),0)\n            if feats:\n                pl_features = np.concatenate((test_features,kfold_X_features),axis=0)\n            \n                \n        if pl_mode==2: # pseudo = test only\n            num_pl_epoch = 1\n            pl_y = torch.tensor(test_preds_fold[:, np.newaxis], dtype=torch.float32).cuda()\n            pl_X = x_test_cuda                \n            if feats:\n                pl_features = test_features\n\n        if pl_mode==3: # pseudo = train[sample same number of test samples] + test\n            num_pl_epoch = 1\n            num_test_examples = len(test_preds_fold)\n            np.random.seed(foldnum)\n            print(x_train_fold.size()[0])\n            np_indices = np.random.permutation(x_train_fold.size()[0])[:num_test_examples]\n            indices = torch.tensor(np_indices).cuda()\n            pl_y = torch.cat((torch.tensor(test_preds_fold[:, np.newaxis], dtype=torch.float32).cuda() ,torch.index_select(y_train_fold, 0, indices)),0)\n            pl_X = torch.cat((x_test_cuda,torch.index_select(x_train_fold, 0, indices)),0)\n            print(pl_y.size())\n            print(pl_X.size())\n            if feats:\n                pl_features = np.concatenate((test_features,kfold_X_features[np_indices]),axis=0)\n        \n        if pl_mode ==4: # Rajesh's idea for PL \n            num_pl_epoch = 2\n            thresh = 0.1\n            num_test_examples = len(test_preds_fold)\n            #print(np.argwhere(test_preds_fold>=0.9))\n            np_indices = np.reshape(np.argwhere((test_preds_fold<=thresh) |( test_preds_fold>=1-thresh )),-1)\n            test_preds_fold = np.round_(test_preds_fold[np_indices])\n            #print(np.sum(test_preds_fold))\n            indices = torch.tensor(np_indices).cuda()\n            pl_y = torch.tensor(test_preds_fold[:, np.newaxis], dtype=torch.float32).cuda()\n            pl_X = torch.index_select(x_test_cuda, 0 ,  indices)      \n            if feats:\n                pl_features = test_features[np_indices]\n        \n        if pl_mode ==5: # Rajesh's idea for PL + train\n            num_pl_epoch = 1\n            thresh = 0.2\n            num_test_examples = len(test_preds_fold)\n            # print(np.argwhere(test_preds_fold>=0.9))\n            np_indices = np.reshape(np.argwhere((test_preds_fold<=thresh) |( test_preds_fold>=1-thresh )),-1)\n            test_preds_fold = np.round_(test_preds_fold[np_indices])\n            # print(np.sum(test_preds_fold))\n            indices = torch.tensor(np_indices).cuda()\n            pl_y = torch.cat((torch.tensor(test_preds_fold[:, np.newaxis], dtype=torch.float32).cuda(),y_train_fold),0)\n            pl_X = torch.cat((torch.index_select(x_test_cuda, 0 ,  indices),x_train_fold),0)\n            if feats:\n                pl_features = np.concatenate((test_features[np_indices],kfold_X_features),axis=0)\n            print(\"count_0:\",len(test_preds_fold)- np.sum(test_preds_fold))\n            print(\"count_1:\",np.sum(test_preds_fold))\n                \n        print(\"pl_y size:\",pl_y.size())\n        pl = MyDataset(torch.utils.data.TensorDataset(pl_X, pl_y))\n        pl_loader = torch.utils.data.DataLoader(pl, batch_size=batch_size, shuffle=True)\n\n        #train model on pl fold\n        for pl_epoch in range(num_pl_epoch):\n            # set model to train mode\n            model.train()\n            avg_loss = 0.\n            for i, (x_batch, y_batch, index) in enumerate(pl_loader):\n                if feats:       \n                    f = pl_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n                \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                \n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n            elapsed_time = time.time() - start_time \n            print('PL Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                pl_epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n        # PL COMPLETE\n        avg_losses_f.append(avg_loss)\n        avg_val_losses_f.append(avg_val_loss) \n        # predict all samples in the test set batch per batch\n        for i, (x_batch,) in enumerate(test_loader):\n            if feats:\n                f = test_features[i * batch_size:(i+1) * batch_size]\n                y_pred = model([x_batch,f]).detach()\n            else:\n                y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n\n    print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n    return train_preds, test_preds\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ec1e88642878c7e0a59a377c144fe6cf666fb80"},"cell_type":"code","source":"def ensemble_pytorch_model_run_cv(x_train,y_train,features,x_test, model_obj, feats = False,clip = True):\n    seed_everything()\n    avg_losses_f = []\n    avg_val_losses_f = []\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((n_epochs,len(x_train)))\n    # matrix for the predictions on the test set\n\n    test_preds = []\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        seed_everything(i*1000+i)\n        x_train = np.array(x_train)\n        y_train = np.array(y_train)\n        if feats:\n            features = np.array(features)\n        x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        if feats:\n            kfold_X_features = features[train_idx.astype(int)]\n            kfold_X_valid_features = features[valid_idx.astype(int)]\n        x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n        \n        model = copy.deepcopy(model_obj)\n\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n\n        step_size = 5000\n        base_lr, max_lr = 0.0001, 0.003   \n        optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                                 lr=max_lr)\n        \n        ################################################################################################\n        scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n                   step_size=step_size, mode='exp_range',\n                   gamma=0.99994)\n        ###############################################################################################\n\n        train = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))\n        valid = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))\n        \n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        test_preds_fold = np.zeros((n_epochs,len(x_test)))\n        print(f'Fold {i + 1}')\n        for epoch in range(n_epochs):\n            start_time = time.time()\n            model.train()\n\n            avg_loss = 0.  \n            for i, (x_batch, y_batch, index) in enumerate(train_loader):\n                if feats:       \n                    f = kfold_X_features[index]\n                    y_pred = model([x_batch,f])\n                else:\n                    y_pred = model(x_batch)\n\n                if scheduler:\n                    scheduler.batch_step()\n\n                # Compute and print loss.\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                if clip:\n                    nn.utils.clip_grad_norm_(model.parameters(),1)\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n                \n            model.eval()\n            \n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            #test_preds_fold = np.zeros((len(x_test)))\n            \n            avg_val_loss = 0.\n            for i, (x_batch, y_batch,index) in enumerate(valid_loader):\n                if feats:\n                    f = kfold_X_valid_features[index]            \n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n                \n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]\n                train_preds[epoch,valid_idx] = valid_preds_fold\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n        #avg_losses_f.append(avg_loss)\n        #avg_val_losses_f.append(avg_val_loss) \n        # predict all samples in the test set batch per batch\n            for i, (x_batch,) in enumerate(test_loader):\n                if feats:\n                    f = test_features[i * batch_size:(i+1) * batch_size]\n                    y_pred = model([x_batch,f]).detach()\n                else:\n                    y_pred = model(x_batch).detach()\n\n                test_preds_fold[epoch,i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            \n        #train_preds[valid_idx] = valid_preds_fold\n        test_preds.append(test_preds_fold)\n\n        avg_losses_f.append(avg_loss)\n        avg_val_losses_f.append(avg_val_loss) \n    print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n    return train_preds, test_preds\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ab5fb58ed2eb139589f158352f842a2cb577374"},"cell_type":"markdown","source":"### Layers Different"},{"metadata":{"_uuid":"a51786e25111195c779cb9ae7a451d5419460d84","trusted":true},"cell_type":"code","source":"# These from BiLSTM\n## Start\nT_epsilon = 1e-7\n\nclass Embed_Layer(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n        super(Embed_Layer, self).__init__()\n        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n        if use_pretrained_embedding:\n            # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n\n    def forward(self, x, dropout_p=0.25):\n        return nn.Dropout(p=dropout_p)(self.encoder(x))\n\n\nclass GRU_Layer(nn.Module):\n    def __init__(self):\n        super(GRU_Layer, self).__init__()\n        self.gru = nn.GRU(input_size=300,\n                          hidden_size=gru_len,\n                          bidirectional=True)\n        '''\n        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n       '''\n        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n        #                           hidden_size=gru_len,\n        #                           bidirectional=True)\n\n    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n    def init_weights(self):\n        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n        for k in ih:\n            nn.init.kaiming_uniform_(k)\n        for k in hh:\n            nn.init.orthogonal_(k)\n        for k in b:\n            nn.init.constant_(k, 0)\n\n    def forward(self, x):\n        return self.gru(x)\n\n\n# core caps_layer with squash func\nclass Caps_Layer(nn.Module):\n    def __init__(self, input_dim_capsule, num_capsule, dim_capsule, \n                 routings, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Caps_Layer, self).__init__(**kwargs)\n\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size  # 暂时没用到\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = self.squash\n        else:\n            self.activation = nn.ReLU(inplace=True)\n\n        if self.share_weights:\n            self.W = nn.Parameter(\n                nn.init.kaiming_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n        else:\n            self.W = nn.Parameter(\n                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n    def forward(self, x):\n\n        if self.share_weights:\n            u_hat_vecs = t.matmul(x, self.W)\n        else:\n            print('add later')\n\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n        b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n        for i in range(self.routings):\n            b = b.permute(0, 2, 1)\n            c = F.softmax(b, dim=2)\n            c = c.permute(0, 2, 1)\n            b = b.permute(0, 2, 1)\n            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n            # outputs shape (batch_size, num_capsule, dim_capsule)\n            if i < self.routings - 1:\n                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    # text version of squash, slight different from original one\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = t.sqrt(s_squared_norm + T_epsilon)\n        return x / scale\n    \nclass Capsule_Main(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None):\n        super(Capsule_Main, self).__init__()\n        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n        self.gru_layer = GRU_Layer()\n        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n        self.gru_layer.init_weights()\n        self.caps_layer = Caps_Layer()\n        self.dense_layer = Dense_Layer()\n\n    def forward(self, content):\n        content1 = self.embed_layer(content)\n        content2, _ = self.gru_layer(\n            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n        content3 = self.caps_layer(content2)\n        output = self.dense_layer(content3)\n        return output\n    \nclass Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.kaiming_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n\n## End","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8caecb207a12d4a5524fe16e4524b31c7da8bac"},"cell_type":"markdown","source":"### Models\n\nBinary LSTM with an attention layer and an additional fully connected layer. Also added extra features taken from a winning kernel of the toxic comments competition. Also using CLR and a capsule Layer. Blended together in concatentation.\n\nInitial idea borrowed from: https://www.kaggle.com/ziliwang/baseline-pytorch-bilstm"},{"metadata":{"_kg_hide-input":false,"_uuid":"26e90f1839b084c126414250eac1f636b0c88937","trusted":true},"cell_type":"code","source":"# create Models:\n\nclass BiLSTMNeuralNet_Meta(nn.Module):\n    def __init__(self,hidden_size,fc_layer,embedding_matrix=embedding_matrix):\n        super(BiLSTMNeuralNet_Meta, self).__init__()\n        gru_len = hidden_size\n        Routings = 4 #5\n        Num_capsule = 5\n        Dim_capsule = 5#16\n        dropout_p = 0.25\n        rate_drop_dense = 0.28\n        LR = 0.001\n        \n        # num_classes = 30\n        fc_layer1 = fc_layer\n\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n\n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        \n        self.linear = nn.Linear(hidden_size*8+1+features.shape[1], fc_layer1) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.fc = nn.Linear(fc_layer**2,fc_layer)\n        self.out = nn.Linear(fc_layer, 1)\n        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n        self.caps_layer = Caps_Layer(input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \n                 routings=Routings, kernel_size=(9, 1), share_weights=True)\n    \n    def forward(self, x):\n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        ##Capsule Layer        \n        content3 = self.caps_layer(h_gru)\n        content3 = self.dropout(content3)\n        batch_size = content3.size(0)\n        content3 = content3.view(batch_size, -1)\n        content3 = self.relu(self.lincaps(content3))\n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n\n                #[512,160]\n        conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n\n        out = self.out(conc)\n        #print(out.size())\n        return out\n\n\nclass GRU_Att_NeuralNet_Meta(nn.Module):\n    def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):\n        super(GRU_Att_NeuralNet_Meta, self).__init__()\n        drp = 0.1\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n\n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.lstm.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.gru.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n\n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n\n        self.linear = nn.Linear(hidden_size*8 +features.shape[1], lin_size)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(drp)\n        self.out = nn.Linear(lin_size, 1)\n\n    def forward(self, x):\n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n\n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n\n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool,f), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out\n    \n\nclass Alex_NeuralNet_Meta(nn.Module):\n    def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):\n        super(Alex_NeuralNet_Meta, self).__init__()\n        self.hidden_size = hidden_size\n        drp = 0.1\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n\n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.lstm.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n\n        for name, param in self.gru.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n\n        self.linear = nn.Linear(hidden_size*6 + features.shape[1], lin_size)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(drp)\n        self.out = nn.Linear(lin_size, 1)\n\n    def forward(self, x):\n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        #print(\"emb\", h_embedding.size())\n        h_lstm, _ = self.lstm(h_embedding)\n        #print(\"lst\",h_lstm.size())\n        h_gru, hh_gru = self.gru(h_lstm)\n        hh_gru = hh_gru.view(-1, 2*self.hidden_size )\n        #print(\"gru\", h_gru.size())\n        #print(\"h_gru\", hh_gru.size())\n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        #print(\"avg_pool\", avg_pool.size())\n        #print(\"max_pool\", max_pool.size())\n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n        #print(\"f\", f.size())\n        conc = torch.cat(( hh_gru, avg_pool, max_pool,f), 1)\n        #print(\"conc\", conc.size())\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out\n    \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed4fa0b55b4fd4a84b4078f12a2af99f04f8e5fa","trusted":true},"cell_type":"code","source":"class cnn2d_NeuralNet(nn.Module):\n    def __init__(self,hidden_size,lin_size, embedding_matrix=embedding_matrix):\n        super(cnn2d_NeuralNet, self).__init__()\n        \n        fc_layer = lin_size\n        fc_layer1 = lin_size\n        nbr_filers_1cnn = 5\n        nbr_filers_2cnn = 10\n\n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        \n        # CNN2D\n        self.CNN2D_1 = nn.Conv2d(\n            in_channels = 1,\n            out_channels = nbr_filers_1cnn, #number of filters\n            kernel_size = (12,10),\n            stride = (2,4),\n            padding = (3,5)\n            )#22,21\n        self.BN_1 = nn.BatchNorm2d(nbr_filers_1cnn)\n        \n        self.MaxPool2D_1 = nn.MaxPool2d(kernel_size=(3,3), stride = (3,3)) ##10x9\n        \n        self.CNN2D_2 = nn.Conv2d(\n            in_channels = nbr_filers_1cnn,\n            out_channels = nbr_filers_2cnn, #number of filters\n            kernel_size = (5,5),\n            stride = (2,2),\n            padding = (1,1),\n            )#22,21\n        self.BN_2 = nn.BatchNorm2d(nbr_filers_2cnn)\n        self.MaxPool2D_2 = nn.MaxPool2d(kernel_size=(5,5)) ##10x9  \n        \n        #self.linCNN = nn.Linear(10, 1)\n\n        #Dense\n        self.bn = nn.BatchNorm1d(14, momentum=0.5)\n        #+3\n        self.linear = nn.Linear(hidden_size*8+10+features.shape[1], fc_layer1) #643:80 - 483:60 - 323:40\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.fc = nn.Linear(fc_layer**2,fc_layer)\n        self.out = nn.Linear(fc_layer, 1)\n\n    \n    def forward(self, x):\n        \n        h_embedding = self.embedding(x[0])\n        h_embedding = torch.squeeze(\n            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n\n        ## CNN2d ##\n        x_h_gru = h_gru.unsqueeze(1)\n\n        cnn_1 = self.CNN2D_1(x_h_gru)\n        cnn_1 = self.BN_1(cnn_1)\n        cnn_1 = self.relu(cnn_1)\n        max_pool_1 = self.MaxPool2D_1(cnn_1)\n        cnn_2 = self.CNN2D_2(max_pool_1)\n        #print(cnn_2.shape)\n        cnn_2 = self.BN_2(cnn_2)\n        cnn_2 = self.relu(cnn_2)\n        max_pool_2 = self.MaxPool2D_2(cnn_2)\n        #print(max_pool_2.shape)       \n        cnn_out = max_pool_2.reshape(max_pool_2.size(0),-1)\n        \n\n        ##Attention Layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        f = torch.tensor(x[1], dtype=torch.float).cuda()\n\n        conc = torch.cat((h_lstm_atten, h_gru_atten,cnn_out, avg_pool, max_pool,f), 1)\n        conc = self.relu(self.linear(conc))\n        #print(conc.size())\n        conc = self.bn(conc)\n        conc = self.dropout(conc)\n\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"551ff8837a1b0899e0925c1a3801547adeddccae","trusted":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\n# always call this before training for deterministic results\nseed_everything()\n\nx_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b8f851d15f3e34d40eab4146393c145886b3966","trusted":true},"cell_type":"code","source":"test_preds_1 = pytorch_model_run_full(x_train,y_train,features,x_test,GRU_Att_NeuralNet_Meta(80,16, embedding_matrix=embedding_matrix), feats = True)\ntest_preds_2 = pytorch_model_run_full(x_train,y_train,features,x_test,cnn2d_NeuralNet(60,14, embedding_matrix=glove_embeddings), feats = True)\ntest_preds_3 = pytorch_model_run_full(x_train,y_train,features,x_test,Alex_NeuralNet_Meta(62,28, embedding_matrix=paragram_embeddings), feats = True)\n#test_preds_4 = pytorch_model_run_full(x_train,y_train,features,x_test,BiLSTMNeuralNet_Meta(59,22, embedding_matrix=embedding_matrix), feats = True)\ntest_preds_5 = pytorch_model_run_full(x_train,y_train,features,x_test,GRU_Att_NeuralNet_Meta(80,16, embedding_matrix=glove_embeddings), feats = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08f68afc30978b7a86c2567170ec535bfa83d18e"},"cell_type":"code","source":"# best threshold is 0.3300 with F1 score: 0.6997\n\ntest_preds = [test_preds_1,test_preds_2,test_preds_3,test_preds_5]\nmeta_preds = pd.DataFrame(np.array(test_preds).T)\nweights = np.array([0.25315958, 0.24013115, 0.2043472 , 0.29492363])\ntest_preds = np.dot(np.array(meta_preds),weights)\nbest_thr = 0.32","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8bf091cac7ce485afc35bdcc50d3a9ea59f8e2a","trusted":true},"cell_type":"code","source":"if debug:\n    df_test = pd.read_csv(\"../input/test.csv\")[:20000]\nelse:\n    df_test = pd.read_csv(\"../input/test.csv\")\nsubmission = df_test[['qid']].copy()\nsubmission['prediction'] = (test_preds > best_thr).astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13dea8e1c900175db765f3fac41bcaec035d4476","trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f63cda14397adf5dc91a12363c1092a6f3c93543","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d1b3fc0c8bc59f91a203adcf3dbd9d89759c8df","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc3e72f43d1ddba1a9d82be0cf9b642f6d5795d6","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10c39195707e4710e11d31bafd4bde1e21ff2d6f","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"162772df446ca6ad6dda6a003d8ea2e58106192f","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdca46457c79c6b56409e6cfbc76d90de4d19cbe","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"717f6f2fbaa37d01db018baafcf8216fee98ce6a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a713658e272a3d7ec00267faa5e3d02362269ed","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ce9e3159ce70cdaa0d706249cf80ea3157b39d0","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fb26d857e36473fe7a458cf61692ceff5cadceb","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dee2aefa02222d4dd5e67640708cad59285a796","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8682fc20ab59210e2aba1381232e68ce4e9519d2","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2681fc633782d5a7005a0db95de14c6b05b88dac","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39e764384d4073282b6a51bcd1ecf382134a33a6","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3703e549d2d8d5bb0948c2771b1e6636f0fccef2","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}