{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import operator\nfrom sklearn.metrics import f1_score\nfrom sklearn.linear_model import LogisticRegressionCV\nimport string\nimport numpy as np\nimport pandas as pd \nfrom sklearn.feature_extraction.text import CountVectorizer\nimport os\nprint(os.listdir(\"../input\"))\nfrom sklearn.model_selection import train_test_split\nimport re\nimport time\nimport gc\nimport random\nimport os\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\ndef warn(*args, **kwargs):\n    pass\nimport warnings\nwarnings.warn = warn","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f000201adfcfe81fbaa9f0ac3bfaf97f44834477"},"cell_type":"markdown","source":"# Analysis of the dataset"},{"metadata":{"trusted":true,"_uuid":"cc5a0c98c754e0af8bb729a6e6a312bdc0fe6579"},"cell_type":"code","source":"data = pd.read_csv(\"../input/train.csv\",header=None,low_memory=False)\ndata_test = pd.read_csv(\"../input/test.csv\",low_memory=False)\n\nsentences = data[1][1:]\nlabels = data[2][1:]\nsentences_test = []\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf35a281eb78f401f3c4219e906d352d9d8d0251"},"cell_type":"markdown","source":"There are 1.2 million sincere questions and 80,000 insincere questions in the training data:"},{"metadata":{"trusted":true,"_uuid":"3876aac5b663564aa2fda66163c5dc48f262e773"},"cell_type":"code","source":"labels.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef88a7296912a460ad8f2754e80a73571d8830b8"},"cell_type":"code","source":"sentences.apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log')\nplt.title('Distribution of question text length in words')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1427990de69ad757fb15ad495d94fae4d8e2a564"},"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(sentences.apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(sentences.apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6e90b1a3fa9282699ba6c5410268b6265ee95d0"},"cell_type":"markdown","source":"As shown in the histogram above, there is only one (1!) sentence with over 120 words. The bulk of sentences are less than 40 words long."},{"metadata":{"trusted":true,"_uuid":"734dcf671fecbfdeb9a01d03eac029dc0db24886"},"cell_type":"code","source":"vec = CountVectorizer().fit(sentences)\nbag_of_words = vec.transform(sentences)\nsum_words = bag_of_words.sum(axis=0)\n\nwords_freq = [(word, sum_words[0, idx]) for word, idx in     vec.vocabulary_.items()]\nwords_freq = sorted(words_freq, key = lambda x: x[1], reverse=True)\nwords_freq_dict = dict(words_freq)\nword_numbers = list(words_freq_dict.values())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aed813d2e3e97abe73580a73c39f9b71ffce683a"},"cell_type":"markdown","source":"The total number of words in our whole corpus and the number of unique words:"},{"metadata":{"trusted":true,"_uuid":"3111ddb0804cae2f8ac47e8455d6b2aa78ba177c"},"cell_type":"code","source":"print(np.sum(np.array(word_numbers)))\nprint(len(word_numbers))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d05158405b4cd154cbc62342fe607bb0e2710ff5"},"cell_type":"code","source":"sorted(word_numbers, reverse=True)[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e58afaf7856b3b51493c65b5b7ac4ebd3c24f54"},"cell_type":"markdown","source":"Distribution over the number as a function of their frequency (number of appearances in the corpus)"},{"metadata":{"trusted":true,"_uuid":"ce1629e6c8abcabd9eb7b3778fe9a79321e40f9f"},"cell_type":"code","source":"plt.hist(np.log10(word_numbers), bins=6)\nplt.title(\"Log-log distribution of word frequencies\")\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0d5a0e7dd1a2f432e80576c16f84b454208ccad"},"cell_type":"markdown","source":"From the plot, there are only about 10 words that appear 10^6 times, and over 10^5 words that appear less than 10 times. Most words occur infrequently."},{"metadata":{"_uuid":"5c9e38f369c64142042cc1b15b3b7ae52a4040a3"},"cell_type":"markdown","source":"Most frequent words:"},{"metadata":{"trusted":true,"_uuid":"8fe77de838fe3e1ce287c68fb9e4ee497e83e7be"},"cell_type":"code","source":"words_freq[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f66c5dd21a9e61847431f879ee82fc3167c7055e"},"cell_type":"markdown","source":"Checking the embeddings"},{"metadata":{"trusted":true,"_uuid":"0b55c0d694ddcdd88b9704f69f090238fb652d0e"},"cell_type":"code","source":"def build_vocab(sentences, verbose =  True):\n    \"\"\"\n    :param sentences: list of list of words\n    :return: dictionary of words and their count\n    \"\"\"\n    vocab = {}\n    for sentence in tqdm(sentences, disable = (not verbose)):\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"673a4df6c5d4e90fac3401896f0204a330698fde"},"cell_type":"code","source":"sentences = sentences.apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\nprint({k: vocab[k] for k in list(vocab)[:5]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5acb19d95da5c3d5986e5e9bc79335ede3998076"},"cell_type":"code","source":"def check_vocab_glove(corpus_vocab):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, 1\n    \n    embedding_vocab = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    oov = {}\n    inv = {}\n    \n    for word, val in tqdm(corpus_vocab.items()):\n        if word in embedding_vocab:\n            inv[word] = val\n        else:\n            oov[word] = val\n    return oov, inv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"142ff7777c9ba975dbca8275041ad4441192790f"},"cell_type":"code","source":"oov, inv = check_vocab_glove(vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fab295676054cb54ddb39552630f2627f2510f24"},"cell_type":"code","source":"oov_words = list(oov.values())\ninv_words = list(inv.values())\nunique_inv = len(inv)\nunique_oov = len(oov)\ntotal_inv = np.sum(np.array(inv_words))\ntotal_oov = np.sum(np.array(oov_words))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a3cc7fdceac80b2568e67be4512f5064d0597e2"},"cell_type":"markdown","source":"If we compare the percentages of unique words found in the vocab vs. the total number of words found in the vocab, we can see that although coverage for unique words is poor (33%), the coverage of the corpus as a whole is much better (88%) since the oov words occur much less frequently:\n"},{"metadata":{"trusted":true,"_uuid":"dccf1769336288bf19f56a6771ddff2fecf1c230"},"cell_type":"code","source":"print(unique_inv/(unique_inv+unique_oov))\nprint(total_inv/(total_inv+total_oov))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1187255a285110766b4b6ee2da58a779c0a28b2a"},"cell_type":"markdown","source":"However, the most frequently occuring OOV word is still occurring a lot..."},{"metadata":{"trusted":true,"_uuid":"0c089597e3fc09bba7ddcf5ffb50f183eeed7ea0"},"cell_type":"code","source":"np.max(np.array(oov_words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fce279a0567a599b73be4ceb1fc24555d69dad76"},"cell_type":"code","source":"#oov_words_freq =sorted(oov_, key = lambda x: x[1], reverse=True)\noov_words_freq = sorted(oov.items(), key=operator.itemgetter(1),reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcd34aef0185b1a2cbe2be52cbde2ab857e8507c"},"cell_type":"markdown","source":"What could it be? It's \"India?\"?"},{"metadata":{"trusted":true,"_uuid":"6eab3a49d4a68cb6af012d08b2c431b17b546632"},"cell_type":"code","source":"oov_words_freq[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3127fbde9265f66095ccb0dc426622e0b5d904e1"},"cell_type":"markdown","source":"A lot of words are \"out of vocab\" (OOV) when they are actually in it! They just have punctuation appended to the end. `clean_text` in following \"data preprocessing\" section takes care of that."},{"metadata":{"_uuid":"de3974782097f4abd4027e87f96033ad5a8effaf"},"cell_type":"markdown","source":"# Data preprocessing"},{"metadata":{"trusted":true,"_uuid":"8d79a43b7b5f344cc8327c671344d7bed1b46a45"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    \"\"\"Replace commonly misspelt words or contractions (e.g. can't -> cannot)\"\"\"\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"180a23e279fe494b4d7e1edf9cc8236aeaab5a35"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n    \n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n    \n    \n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n    #shuffling the data\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, tokenizer.word_index, train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b28bfe330b33eeb320703b16eb53469861dd457"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d4f46f09306f4e4b78b8d247c772ba7e42abbda"},"cell_type":"code","source":"maxlen = 72 # max number of words in a question to use\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\n\nstart_time = time.time()\n\ntrain_X, test_X, train_y, word_index, train_df = load_and_prec()\nembedding_matrix = load_glove(word_index)\n\ntotal_time = (time.time() - start_time) / 60\nprint(\"Took {:.2f} minutes\".format(total_time))\n\nprint(np.shape(embedding_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c7cbac7b097548d1bdd8da9a008b95137e24745"},"cell_type":"code","source":"finalword_index = word_index.items()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d3447d18f349f81306891fe721a09f0b8bd9579"},"cell_type":"markdown","source":"Let's see now how many words are in the vocabulary."},{"metadata":{"trusted":true,"_uuid":"f47fb3a530d2f422820007144d174a8177d5f9e2"},"cell_type":"code","source":"i = 0\ndef vocab_after_preprocess(train_df):\n\n    inv = {}\n    oov = {}\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, 1\n    i = 0\n    embedding_vocab = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    for sentence in train_df[\"question_text\"]:\n        i += 1\n        words = sentence.split()\n        for word in words:\n            if word in embedding_vocab:\n                if word in inv:\n                    inv[word] +=1\n                else:\n                    inv[word]=1\n            else:\n                if word in oov:\n                    oov[word] += 1\n                else:\n                    oov[word] = 1\n    return inv, oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"974a35197fe843c5a138a40caa772ce246bb1117"},"cell_type":"code","source":"inv_after, oov_after = vocab_after_preprocess(train_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"623a433c6cbf6c1df42cf766e6fd921b01e11541"},"cell_type":"markdown","source":"The preprocessing worked! Who would have thought? (Probably the original author of this function thought so.)"},{"metadata":{"trusted":true,"_uuid":"f804cff25fa324a79c95d45af58847270894065e"},"cell_type":"code","source":"print(len(inv_after)/(len(inv_after)+len(oov_after)))\nprint(sum(inv_after.values())/(sum(inv_after.values())+sum(oov_after.values())))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3829220d633a6c7f5a643ffa914909a25cc7d06"},"cell_type":"markdown","source":"Now the vocabulary coverage for unique words is 62% compared to the previous 33%, and for all words it's 99% instead of 88%."},{"metadata":{"trusted":true,"_uuid":"e8b4402a9cee538f78e82a2b3a64ae1e7e7ce058"},"cell_type":"code","source":"sorted_dict = sorted(oov_after.items(), key=lambda x: x[1], reverse=True)\nsorted_dict[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28dd61105b19497fd8429af4c4481cad104c7f97"},"cell_type":"markdown","source":"Some of the most frequently-occuring OOV words are actually numbers that were preprocessed into `#`'s. Others include names and proper nouns like \"Brexit\" and \"Redmi\"."},{"metadata":{"_uuid":"ac4e98fc1eea2a6c16ea763f826cb5b700bb0c00"},"cell_type":"markdown","source":"# Simple Logistic Regression"},{"metadata":{"_uuid":"e73dbaefd8c1a2a31a76dbf3481379d97ef26950"},"cell_type":"markdown","source":"Without text preprocessing:"},{"metadata":{"trusted":true,"_uuid":"471c2a1dcc92878d4e73876ee280add13b23fff2"},"cell_type":"code","source":"y = data[2][1:]\ny = y.values\nvectorizer = CountVectorizer(min_df=1)\nsentences = data[1][1:]\nX = vectorizer.fit_transform(list(sentences))\n\nX[0].nonzero()\n\nLR_model = LogisticRegressionCV(Cs=[0.1,0.05,0.01,0.005,0.003,0.001,0.0001],cv=5,random_state=0, solver='lbfgs').fit(X, y)\n\npreds = LR_model.predict(X)\n\npreds = [int(x) for x in preds]\ny_true = [int(y_) for y_ in y]\n\nf1_score(preds,y_true)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1d04030af9bb7fe3f5c3b75606c63daf622616e"},"cell_type":"markdown","source":"With text preprocessing - __Works awful for some reason__"},{"metadata":{"trusted":true,"_uuid":"9f8f280e7a3095c127d9bca737a68f6601bce22b"},"cell_type":"code","source":"text_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"993ee71226e1584d404902d28758f9ddafe9eba7"},"cell_type":"code","source":"from scipy.sparse import coo_matrix\nfrom scipy.sparse import csr_matrix\nimport tqdm\ncol = []\nrow = []\n\nfor ind, elem in tqdm.tqdm(enumerate(train_X)):\n    col_ = [x for x in elem[elem>0]]\n    row_ = np.ones(len(col_),dtype=np.int32)\n    row_ = row_ * ind\n    row.extend(row_)\n    col.extend(col_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06a3dc60dafe94170c21245884e228917df8603e"},"cell_type":"code","source":"data = np.ones(len(row))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"117b99ceb1adbbb845d619d2da228b8dd29f246a"},"cell_type":"code","source":"lr_train_data = csr_matrix((data, (row, col)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d12d92303b8c6463b7bac20c88424a5464f75ba6"},"cell_type":"code","source":"LR_model = LogisticRegressionCV(Cs=[0.1,0.05,0.01,0.005,0.003,0.001,0.0001],cv=5,random_state=0, solver='lbfgs').fit(lr_train_data,train_y)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a67e1b7a75831f5506906897baf8e77bbc7ff8d"},"cell_type":"code","source":"preds = LR_model.predict(lr_train_data)\n\npreds = [int(x) for x in preds]\ny_true = [int(y_) for y_ in y]\n\nf1_score(preds,y_true)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce556830fde0131fd4e7e30cda8424d1568b87d0"},"cell_type":"markdown","source":"# Simple Bi-directional LSTM model"},{"metadata":{"_uuid":"002eae63a3ffea7a70f3ed6db50fff1ffae75e3f"},"cell_type":"markdown","source":"## Parameter definition"},{"metadata":{"trusted":true,"_uuid":"dfe95bb81a919a4a305bb8258e6a3528ba045f91"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\n\nbatch_size = 512\ntrain_epochs = 6\nepochs=5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e98f6e49bea16e7f19d95cfabd1d8ecc11af0825"},"cell_type":"code","source":"class BiLSTM(nn.Module):\n    def __init__(self, embedding_matrix, static=True, hidden_dim=128, lstm_layer=2, dropout=0.2):\n        super(BiLSTM, self).__init__()\n        self.hidden_dim = hidden_dim\n        self.dropout = nn.Dropout(p=dropout)\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        \n        if static:\n            self.embedding.weight.requires_grad = False\n        self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,\n                            hidden_size=hidden_dim,\n                            num_layers=lstm_layer, \n                            dropout = dropout,\n                            bidirectional=True)\n        self.hidden2label = nn.Linear(hidden_dim*lstm_layer*2, 1)\n    \n    def forward(self, sents):\n        x = self.embedding(sents)\n        x = torch.transpose(x, dim0=1, dim1=0)  # Swap batch and sentence dimensions\n        lstm_out, (h_n, c_n) = self.lstm(x)\n        y = self.hidden2label(self.dropout(torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)))\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6aee60fc15f70f59a371d1138f9682e1077b2ef4"},"cell_type":"code","source":"X_train, X_eval, y_train, y_eval = train_test_split(train_X,train_y,test_size=0.05)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4c59b9146eee1753b246eaf24d956a821400cc4"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0778457380d5c69bba091630388323e742cce04"},"cell_type":"code","source":"def train_BiLSTM(epochs, X_train, X_eval, y_train, y_eval):\n\n    model = BiLSTM(embedding_matrix)\n    model.cuda()\n    \n    x_train = torch.tensor(X_train, dtype=torch.long).cuda()\n    y_train = torch.tensor(y_train[:, np.newaxis], dtype=torch.float32).cuda()\n    x_val = torch.tensor(X_eval, dtype=torch.long).cuda()\n    y_val = torch.tensor(y_eval[:, np.newaxis], dtype=torch.float32).cuda()\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n    optimizer = torch.optim.Adam(model.parameters())\n    \n    train = torch.utils.data.TensorDataset(x_train, y_train)\n    valid = torch.utils.data.TensorDataset(x_val, y_val)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        model.train()\n        \n        avg_loss = 0\n        for x_batch, y_batch in train_loader:\n            y_pred = model(x_batch)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        model.eval()\n        valid_preds = np.zeros((x_val.size(0)))\n        avg_val_loss = 0.\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_pred = model(x_batch).detach()\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, epochs, avg_loss, avg_val_loss, elapsed_time))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2eb2e45bb8e8d6063a0bf8d7e36bc668c269590"},"cell_type":"code","source":"model = train_BiLSTM(train_epochs, X_train, X_eval, y_train, y_eval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac299e2652553e639125a074ae5fe023b29d2d1"},"cell_type":"code","source":"x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\ny_pred_test = []\nfor i, (x_batch,) in enumerate(test_loader):\n    y_pred = model(x_batch).detach()\n    y_pred_test.extend(y_pred.cpu().numpy())\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebabdd0321e75b30dd0bb8611ebabc6ba4b07e40"},"cell_type":"markdown","source":"Prepare a submission CSV from the predictions on the test set:"},{"metadata":{"trusted":true,"_uuid":"ecf44898e62937f625adbb08a785eae88c584a0d"},"cell_type":"code","source":"submission = data_test[['qid']].copy()\nthreshold = 0.5\nsubmission['prediction'] = np.array(y_pred_test) > threshold\nsubmission['prediction'] = submission['prediction'].astype(int)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1995937181b1c819fd03236dff23a43c89650ad1"},"cell_type":"markdown","source":"# Conclusion\nFrom participating in this competition, we learnt that \n- Preprocessing makes a difference\n- Other Kaggle kernels are very helpful\n- BiLSTM models give good results for NLP cases\n- Data analysis is important, e.g. analyzing OOV words"},{"metadata":{"trusted":true,"_uuid":"82d50d0aa66e291f61530d07ba9c7684ebdb72e5"},"cell_type":"markdown","source":"Both of us worked on most of the notebook together, and specifically Alexandru did the logistic regression model and Christabella did the data preprocessing."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}