{"cells":[{"metadata":{"_uuid":"e94827fb932bf7407affe2d32b42b683a1c89ec3"},"cell_type":"markdown","source":"\n\n"},{"metadata":{"trusted":true,"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport operator \nimport re\nimport gc\nimport keras\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nsns.set_style('whitegrid')\nimport math\n\n\n\nfrom keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e432bb170a329c79f58f78527bbbe4e857b2c41"},"cell_type":"markdown","source":"## Loading data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\ndf = pd.concat([train ,test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ec4f04ea18218d0f8b46e6e00087db42e70434"},"cell_type":"code","source":"print(\"Number of texts: \", df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a453d10106aea7f6bf3f9bd952f2e02dd7a7472c"},"cell_type":"markdown","source":"## Loading embeddings\n#### In this v2, I am using GloVe only, for memory usage purposes mostly."},{"metadata":{"trusted":true,"_uuid":"807e734e0ce617480c824f8bf26f0672f38397d5"},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44662473bad36713bbe610e84039145c81e941b2"},"cell_type":"code","source":"glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a0a98920daa420bf37a910739cda6bc716ba872"},"cell_type":"code","source":"print(\"Extracting GloVe embedding\")\nembed_glove = load_embed(glove)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5f9048e53a5cb91bdaa518d33fda756f1038562"},"cell_type":"markdown","source":"## Vocabulary and Coverage functions\n> Again, check Dieter's work if you haven't, those are his."},{"metadata":{"trusted":true,"_uuid":"c362fc1868aaa53dfc0bd91d453419815eba7c47"},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dcde9d74210955fdf0b3e855ae69735c3a5c432"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f35a7213fc9a7e80a7c210d11b3a8094d3a8e07e"},"cell_type":"code","source":"def check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.3%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31a3aff72321746bdf6731236d183ee28931793b"},"cell_type":"markdown","source":"## Starting point"},{"metadata":{"trusted":true,"_uuid":"cbf1970d63ea82cc32817b6d2e5f2b976276a08f"},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42c2b82740ac82f5678a46b7c82b5525616c1304","scrolled":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b32cdaef54a0e7b428d1b82ee53afec9f5dd49b"},"cell_type":"markdown","source":"#### Apply lowerization (necessary if using paragram)\n> Fasttext and GloVe understand capitalization, but you still win a bit."},{"metadata":{"trusted":true,"_uuid":"c657fb61a5942ee9807258364be3a2d1058471e6"},"cell_type":"code","source":"df['lowered_question'] = df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d0ac02d8aec1460f3bdc19f6db815fcb84443a8"},"cell_type":"markdown","source":"#### If you apply lowerization, you lost a bit of informations on other embeddings :\n> Therer are words known that are known with upper letters and unknown without. Let us fix that :\n- word.lower() takes the embedding of word if word.lower() doesn't have an embedding"},{"metadata":{"trusted":true,"_uuid":"7703322c717121d6f8ee06ab7316232c75448ebe"},"cell_type":"code","source":"def add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77757cf80eeda36376dca64737c61d3100c1acf4"},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)\nadd_lower(embed_glove, vocab)\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0cd0a201ddd3558ece968bbede941e7f7aee386d"},"cell_type":"markdown","source":"### What's wrong ?"},{"metadata":{"trusted":true,"_uuid":"68ee5f7fdf4def1aa67f76144ed4df9380087619"},"cell_type":"code","source":"oov_glove[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"768c3eca0b5342473dd9c9e208eb90e82eb13328"},"cell_type":"markdown","source":"#### First faults appearing are : \n- Contractions \n- Words with punctuation in them\n\n> Let us correct that."},{"metadata":{"_uuid":"3849b08b9edb1aea040dd61c9fa3f2bb1a58482a"},"cell_type":"markdown","source":"## Contractions"},{"metadata":{"trusted":true,"_uuid":"fb3eb7d653a20b4d8157b08de690b49dfb0148eb"},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"348a75ce1e902dd34eff964b6a511310504fcfa6"},"cell_type":"code","source":"def known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9e43ac139947e32fd3a9f357ebbe1110a56d875"},"cell_type":"code","source":"print(\"- Known Contractions -\")\nprint(\"   Glove :\")\nprint(known_contractions(embed_glove))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a082b84860096732f927b97c421bb0a4c6579ca2"},"cell_type":"markdown","source":"#### Not a lot of contractions are known. (FastText knows none)\n> We use the map to replace them"},{"metadata":{"trusted":true,"_uuid":"d7d6f63b2da86b33d2650114630c57d47f3cc567"},"cell_type":"code","source":"def clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeebeb5d860a0d5c5d8c9cdfc30ed652cea17634"},"cell_type":"code","source":"df['treated_question'] = df['lowered_question'].apply(lambda x: clean_contractions(x, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"490030c049c9c5d32e892da02816de053f8a252e"},"cell_type":"code","source":"vocab = build_vocab(df['treated_question'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dba213d3a9fefce81a3ab41e2ba95846f87da997"},"cell_type":"markdown","source":"## Now, let us deal with special characters"},{"metadata":{"trusted":true,"_uuid":"70f766c527f6b7d87a0213306d20db49a5cc014f"},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c95828228f2b28c55a02890c5a882037cad18262"},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69129cad1f08fda5912051d273db012f9adb081a"},"cell_type":"code","source":"print(\"Glove :\")\nprint(unknown_punct(embed_glove, punct))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0b4dc88770ed22b650dcd590a343f8169dbd9d2"},"cell_type":"markdown","source":"#### Some characters are unknown.\n> We use a map to replace unknown characters with known ones.\n\n> We make sure there are spaces between words and punctuation\n"},{"metadata":{"trusted":true,"_uuid":"2ebe824ff42bc0f93aca2884b430cc96c12857b8"},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3825a0af80d6443e6e2665edadf2c394d8c92ff"},"cell_type":"code","source":"def clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b47c945e6f02c2a9f72aedd72da4681695cb20dd"},"cell_type":"code","source":"df['treated_question'] = df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d89484c4cd04be56e6d81ea6b2b9ded41371ef8"},"cell_type":"code","source":"vocab = build_vocab(df['treated_question'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"4d206d53f696e7d9ee2f7463c8514c2f3bf6ce15"},"cell_type":"code","source":"oov_glove[:100]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63a73741168e19b59ff8efaf167e2fe113725d7e"},"cell_type":"markdown","source":"### What's still missing ? \n- Unknown words\n- Acronyms\n- Spelling mistakes\n\n## We can correct manually most frequent mispells\n\n#### For example, here are some mistakes and their frequency\n- qoura : 85 times\n- mastrubation : 38 times\n- demonitisation : 30 times\n- …"},{"metadata":{"trusted":true,"_uuid":"74b14fa49c71d1041f936642886830079966a2a3"},"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bdc15dd0f50de811510fd870fcaae9c137c42c3"},"cell_type":"code","source":"def correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c887a0a6f7498724a2c377e9e29fac683ea3a59b"},"cell_type":"code","source":"df['treated_question'] = df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63aba43671245f1ebc87540a8bd49a174791c378"},"cell_type":"code","source":"vocab = build_vocab(df['treated_question'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30bc437e1e4f7bf8da692c371f195fb5c58cd548"},"cell_type":"code","source":"len(vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f03014551050ac4ac338f50889ba1ef88852ef89"},"cell_type":"code","source":"vocab['and']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"442f9441cd1529655e6ca04b2ca542bab3128a11"},"cell_type":"code","source":"vocab_filter={}\nfor i in vocab.keys():\n    if vocab[i]>20:\n        vocab_filter[i]=vocab[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efc7796b2ae06ff3b890df49e8746a27440d76d3"},"cell_type":"code","source":"len(vocab_filter)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"53d4e5ec2da13b92573d3d7518913a8e4a8676fd"},"cell_type":"code","source":"del oov_glove, df, vocab, vocab_filter\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"678119fc22ac8aac0363726881b14630ef1d3a29"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b655afc7a5d3e03c1b755e7523ee5d5d59ec742b"},"cell_type":"code","source":"test.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e7249a2c9ee6fbcc459fbbc68c8d1bc9da0ebc9"},"cell_type":"markdown","source":"# Part 2 : Checking the improvement\n\n## Applying treatment\n\nI only treat train data here, because I do not intend to submit my results."},{"metadata":{"trusted":true,"_uuid":"e0d96c70fc518f727c68088bbf1fcbaad95f798f"},"cell_type":"code","source":"# Lowering\ntrain['treated_question'] = train['question_text'].apply(lambda x: x.lower())\n# Contractions\ntrain['treated_question'] = train['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Special characters\ntrain['treated_question'] = train['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n# Spelling mistakes\ntrain['treated_question'] = train['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"155cefe9997e20fcd9cc0934d212b927b76165ed"},"cell_type":"code","source":"# Lowering\ntest['treated_question'] = test['question_text'].apply(lambda x: x.lower())\n# Contractions\ntest['treated_question'] = test['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Special characters\ntest['treated_question'] = test['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n# Spelling mistakes\ntest['treated_question'] = test['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c995fd5f0a31b215dfad14c6f8519c6c0ed7bb0e"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d984ea38db9f8aab46e93edf2fa33da915b4b61"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e652c150f39d37c989f6d325037bce8c9ff630ad"},"cell_type":"markdown","source":"## Data for the network\n### Texts\n#### Parameters\nI took the same for both models. len_voc can de reduced for the treated model."},{"metadata":{"_uuid":"c463d1b1a2b73fcfdd0f9c0841e837f7c90e126a"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"4405fd1b7f5292d874f4a3b88a2bccde6af41152"},"cell_type":"code","source":"p=[len(s) for s in train['treated_question']]\nprint(np.mean(p))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7008f960a57db94de19b9a2630afce3d5bc606d6"},"cell_type":"code","source":"# now let us check in the number of Percentage\nCount_Normal_transacation = len(train[train[\"target\"]==0]) # normal transaction are repersented by 0\nCount_insincere_transacation = len(train[train[\"target\"]==1]) # fraud by 1\nPercentage_of_Normal_transacation = Count_Normal_transacation/(Count_Normal_transacation+Count_insincere_transacation)\nprint(\"percentage of normal transacation is\",Percentage_of_Normal_transacation*100)\nPercentage_of_insincere_transacation= Count_insincere_transacation/(Count_Normal_transacation+Count_insincere_transacation)\nprint(\"percentage of fraud transacation\",Percentage_of_insincere_transacation*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddfa959bc0a4d51f89709711935dbdd945ee24b0"},"cell_type":"code","source":"# insincere_indices= np.array(train[train.target==1].index)\n# normal_indices = np.array(train[train.target==0].index)\n# # now let us a define a function for make undersample data with different proportion\n# #different proportion means with different proportion of normal classes of data\n# def undersample(normal_indices,insincere_indices,times):#times denote the normal data = times*fraud data\n#     Normal_indices_undersample = np.array(np.random.choice(normal_indices,(times*Count_insincere_transacation),replace=False))\n#     print(len(Normal_indices_undersample))\n#     undersample_data= np.concatenate([insincere_indices,Normal_indices_undersample])\n\n#     undersample_data = train.iloc[undersample_data,:]\n#     #print(undersample_data)\n#     print(len(undersample_data))\n\n#     print(\"the normal transacation proportion is :\",len(undersample_data[undersample_data.target==0])/len(undersample_data))\n#     print(\"the fraud transacation proportion is :\",len(undersample_data[undersample_data.target==1])/len(undersample_data))\n#     print(\"total number of record in resampled data is:\",len(undersample_data))\n#     return(undersample_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22ae1d702419f6c43e8c48cc742dc77e0e2d60f8"},"cell_type":"code","source":"# Undersample_data = undersample(normal_indices,insincere_indices,4)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14f049bc8733d0454097fe792c2030a29e8b78b9"},"cell_type":"code","source":"# Undersample_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b63ee2a810ee44f2f532ee24f06be73cfbfd505"},"cell_type":"code","source":"questions=train.iloc[:,3].values.tolist()\nlabels=train.iloc[:,2].values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ac675f66deb147540d5fc264e9c158a1cd1b3e0"},"cell_type":"code","source":"len_voc = 90000\nmax_len = 70","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67cd3d8f6e0888d8584c05697d7788a37433a168"},"cell_type":"markdown","source":"#### Untreated text\nWe apply a standard tokenizer and padding."},{"metadata":{"trusted":true,"_uuid":"b7a2f11e8a9a3f985f56f8d286e505eb345ca903"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cd569c2b10bc2c4836c482f52250c5324e69193"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b07c8c6e545e47e42babeefc48dfe5b133d424f8"},"cell_type":"code","source":"test['treated_question'][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"add61c5faefd22eb608b3cbcdf96a5e4d75178b0"},"cell_type":"code","source":"def make_data(X):\n    t = Tokenizer(num_words=len_voc)\n    t.fit_on_texts(X)\n    X = t.texts_to_sequences(X)\n    X = pad_sequences(X, maxlen=max_len)\n    return X, t.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b846bba8e70c0a5e0452e5260aec1e2b4963ddd"},"cell_type":"code","source":"X, word_index = make_data(questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fd32fdfe2af89d43f819992cdf663392ff7a829"},"cell_type":"code","source":"X_test, word_index_test=make_data(test['treated_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72c696563bc690fcb2f5e02cb6d5ad47daa8b94e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff1e8e3052ece84630f50d574151de41a9a88e6f"},"cell_type":"markdown","source":"#### Treated text\nSame thing, but with no filters."},{"metadata":{"trusted":true,"_uuid":"4120fd0029e1bd6012c57df3ffa992fda55d1d35"},"cell_type":"code","source":"# def make_treated_data(X):\n#     t = Tokenizer(num_words=len_voc, filters='')\n#     t.fit_on_texts(X)\n#     X = t.texts_to_sequences(X)\n#     X = pad_sequences(X, maxlen=max_len)\n#     return X, t.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dc97debc1714707553504e6288d77a216570cdd"},"cell_type":"code","source":"# X_treated, word_index_treated = make_data(train['treated_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"686cb0b60b3a0ac387df5ec3fe6b0df6bbe69af9"},"cell_type":"markdown","source":"#### Splitting"},{"metadata":{"trusted":true,"_uuid":"19ea65cf3767f7d56c11f958b4c94411bf8e985a"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3df599e7cc581d1883dae3667d9c80d123f71d"},"cell_type":"code","source":"y = labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f435f338b9d5fdafb7a4d98ad093e26e54eec2ee"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=420)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"937fae0dddb6a6b537ebfe47a493a2caa1a17164"},"cell_type":"code","source":"#X_t_train, X_t_val, _, _ = train_test_split(X_treated, y, test_size=0.1, random_state=420)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e4dbcd05c7778345dc224dd9c20a13fdee71d18"},"cell_type":"code","source":"print(f\"Training on {X_val.shape[0]} texts\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6194a6019bbfd45887f39de1c11f4a7d15a8ebf1"},"cell_type":"code","source":"type(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1104c6805a2e3b34452c34a2585ed86963c690f"},"cell_type":"code","source":"Count_Normal = np.sum(y_val)# normal transaction are repersented by 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be8d2b7cda219d93ce8f8acc4daedcf3224d119f"},"cell_type":"code","source":"Count_Normal","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16b621cdaccbed7c1903644908bb60b2ee0451c6"},"cell_type":"markdown","source":"### Embeddings\nI use GloVe here, because I got better results with it than with others. But feel free to change that."},{"metadata":{"trusted":true,"_uuid":"b606ff10b1a4312c102ea74048945c950daa84a2"},"cell_type":"code","source":"def make_embed_matrix(embeddings_index, word_index, len_voc):\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    word_index = word_index\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len_voc, embed_size))\n    \n    for word, i in word_index.items():\n        if i >= len_voc:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6837532d94b2b0cca1a7f4f08b53e90a17ad814","_kg_hide-output":true},"cell_type":"code","source":"embedding = make_embed_matrix(embed_glove, word_index, len_voc)\n\ndel word_index\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47413d67935b998fe654fab0e81018917a106df8","_kg_hide-output":true},"cell_type":"code","source":"#  embedding_test = make_embed_matrix(embed_glove, word_index_test, len_voc)\ndel word_index_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a59c89c8f473156264420fa70f82aee89176d06"},"cell_type":"code","source":"# batch_size = 128\n\n# def batch_gen(train_df,y):\n#     n_batches = math.ceil(len(train_df) / batch_size)\n#     while True: A\n#         #train_df = train_df.sample(frac=1.)  # Shuffle the data.\n#         for i in range(n_batches):\n#             texts = train_df[i*batch_size:(i+1)*batch_size]\n#             text_arr = np.array([text for text in texts])\n#             yield text_arr, np.array(y[i*batch_size:(i+1)*batch_size])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddadc6e782815c04bfd8fa3f1e23f22050446768"},"cell_type":"markdown","source":" Note that we have two embedding matrices, one for each pre-treatment."},{"metadata":{"_uuid":"b3a06de1cca31af1e4191314ec860a86de413460"},"cell_type":"markdown","source":"## f1 metric for Keras"},{"metadata":{"trusted":true,"_uuid":"d6b7a09c1a91bf48495fd53606d3f0dfc53039cf"},"cell_type":"code","source":"from keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50e66352313d185ea77466210714bb11358c46c0"},"cell_type":"code","source":"def f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66eb6a8194f17990c2f71ad7ab7642af0094bb0c"},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true,"_uuid":"2d9f84ba67646af02f57eb151083ad969092227c"},"cell_type":"code","source":"from keras.models import Model, Sequential\nfrom keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras import regularizers\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dc9135fac68c013684dc62ed1da4e71582696f9"},"cell_type":"code","source":"def make_model(embedding_matrix, embed_size=300, loss='binary_crossentropy'):\n    inp    = Input(shape=(max_len,))\n    x      = Embedding(len_voc, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x      = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x      = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    avg_pl = GlobalAveragePooling1D()(x)\n    max_pl = GlobalMaxPooling1D()(x)\n    concat = concatenate([avg_pl, max_pl])\n    dense  = Dense(64, activation=\"relu\")(concat)\n    drop   = Dropout(0.1)(concat)\n    output = Dense(1, activation=\"sigmoid\")(concat)\n    \n    model  = Model(inputs=inp, outputs=output)\n    model.compile(loss=loss, optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0e3d98d31adcfad298d2ac68437ed9f21259940"},"cell_type":"code","source":"model = make_model(embedding)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9739b5209c631516d325de4f1d4d22a9b8dd4468"},"cell_type":"code","source":"#model_treated = make_model(embedding_treated)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90912bb425aaec0c9599d8c9e8feb1846bb394bc"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93ef900ae4fdf2dd3f40c06dcea17d18ee99bcc3"},"cell_type":"markdown","source":"###  Callbacks"},{"metadata":{"trusted":true,"_uuid":"7095e222de45bb20019775c39c07f1344598d4b8"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eba688a58697762bf43d02dceb8036b292bfaef"},"cell_type":"code","source":"checkpoints = ModelCheckpoint('weights.hdf5', monitor=\"val_f1\", mode=\"max\", verbose=True, save_best_only=True)\n#checkpoints2 = ModelCheckpoint('weights2.hdf5', monitor=\"val_loss\", mode=\"max\", verbose=True, save_best_only=True)\n\nreduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6c3216bd562c5468feb332c95fca62d4483e475"},"cell_type":"code","source":"#checkpoints_treated = ModelCheckpoint('treated_weights.hdf5', monitor=\"val_f1\", mode=\"max\", verbose=True, save_best_only=True)\n#reduce_lr_treated = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d43376512f83085b4fc0ef7625ab0026bd29b9c"},"cell_type":"markdown","source":"### Fitting"},{"metadata":{"trusted":true,"_uuid":"fa9274ac481d0025733fd25bfbd37e68e0d1c882"},"cell_type":"code","source":"epochs = 5\nbatch_size = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d6a666e70a041b124949d6fd5d91271a885a98e"},"cell_type":"code","source":"history = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, \n                    validation_data=[X_val, y_val], callbacks=[checkpoints, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd23764b08aa7d3834760e71df67e3f4700893e5"},"cell_type":"code","source":" plt.figure(figsize=(12,8))\n plt.plot(history.history['acc'], label='Train Accuracy')\n plt.plot(history.history['val_acc'], label='Test Accuracy')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"872fab897ea7e025fa00f0bf7c56d82a7546d09a"},"cell_type":"code","source":"# history = model_treated.fit(X_t_train, y_train, batch_size=batch_size, epochs=epochs, \n                            #validation_data=[X_t_val, y_val], callbacks=[checkpoints_treated, reduce_lr_treated])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0630350a71b2227a59df1ed198d9b3c7f5246bf9"},"cell_type":"code","source":"# plt.figure(figsize=(12,8))\n# plt.plot(history.history['acc'], label='Train Accuracy')\n# plt.plot(history.history['val_acc'], label='Test Accuracy')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bfbdc1fecdaf1ec2a138a7b80d505a1489a24c9"},"cell_type":"code","source":" model.load_weights('weights.hdf5')\n# model_treated.load_weights('treated_weights.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47a9f7d04c0f42558661e23b93b167836ff50920"},"cell_type":"code","source":"# model2 = Sequential([\n#     Embedding(len_voc, 300, input_length=max_len,dropout=0.2,weights=[embedding]),\n#     #BatchNormalization(),\n#     #Dropout(0.5),\n#     Bidirectional(LSTM(300,activation='relu',recurrent_dropout=0.2,consume_less='gpu')),\n#     #BatchNormalization(),\n#     Dropout(0.5),\n#     Dense(1, activation='sigmoid')])\n# model2.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n# model2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0efed69fba3509b06a2408cb478ae2ed1f7fc1c"},"cell_type":"code","source":"type(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe9e4896a5eb721d872774d82797a7bcd77efa05"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4595165bb1be0fa273a6498a0ff52102ec2d841c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72afcf42d287ed520da51db907d47de37db4f5ef"},"cell_type":"code","source":"# history = model2.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, \n#                     validation_data=[X_val, y_val], callbacks=[checkpoints2, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1415ea70292cc202ff95c28025b1dc31ac1911a5"},"cell_type":"markdown","source":"## Results\n### Predictions"},{"metadata":{"trusted":true,"_uuid":"881237d39c2f83b493bde867f0f123e01ce29f0b"},"cell_type":"code","source":" pred_val = model.predict(X_val, batch_size=512, verbose=1)\n# pred_t_val = model_treated.predict(X_t_val, batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"821da6d03180134f4127bc6c2a092de6f6dc5b4c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8131c91988d2cc2daaca90591fd2748c05261b31"},"cell_type":"code","source":"pred_test= model.predict(X_test, batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3440b432d6f372a30f1e6c62df829372eb962f0e"},"cell_type":"code","source":"pred_test[0:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c7b54e85e89de7dc05687336ded94c102e7642a"},"cell_type":"code","source":"# pred_test2[0:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a3f50ba8e8536d361b7e7a9d73fc8170dcd614a"},"cell_type":"markdown","source":"#### F1 Scores"},{"metadata":{"trusted":true,"_uuid":"da42564946ee7e2fa91ad12df00cf02cf513b237"},"cell_type":"code","source":" from sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76f401f5d9f6ee6f06d979b08fd4a15c35c5b0e5"},"cell_type":"code","source":"def tweak_threshold(pred, truth):\n    thresholds = []\n    scores = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        thresholds.append(thresh)\n        score = f1_score(truth, (pred>thresh).astype(int))\n        scores.append(score)\n        return np.max(scores), thresholds[np.argmax(scores)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46877e6bb90d3e8fc25a932077e10c202537790d"},"cell_type":"code","source":"score_val, threshold_val = tweak_threshold(pred_val, y_val)\n\nprint(f\"Scored {round(score_val, 4)} for threshold {threshold_val} with treated texts on validation data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5148252a60d1349f149d93473305b690a386604"},"cell_type":"code","source":"pred_val[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e22a81d4059e11df635a83233c9b28e34cc0d9cd"},"cell_type":"code","source":"y_val[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5f0a3a88aedd8489b50e83bd3ae63c550b4d0ad"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32eb17667b4507096d37fa5d030aabbf7c1fbf2c"},"cell_type":"code","source":"submission = test[['qid']].copy()\nsubmission['prediction'] = (pred_test > 0.6).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c552c4730a85a025bb69891ded511c353c706de2"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"522e7f04cc0e9829a8a8d45cc5dea8a8dc5d704a"},"cell_type":"code","source":"!head submission.csv\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f16f582524d7131457f28417e47f9d26bf00e02"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}