{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\ntqdm.pandas()\nimport gc\nimport os\nimport operator\nimport keras\nfrom keras import backend as K\nfrom keras.callbacks import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Model\nfrom keras.layers import CuDNNGRU, CuDNNLSTM, Dense, Embedding, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, Conv1D, SpatialDropout1D\nfrom keras.optimizers import Adam\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\nprint(train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_emb(file):\n    def get_coefs(word, *arr):\n        return word, np.asarray(arr, dtype='float32')\n    embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding=\"latin\"))\n    \n    return embedding_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glove = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Loading Glove Embeddings')\nembed_glove = load_emb(glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(sentences, verbose=True):\n    \n    vocab = {}\n    \n    for sentence in tqdm(sentences):\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n                \n    return vocab    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train_df[\"question_text\"].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print({k: vocab[k] for k in list(vocab)[234:245]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(sentences[:2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_coverage(vocab, embedding_index):\n    known_words = {}\n    unknown_words = {}\n    num_known_words = 0\n    num_unknown_words = 0\n    \n    for word in tqdm(vocab):\n        try:\n            known_words[word] = embedding_index[word]\n            num_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            num_unknown_words += vocab[word]\n            pass\n    print(\"Found embeddings for {:.2%} of the Vocab\".format(len(known_words)/len(vocab)))\n    print(\"Found embeddings for {:.2%} of all text\".format(num_known_words/(num_known_words+num_unknown_words)))\n        \n    sorted_x = sorted(unknown_words.items(), key = operator.itemgetter(1))[::-1]\n        \n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def known_contractions(embedding_index):\n    known= []\n    for contraction in contraction_mapping:\n        if contraction in embedding_index:\n            known.append(contraction)\n    return known","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Known contractions in Glove embedding:\")\nprint(known_contractions(embed_glove))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['treated_question'] = train_df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train_df[\"treated_question\"].progress_apply(lambda x: x.split()).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punc = \"/-'?!.#$%\\'()*+-/:;<=>,@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = ''\n    count = 0\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unknown punctuations in Glove:\")\nprint(unknown_punct(embed_glove, punc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'rupee' in embed_glove","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(text):\n    text = str(text)\n    \n    for p in punc:\n        text = text.replace(p, f' {p} ')\n    for pun in \"₹\":\n        text = text.replace(pun, \"rupee\")\n            \n    return text    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"cleaned_question\"] = train_df[\"treated_question\"].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train_df[\"cleaned_question\"].progress_apply(lambda x: x.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def correct_spelling(text, dic):\n    for m in dic.keys():\n        if m in text:\n            text = text.replace(m, dic[m])\n            \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"cleaned_question\"] = train_df[\"cleaned_question\"].progress_apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train_df[\"cleaned_question\"].progress_apply(lambda x: x.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x, val_x = train_test_split(train_df[[\"cleaned_question\", \"target\"]], test_size = 0.2, random_state=2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 300 #dimension of word embedding\nvocab_len = 70000 #length of vocabulary\nmax_len = 100 #maximum number of words in a sentence\n\ntrain_X = train_x[\"cleaned_question\"].values\ntrain_Y = train_x[\"target\"].values\nval_X = val_x[\"cleaned_question\"].values\nval_Y = val_x[\"target\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=vocab_len)\ntokenizer.fit_on_texts(list(train_X))\ntrain_sentences = tokenizer.texts_to_sequences(list(train_X))\ntrain_sentences = pad_sequences(train_sentences, maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sentences[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_sentences = tokenizer.texts_to_sequences(val_X)\nval_sentences = pad_sequences(val_sentences, maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_sentences[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df, sentences, vocab, oov, train_x, train_X, val_x, val_X\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_embed_matrix(embedding_index, word_index, len_voc):\n    all_emb = np.stack(embedding_index.values())\n    mean_emb = all_emb.mean()\n    std_emb = all_emb.std()\n    embed_sz = all_emb.shape[1]\n    word_index = word_index\n    embedding_matrix = np.random.normal(mean_emb, std_emb, (len_voc, embed_sz))\n    \n    for word, i in word_index.items():\n        if i>= len_voc:\n            continue\n        embedding_vector = embedding_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector \n        return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix = make_embed_matrix(embed_glove, word_index, vocab_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del embed_glove\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input(shape=(max_len, ))\nx = Embedding(vocab_len, embed_size, weights=[embed_matrix], trainable=False)(inp)\nx = SpatialDropout1D(0.125)(x)\nx = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\nx = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\nx = Conv1D(64, kernel_size=1, activation=\"relu\")(x)\ny = GlobalMaxPooling1D()(x)\nz = GlobalAveragePooling1D()(x)\nx = concatenate([y, z])\nx = Dense(64, activation=\"relu\")(x)\nx = Dropout(0.3)(x)\nx = Dense(1, activation = 'sigmoid')(x)\n\nmodel = Model(inputs=inp, outputs=x)\n\nmodel.compile(loss = 'binary_crossentropy', optimizer=\"adam\", metrics = [\"accuracy\"])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One Cycle Policy implementation from https://github.com/keras-team/keras-contrib/blob/master/keras_contrib/callbacks/cyclical_learning_rate.py"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency.\n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore\n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where\n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored\n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on\n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    The amplitude of the cycle can be scaled on a per-iteration or\n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each\n        cycle iteration.\n    For more detail, please see paper.\n    # Example for CIFAR-10 w/ batch size 100:\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    # References\n      - [Cyclical Learning Rates for Training Neural Networks](\n      https://arxiv.org/abs/1506.01186)\n    \"\"\"\n\n    def __init__(\n            self,\n            base_lr=0.001,\n            max_lr=0.006,\n            step_size=2000.,\n            mode='triangular',\n            gamma=1.,\n            scale_fn=None,\n            scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        if mode not in ['triangular', 'triangular2',\n                        'exp_range']:\n            raise KeyError(\"mode must be one of 'triangular', \"\n                           \"'triangular2', or 'exp_range'\")\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1 / (2.**(x - 1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma ** x\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr is not None:\n            self.base_lr = new_base_lr\n        if new_max_lr is not None:\n            self.max_lr = new_max_lr\n        if new_step_size is not None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n\n    def clr(self):\n        cycle = np.floor(1 + self.clr_iterations / (2 * self.step_size))\n        x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr - self.base_lr) * \\\n                np.maximum(0, (1 - x)) * self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr - self.base_lr) * \\\n                np.maximum(0, (1 - x)) * self.scale_fn(self.clr_iterations)\n\n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())\n\n    def on_batch_end(self, epoch, logs=None):\n\n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n        K.set_value(self.model.optimizer.lr, self.clr())\n\n        self.history.setdefault(\n            'lr', []).append(\n            K.get_value(\n                self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clr =  CyclicLR(base_lr=0.0005,\n                max_lr=0.005,\n                step_size = 300,\n                mode=\"exp_range\",\n               gamma = 0.99994)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_sentences, train_Y, batch_size=1024, epochs=5, validation_data=(val_sentences, val_Y), callbacks = [clr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_glove = model.predict([val_sentences], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef tweak_threshold(pred, truth):\n    thresholds = []\n    scores = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        thresholds.append(thresh)\n        score = f1_score(truth, (pred>thresh).astype(int))\n        scores.append(score)\n    return np.max(scores), thresholds[np.argmax(scores)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_val, threshold_val = tweak_threshold(pred_val_glove, val_Y)\n\nprint(f\"Scored {round(score_val, 4)} for threshold {threshold_val} on glove embedding on validation data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')\nprint(test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['treated_question'] = test_df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"cleaned_question\"] = test_df[\"treated_question\"].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"cleaned_question\"] = test_df[\"cleaned_question\"].progress_apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x = test_df[\"cleaned_question\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X = tokenizer.texts_to_sequences(list(test_x))\ntest_X = pad_sequences(test_X, maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_x\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y_glove = model.predict([test_X], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del embed_matrix, model, inp, x\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paragram = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_paragram = load_emb(paragram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_matrix = make_embed_matrix(embed_paragram, word_index, vocab_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input(shape=(max_len, ))\nx = Embedding(vocab_len, embed_size, weights=[embed_matrix], trainable=False)(inp)\nx = SpatialDropout1D(0.125)(x)\nx = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\nx = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\nx = Conv1D(64, kernel_size=1, activation=\"relu\")(x)\ny = GlobalMaxPooling1D()(x)\nz = GlobalAveragePooling1D()(x)\nx = concatenate([y, z])\nx = Dense(64, activation=\"relu\")(x)\nx = Dropout(0.3)(x)\nx = Dense(1, activation = 'sigmoid')(x)\n\nmodel = Model(inputs=inp, outputs=x)\n\nmodel.compile(loss = 'binary_crossentropy', optimizer=\"adam\", metrics = [\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_sentences, train_Y, batch_size=512, epochs=5, validation_data=(val_sentences, val_Y), callbacks=[clr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_paragram = model.predict([val_sentences], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_val, threshold_val = tweak_threshold(pred_val_paragram, val_Y)\n\nprint(f\"Scored {round(score_val, 4)} for threshold {threshold_val} on paragram embedding on validation data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_y = 0.5*pred_val_glove + 0.5*pred_val_paragram\n\nscore_val, threshold_val = tweak_threshold(pred_val_y, val_Y)\n\nprint(f\"Scored {round(score_val, 4)} for threshold {threshold_val} on glove and paragram embedding on validation data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y_paragram = model.predict([test_X], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"pred_test_y = 0.5*pred_test_y_glove + 0.5*pred_test_y_paragram\npred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**References**\n\n1. A look at different embeddings - https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n2. improve-your-score-with-text-preprocessing-v2 - https://www.kaggle.com/theoviel/improve-your-score-with-text-preprocessing-v2\n3. Common pitfalls of public kernels - https://www.kaggle.com/c/quora-insincere-questions-classification/discussion/79911#469983\n4. Text Preprocessing Methods for Deep Learning - https://mlwhiz.com/blog/2019/01/17/deeplearning_nlp_preprocess/\n5. https://mlwhiz.com/blog/2019/02/19/siver_medal_kaggle_learnings/"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}