{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# ## some config values \n# embed_size = 300 # how big is each word vector\n# #max_features = 100000 # how many unique words to use (i.e num rows in embedding vector)\n# max_features = 120000\n# maxlen = 72 # max number of words in a question to use\n\n## some config values \nembed_size = 600 # how big is each word vector\n#max_features = 100000 # how many unique words to use (i.e num rows in embedding vector)\nmax_features = None\nmaxlen = 57 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63065b644dbf8dc4f96ec3ae57d746cbdcfdec01"},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization\nfrom keras.optimizers import Adam, Nadam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.callbacks import Callback\n\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\nfrom keras.engine.topology import Layer\n# from tensorflow.keras.layers import Layer\n\nimport gc, re\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"643ca485fcd7164cee3703ac9d119487416adf72"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\"}\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d604ba32099a34133cc446b499a4515e928240c3"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n       # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    \n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    splits = list(StratifiedKFold(n_splits=10,random_state=2018).split(train_X,train_df['target'].values))\n    ## fill up the missing values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## split to train and val\n    train_X, val_X = train_X[splits[0][0]],train_X[splits[0][1]]\n    \n    ## Get the target values\n#     train_y = train_df['target'].values\n    train_y = train_df['target'].values[splits[0][0]]\n    val_y = train_df['target'].values[splits[0][1]]\n    \n    ## Tokenize the sentences\n#     tokenizer = Tokenizer(num_words=max_features)\n    tokenizer = Tokenizer(num_words=max_features, filters='', lower=True)\n    tokenizer.fit_on_texts(list(train_X)+list(test_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n    val_idx = np.random.permutation(len(val_X))\n\n    train_X = train_X[trn_idx]\n    val_X = val_X[val_idx]\n    train_y = train_y[trn_idx]\n    val_y = val_y[val_idx]    \n    \n    return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index\n#     return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7be6184ccb1d6dec64296f4c21e372bb2a6418f3"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = 300\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    #nb_words = len(word_index)\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\") as f:\n        for line in f:\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= nb_words:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: break\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = 300\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    #nb_words = len(word_index)\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\", errors='ignore') as f:\n        for line in f:\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= nb_words:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64856b55a0820a5a7a6e617210746fd32b22debf"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    rocauc = roc_auc_score(y_true, y_proba)\n    p, r, _ = precision_recall_curve(y_true, y_proba)\n    prauc = auc(r, p)\n    search_result = {'threshold': best_threshold, 'f1': best_score, 'rocauc': rocauc, 'prauc': prauc}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a8c0ea863be1ba4cd8624c3ab5ffdc56b9b0ec2"},"cell_type":"code","source":"# https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate/code\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n            scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n        \n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6953bbb11b0fc5d78b7c553c553a8eb5618ba8fb"},"cell_type":"code","source":"# https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50cd31c14d0fd4f87d4c20538bf21f104a29504f"},"cell_type":"code","source":"def model_gru_conv_3(embedding_matrix):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.2)(x)\n    x0 = Bidirectional(CuDNNLSTM(128, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x)\n    x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)\n    z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = \"tanh\")(x1)\n    y1 = GlobalMaxPooling1D()(x1)\n    y2 = GlobalMaxPooling1D()(z)\n    x = concatenate([y1,y2])\n    #x = Dropout(0.1)(x)\n    #x = Dense(8, kernel_initializer=initializers.he_uniform(seed=2018), activation='relu')(x)\n#     x = Dense(32, activation=\"relu\")(x)\n#     x = Dropout(0.1)(x)\n#     x = Dense(1, activation=\"sigmoid\")(x)\n    outp= Dense(1, kernel_initializer=initializers.he_uniform(seed=2018), activation='sigmoid')(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n#     model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b26137f7710c658848beb2fdc073cb66f4d2fea2"},"cell_type":"code","source":"SEED=2018\ndef model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features):\n    embedding_matrix = np.concatenate([embedding_matrix,np.zeros((1,np.shape(embedding_matrix)[1]))])\n    print(np.shape(embedding_matrix))\n    \n    left_context = Input(shape=(maxlen,))\n    document = Input(shape=(maxlen,))\n    right_context = Input(shape=(maxlen,))\n\n    embedder = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)\n    doc_embedding = embedder(document)\n    doc_embedding = SpatialDropout1D(0.2)(doc_embedding)\n    l_embedding = embedder(left_context)\n    l_embedding = SpatialDropout1D(0.2)(l_embedding)\n    r_embedding = embedder(right_context)\n    r_embedding = SpatialDropout1D(0.2)(r_embedding)\n    \n    # I use LSTM RNNs instead of vanilla RNNs as described in the paper.\n    forward = CuDNNLSTM(hidden_dim_1, return_sequences = True)(l_embedding) # See equation (1).\n    backward = CuDNNLSTM(hidden_dim_1, return_sequences = True)(r_embedding) # See equation (2).\n    # Keras returns the output sequences in reverse order.\n    backward = Lambda(lambda x: K.reverse(x, axes = 1))(backward)\n    together = concatenate([forward, doc_embedding, backward], axis = 2) # See equation (3).\n\n#     semantic = TimeDistributed(Dense(hidden_dim_2, kernel_initializer=initializers.he_uniform(seed=SEED), activation = \"relu\"))(together)\n#     semantic = Bidirectional(CuDNNGRU(128, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(together)\n    semantic = Conv1D(hidden_dim_2*8, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = \"tanh\")(together) # See equation (4).\n\n    # Keras provides its own max-pooling layers, but they cannot handle variable length input\n    # (as far as I can tell). As a result, I define my own max-pooling layer here.\n    pool_rnn = GlobalMaxPool1D()(semantic) # See equation (5).\n\n    output = Dense(1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = \"sigmoid\")(pool_rnn) # See equations (6) and (7).\n\n    model = Model(inputs = [left_context, document, right_context], outputs = output)\n    model.compile(optimizer = Adam(), loss = 'binary_crossentropy', metrics = [\"accuracy\"])\n#     model.compile(optimizer = Nadam(), loss = 'binary_crossentropy', metrics = [\"accuracy\"])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8fe26dfe0aabf48ae1c039dc02060a268887308"},"cell_type":"code","source":"def model_lstm_atten(embedding_matrix):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.2)(x)\n    x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n#     x1 = Bidirectional(CuDNNGRU(64, return_sequences=True))(x0)\n    x2 = Bidirectional(CuDNNGRU(96, return_sequences=True))(x0)\n#     x2 = CuDNNGRU(64, return_sequences=True)(x1)\n    y2 = GlobalMaxPooling1D()(x2)\n#     x = Concatenate()([y2, y1])\n#     y2 = BatchNormalization()(y2)\n    y2 = Dropout(0.1)(y2)\n    x = Dense(1, activation=\"sigmoid\")(y2)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n#     model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c0fe054e48345eac59afceaf13e19d423aae529"},"cell_type":"code","source":"def model_lstm_max(embedding_matrix):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.2)(x)\n    x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n#     x1 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x0)\n#     x2 = Bidirectional(CuDNNGRU(64, return_sequences=True))(x0)\n    x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)\n    z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = \"tanh\")(x1)\n#     x2 = CuDNNGRU(64, return_sequences=True)(x1)\n#     y1 = Attention(maxlen)(x1)\n    y2 = Attention(maxlen)(z)\n#     x = Concatenate()([y2, y1])\n#     x = Dense(64, activation=\"relu\")(x)\n#     x = Dense(16, activation=\"relu\")(y2)\n#     x = Dropout(0.1)(x)\n#     x = Dense(1, activation=\"sigmoid\")(x)\n    x = Dense(1, activation=\"sigmoid\")(y2)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n#     model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dafa76f160ebd059e54860672f274cbde66fccf9"},"cell_type":"code","source":"def get_train_list(train_X):\n    return [np.concatenate((np.ones((np.shape(train_X)[0],1))*max_features+1,train_X[:,1:]),1),train_X,np.concatenate((np.ones((np.shape(train_X)[0],1))*max_features+1,train_X[:,::-1][:,1:]),1)]\n\n# https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go\ndef RCNN_train_pred(model, epochs=2):\n    train_X_list = get_train_list(train_X)\n    test_X_list=get_train_list(test_X)\n    val_X_list=get_train_list(val_X)\n    for e in range(epochs):\n        model.fit(train_X_list, train_y, batch_size=512, epochs=1, validation_data=(val_X_list, val_y),callbacks=[clr])\n    pred_val_y = model.predict(val_X_list, batch_size=1024, verbose=0)\n    pred_test_y = model.predict(test_X_list, batch_size=1024, verbose=0)\n    return pred_val_y, pred_test_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d09526e78d882d83a6a6bd09fc72ea7db890172"},"cell_type":"code","source":"def train_pred(model, epochs=2):\n    for e in range(epochs):\n        model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),verbose=1,callbacks=[clr])\n\n    pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n    return pred_val_y, pred_test_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5ecc18cac99759bcdc8cca6dbbb96fd9e3d563e","scrolled":true},"cell_type":"code","source":"train_X, val_X, test_X, train_y, val_y,  word_index = load_and_prec()\nmax_features = len(word_index)\nprint(max_features)\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_fasttext(word_index)\n# embedding_matrix_3 = load_para(word_index)\n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)\n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nembedding_matrix = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis = 1)\n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddca708c3522eef2eb054bcec50acff72b3f825e"},"cell_type":"code","source":"outputs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f713b8197d78b6cc1619ba0aff653083ad44f27f"},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bf9f5df6d089f40a78b99110f3c75e99ad747c3"},"cell_type":"code","source":"pred_val_y, pred_test_y = RCNN_train_pred(model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features+1), epochs = 5)\noutputs.append([pred_val_y, pred_test_y, 'RCNN'])\nresults = threshold_search(val_y, pred_val_y)\nprint(results)\nprint(confusion_matrix(val_y,pred_val_y>results['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1c2df9d9be32ef398531b2e764f6857a38b0149"},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91d80f141017af49b7bdc41432a79610eaa79c23"},"cell_type":"code","source":"pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 4)\noutputs.append([pred_val_y, pred_test_y, 'LSTM w/ max'])\nresults = threshold_search(val_y, pred_val_y)\nprint(results)\nprint(confusion_matrix(val_y,pred_val_y>results['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3920879e12a13218b624b7c56a9039486a78880e"},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18cc18e20596f0b2335518deddaab6dab4501271"},"cell_type":"code","source":"pred_val_y, pred_test_y = train_pred(model_gru_conv_3(embedding_matrix), epochs = 4)\noutputs.append([pred_val_y, pred_test_y, 'LSTM conv 3'])\nresults = threshold_search(val_y, pred_val_y)\nprint(results)\nprint(confusion_matrix(val_y,pred_val_y>results['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"861992751fd5f143848a150c53c22b4445813980"},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d07e638a58b8f89a9788054e4ddcef891a57541"},"cell_type":"code","source":"pred_val_y, pred_test_y = train_pred(model_lstm_max(embedding_matrix), epochs = 4)\noutputs.append([pred_val_y, pred_test_y, 'LSTM w/ atten'])\nresults = threshold_search(val_y, pred_val_y)\nprint(results)\nprint(confusion_matrix(val_y,pred_val_y>results['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"228331ff732f6effdb7d1b561b2f6f2b595efb13"},"cell_type":"code","source":"# from sklearn.linear_model import LogisticRegression\ntrain_meta_data=np.concatenate([np.reshape(i[0],(-1,1)) for i in outputs],axis=1)\ntest_meta_data = np.concatenate([np.reshape(i[1],(-1,1)) for i in outputs],axis=1)\nprint('Corr in Train:')\nprint(pd.DataFrame(train_meta_data).corr())\nprint('Corr in Test')\nprint(pd.DataFrame(test_meta_data).corr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d302254a6077d3bcde4e31bdea4aac757ba4cf1"},"cell_type":"code","source":"coefs = [0.35,0.25,0.2,0.2]\npred_val_y = np.sum([outputs[i][0]*coefs[i] for i in range(len(outputs))], axis = 0)\nresults = threshold_search(val_y, pred_val_y)\nprint(results)\nprint(confusion_matrix(val_y,pred_val_y>results['threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"590a9174eaf14ed93add66c1681137202fb5df7f"},"cell_type":"code","source":"# pred_test_y = np.sum([outputs[i][1] * reg.coef_[i] for i in range(len(outputs))], axis = 0)\ncoefs = [0.35,0.25,0.2,0.2]\n# pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs))], axis = 0)\npred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs))], axis = 0)\n\npred_test_y = (pred_test_y > results['threshold']).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}