{"cells":[{"metadata":{"_uuid":"9f9a22f380d9529cfee7b613f50cde1050cf157d"},"cell_type":"markdown","source":"# Embeddings-keras-v04\nThis is a further improvement on v03 where I added preprocessing. In this kernel, I'll also add cross validation"},{"metadata":{"_uuid":"e53e5e18c8f3e287e4f2991ff80ba765d41208fe"},"cell_type":"markdown","source":"As usual, start with the imports..."},{"metadata":{"trusted":false,"_uuid":"69dc233e0deae38be0abf81b7f4d512d9c2f7c54"},"cell_type":"code","source":"import pandas as pd\nfrom collections import Counter\nimport numpy as np\nimport time\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\nimport operator\nimport re\nimport gc\n\n# We'll have to import all keras stuff here later\nfrom keras.layers import Bidirectional, Dense, Dropout, Embedding, CuDNNLSTM, CuDNNGRU, Input, GlobalMaxPool1D\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Model, load_model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras import optimizers\n\n# Can keras find a gpu?\nfrom keras import backend as K\nprint(K.tensorflow_backend._get_available_gpus())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"caed603f5adfa8f2822b45c21055115a93733783"},"cell_type":"code","source":"# Helper function that will be used in all cells!\ndef timeSince(t0):\n    ''' This function will be used to print the time since t0. \n        Will be called in every cell to give me some measurement. '''\n    print('Cell complete in {:.0f}m {:.0f}s'.format((time.time()-t0) // 60, (time.time()-t0) % 60))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94f0fea3d0c018b07405021cbf4964577c404f5d"},"cell_type":"markdown","source":"## Define constants"},{"metadata":{"trusted":false,"_uuid":"ab465dc46e057d4cb4425c62aa8c361218ef36da"},"cell_type":"code","source":"# Dataset path\n_traindataset = '../input/train.csv'\n_testdataset = '../input/test.csv'\n\n# Embeddings path\n_glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n_paragram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n_wiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n_google_news = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n\nembeddings = [{'name': 'glove', 'path': _glove},\n              {'name': 'paragram', 'path': _paragram},\n              {'name': 'fasttext', 'path': _wiki_news}]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d5be2129aa57ca6420a570cf24fb594466c1267"},"cell_type":"markdown","source":"Load dataset"},{"metadata":{"trusted":false,"_uuid":"24a06dd62dc51329456ea060d7c6e33ab26ebafe"},"cell_type":"code","source":"t0 = time.time()\ndf_train = pd.read_csv(_traindataset)\ndf_test = pd.read_csv(_testdataset)\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d69ce39aafc4603c19e1222dd007a397717a94b"},"cell_type":"markdown","source":"## Preprocessing\nStart data cleaning procedure!"},{"metadata":{"_uuid":"3e316436dca24b361946dcd8acbeac4361662cd3"},"cell_type":"markdown","source":"3. Lowercase the questions"},{"metadata":{"trusted":false,"_uuid":"f0e52f406273b97e3325f2ecbcd31e4bcca99f79"},"cell_type":"code","source":"t0 = time.time()\n\ndf_train['processed_questions'] = df_train['question_text'].fillna(\"_nan_\").apply(lambda x: x.lower())\ndf_test['processed_questions'] = df_test['question_text'].fillna(\"_nan_\").apply(lambda x: x.lower())\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7a703e76274e6440c1b22baf84c8329823d7b8d"},"cell_type":"markdown","source":"4. Remove contractions"},{"metadata":{"trusted":false,"_uuid":"afd7cc6d27892da5508d8d36d21f64070d5b78ca"},"cell_type":"code","source":"t0 = time.time()\n\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \n                       \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \n                       \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \n                       \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \n                       \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \n                       \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\n                       \"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \n                       \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \n                       \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \n                       \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \n                       \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\n                       \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \n                       \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n                       \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \n                       \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \n                       \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \n                       \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \n                       \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\n                       \"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \n                       \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \n                       \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n                       \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \n                       \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \n                       \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \n                       \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \n                       \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \n                       \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \n                       \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \n                       \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \n                       \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \n                       \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \n                       \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \n                       \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\n                       \"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \n                       \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \n                       \"you're\": \"you are\", \"you've\": \"you have\"}\n\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text\n\ndf_train['processed_questions'] = df_train['processed_questions'].apply(lambda x: \n                                                                    clean_contractions(x, \n                                                                                       contraction_mapping))\ndf_test['processed_questions'] = df_test['processed_questions'].apply(lambda x: \n                                                                  clean_contractions(x, \n                                                                                     contraction_mapping))\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29b68a64842f0af6ac740bdcdde603b679939b42"},"cell_type":"markdown","source":"5. Clean special characters such as punctuations etc"},{"metadata":{"trusted":false,"_uuid":"6b3bd560f41390200f8be9226548513cca32224f"},"cell_type":"code","source":"t0 = time.time()\n\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \n                 \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', \n                 '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', \n                 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text\n\ndf_train['processed_questions'] = df_train['processed_questions'].apply(lambda x: clean_special_chars(x, \n                                                                                      punct, \n                                                                                      punct_mapping))\ndf_test['processed_questions'] = df_test['processed_questions'].apply(lambda x: clean_special_chars(x, \n                                                                                      punct, \n                                                                                      punct_mapping))\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18d265d6478c2b2569bab3fba2bbc39f4e254284"},"cell_type":"markdown","source":"6. Remove other special chars using regex"},{"metadata":{"trusted":false,"_uuid":"cdfd7c32e499451cfda2cf6cf5fae7c20cea1860"},"cell_type":"code","source":"t0 = time.time()\n\n# Seems like this lowers the f1 score. Investigate further\n#df_train['processed_questions'] = df_train['processed_questions'].apply(lambda x: re.sub(r'[^\\x20-\\x7e]',r'', x))\n#df_test['processed_questions'] = df_test['processed_questions'].apply(lambda x: re.sub(r'[^\\x20-\\x7e]',r'', x))\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c85ac0a5f4c24e87bf91cc8ce734ee58836b1c24"},"cell_type":"markdown","source":"Create final preprocessed vocabulary so we know how many different words we now have.\nAlso create a tokenizer from this vocabulary"},{"metadata":{"trusted":false,"_uuid":"7fd5e7acbe694a9cf246ec2328dbb15fca72502e"},"cell_type":"code","source":"t0 = time.time()\n\ndef build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef vocab_to_integer(vocab):\n    ''' Map each vocab words to an integer.\n        Starts at 1 since 0 will be used for padding.'''\n    return {word: ii for ii, word in enumerate(vocab, 1)}\n    \n    \nall_questions = pd.concat([df_train['processed_questions'], df_test['processed_questions']])\nfinal_vocab = build_vocab(all_questions)\nword_to_idx = vocab_to_integer(final_vocab)\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b287b10e1de4a38b9ea1b6b46a5cecb30ba0bf2b"},"cell_type":"markdown","source":"## Preprocessing / data cleaning done!\nNow we crete a train/validation data split, tokenize and pad"},{"metadata":{"_uuid":"55fc02be4b6f668a36ffbb005b9da834f4ebe6b0"},"cell_type":"markdown","source":"### Define hyperparameters"},{"metadata":{"trusted":false,"_uuid":"50ba59551a1cab0b8a4cfbde5dc56aa39a9d8ae7"},"cell_type":"code","source":"t0 = time.time()\n\nhparam = {}\nhparam['VOCAB_SIZE'] = len(final_vocab) + 1\nhparam['PAD_LENGTH'] = 77\nhparam['MINIBATCH_SIZE'] = 512\nhparam['LEARNING_RATE'] = 1e-3\nhparam['EPOCHS'] = 4\nhparam['LSTM_HIDDEN_SIZE'] = 128\nhparam['WORD_EMB_DIM'] = 0 # This will be set when we concatenate the embeddings!\nhparam['KFOLDS'] = 10\n\n# To add\n# padding = pre\n# truncating = pre\n    \ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bdaa5343ddffaa85e22383ee36100340ae6a97b"},"cell_type":"markdown","source":"Build original vocab that we can use to add the missing lowercase words to each embedding"},{"metadata":{"trusted":false,"_uuid":"ef99f3f7767731972aa6f2e4fb8b7069ae21bcf7"},"cell_type":"code","source":"t0 = time.time()\n\n# Original vocab\nvocab_original = build_vocab(pd.concat([df_train['question_text'], df_test['question_text']]))\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba47bf4dff586e00b016f3673ac8b05ad706d724"},"cell_type":"markdown","source":"### Load and concatenate (stack) the embeddings"},{"metadata":{"_uuid":"57167cd6628052521580557102d7930b78f8f930"},"cell_type":"markdown","source":"Define functions to help us load embeddings etc..."},{"metadata":{"trusted":false,"_uuid":"2d87b57ac6935df74bbe563f3f1e82004fcc8cd2"},"cell_type":"code","source":"t0 = time.time()\n\ndef load_embed(file):\n    ''' Load the embedding from file '''\n    def get_coefs(word, *arr):\n        return word, np.asarray(arr, dtype='float32')\n    \n    if file.split('/')[-1] == 'wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o) > 100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index\n\ndef check_coverage(vocab, embeddings_index):\n    ''' Checks the coverate of a vocabulary in a given embedding.\n        Returns an array of out of vocab words (oov) '''\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n    print('{} iv words, {} unique'.format(nb_known_words, len(known_words)))\n    print('{} oov words, {} unique'.format(nb_unknown_words, len(unknown_words)))\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.3%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n    return unknown_words\n\ndef create_emb_matrix(nb_words, embed_size):\n    ''' Creates a initial random embedding matrix '''\n    # This is now zeroes which means that all words that doesn't have an embedding\n    # will be a zero vector... Before this was random.normalized! Maybe change back if score gets worse!\n    return np.zeros((nb_words, embed_size), dtype=np.float32)\n\ndef fill_emb_matrix(word_idx, emb_matrix, emb_index):\n    ''' Created a word2vec format matrix that we can use to embed our words '''\n    for word, i in word_idx:\n        emb_vector = emb_index.get(word)\n        if emb_vector is not None:\n            emb_matrix[i] = emb_vector\n    return emb_matrix\n\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07431f2377b65639de026b2a0b1aca3765a0f49f"},"cell_type":"markdown","source":"What we'll do next is to load the embeddings, one at the time.\nAdd missing lowercase words to the embedding.\nConvert it into a word2vec format that takes the word-unique integer and gives a vector.\nConcatenate the embedding with the previous embedding along axis 1.\n3 embeddings that each gives a vector of dim 300 -> final concatenated embedding\nwill thus give us a 900 dimensional vector for each word!\n\n( The above statement might not be true in this version! )"},{"metadata":{"trusted":false,"_uuid":"82b8dd569406c949011fc8eb4795ee26f52ec462"},"cell_type":"code","source":"t0 = time.time()\n\nconc_embedding = None # Concatenated embeddings will be saved as this variable\nword_index = word_to_idx\nnb_words = min(hparam['VOCAB_SIZE'], len(word_index) + 1) # this step should be unescessary???\nhparam['VOCAB_SIZE'] = nb_words\nprint(hparam['VOCAB_SIZE'], len(word_index) + 1)\nprint(f\"Got a vocab size of {nb_words} number of words\")\n\nfor embedding in embeddings:\n    emb_name = embedding['name']\n    emb_path = embedding['path']\n    print(\"Running procedure on {}\".format(emb_name))\n    \n    # Load embedding\n    print(\"Loading {}\".format(emb_name))\n    emb_index = load_embed(emb_path)\n    \n    # Add lowercase words to embedding\n    print(\"Adding lowercase to {}\".format(emb_name))\n    add_lower(emb_index, vocab_original)\n    \n    # Check OOV score\n    _ = check_coverage(final_vocab, emb_index)\n    \n    emb_size = 300\n    hparam['WORD_EMB_DIM'] += emb_size\n    \n    # Convert emb to word2vec format\n    emb_matrix = create_emb_matrix(nb_words, emb_size)\n    print(emb_matrix.size)\n    print(emb_matrix.shape)\n    emb_matrix = fill_emb_matrix(word_index.items(), emb_matrix, emb_index)\n    \n    # Save or concatenate this embedding with the previous embedding\n    if conc_embedding is not None:\n        conc_embedding = np.concatenate((conc_embedding, emb_matrix), axis=1)\n        print(\"concatenated! now got shape: {}\".format(conc_embedding.shape))\n        #conc_embedding += emb_matrix\n        #print(\"Added! now got shape: {}\".format(conc_embedding.shape))\n    else:\n        conc_embedding = emb_matrix\n    \n    # Memory management!\n    del emb_matrix, emb_index, emb_name, emb_path, emb_size\n    import gc; gc.collect()\n    \ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0585375f5757cf01f5c15573186ed1b729af8078"},"cell_type":"markdown","source":"### Embed all questions to word-unique integers & pad to same length"},{"metadata":{"trusted":false,"_uuid":"14753444861c9338369365935b09aa5e0cdd0b0b"},"cell_type":"code","source":"t0 = time.time()        \n\ndef embed_word_to_int(X, vocab_to_int):\n    embedded_X = []\n    for q in X:\n        tmp_X = []\n        for w in q.split():\n            tmp_X.append(vocab_to_int[w])\n        embedded_X.append(tmp_X)\n    return embedded_X\n\n# Embed each word as a unique integer\nX_train = embed_word_to_int(df_train['processed_questions'].values, word_to_idx)\nX_test = embed_word_to_int(df_test['processed_questions'].values, word_to_idx)\n\npad_length = hparam['PAD_LENGTH']\n\n# Pad the questions to the same length\nX_train_pad = pad_sequences(X_train, maxlen=pad_length, padding='pre', truncating='pre')\nX_test_pad = pad_sequences(X_test, maxlen=pad_length, padding='pre', truncating='pre')\n\n#print(\"y_train.shape: {}\".format(y_train.shape))\n#print(\"X_test.shape: {}\".format(X_test.shape))\n\nprint(df_train['processed_questions'][3333])\nprint(X_train[3333])\nprint(X_train_pad[3333])\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72b21635c4813d7c8544d814f006837e0ddf555b"},"cell_type":"markdown","source":"### Create cross validation pipeline!\nIn this kernel we'll use cross validation."},{"metadata":{"_uuid":"a89fa5382eae270ebbaf3d7db8f0ed04a4ad7915"},"cell_type":"markdown","source":"First we define helper functions that will be used in the cross validation loop"},{"metadata":{"trusted":false,"_uuid":"9be7fc4096bf48ed2df46a810e162ec477b32ffa"},"cell_type":"code","source":"t0 = time.time()\n\ndef train_val_pred(dataset, hparam, embedding_matrix):\n    ''' This function will train a model using some embedding matrix.\n        The prediction threshold will then be calculated based on the threshold that gives\n        the best f1 score on the validation data. The predictions on the test set will then\n        be returned along with the best calculated f1 score '''\n    \n    # Get data from dataset\n    X_train = dataset['X_train']\n    y_train = dataset['y_train']\n    X_val = dataset['X_val']\n    y_val = dataset['y_val']\n    X_test = dataset['X_test']\n\n    # Get hyperparameters\n    VOCAB_SIZE = hparam['VOCAB_SIZE']\n    PAD_LENGTH = hparam['PAD_LENGTH']\n    MINIBATCH_SIZE = hparam['MINIBATCH_SIZE']\n    LEARNING_RATE = hparam['LEARNING_RATE']\n    EPOCHS = hparam['EPOCHS']\n    LSTM_HIDDEN_SIZE = hparam['LSTM_HIDDEN_SIZE']\n    WORD_EMB_DIM = hparam['WORD_EMB_DIM']\n    \n    # Create the model\n    inp = Input(shape=(PAD_LENGTH,))\n    x = Embedding(VOCAB_SIZE, WORD_EMB_DIM, weights=[embedding_matrix], trainable=False)(inp)\n    # x = Bidirectional(CuDNNLSTM(LSTM_HIDDEN_SIZE, return_sequences=True))(x)\n    x = CuDNNLSTM(LSTM_HIDDEN_SIZE, return_sequences=True)(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(16, activation=\"relu\")(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    adam = optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n    model.summary()\n    \n    # Train model\n    model.fit(X_train, y_train, epochs=EPOCHS, batch_size=MINIBATCH_SIZE, \n          validation_data = (X_val, y_val))\n    \n    # Get threshold that gives best f1 score on validation set\n    val_preds = model.predict(X_val, batch_size=MINIBATCH_SIZE, verbose=1)\n    best_f1 = -1\n    best_thresh = -1\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        f1 = metrics.f1_score(y_val, (val_preds > thresh).astype(int))\n        if f1 > best_f1:\n            best_f1 = f1\n            best_thresh = thresh\n    print(\"Best f1 score = {} at tresh {}\".format(best_f1, best_thresh))\n    \n    # Get predictions on test set\n    test_preds = model.predict(X_test)\n    \n    # Some memory management!\n    del embedding_matrix, model, inp, x, adam\n    import gc; gc.collect()\n    \n    # Return predictions and the thresh that gave best f1 score on validation data\n    return test_preds, val_preds, best_thresh, best_f1\n    \ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05793c76fc5b7145f343ec20b72e32392d7709f7"},"cell_type":"markdown","source":"Create k-fold"},{"metadata":{"trusted":false,"_uuid":"fc71f4dbe4db45c08d0ae6560439624653870299"},"cell_type":"code","source":"t0 = time.time()\n\n# We use StratifiedKFold for iterating over the k-folds\nkfold = StratifiedKFold(n_splits=hparam['KFOLDS'], shuffle=True, random_state=2019)\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d822733f9a2033031e07fca8cf298796e3e53235"},"cell_type":"markdown","source":"Run cross validation"},{"metadata":{"trusted":false,"_uuid":"2750338dcbfff8b45bb4b519c32675a5f9da853b"},"cell_type":"code","source":"t0 = time.time()\n\nX = X_train_pad\ny = df_train['target'].values\n\nresults = [] # All results are saved in this list!\n\nfor fold, (train_index, val_index) in enumerate(kfold.split(X, y)):\n    X_train, X_val = X[train_index], X[val_index]\n    y_train, y_val = y[train_index], y[val_index]\n    \n    print(f\"Training on {len(X_train)} and validating on {len(X_val)} number of words\")\n    \n    # Create dataset placeholder\n    dataset = {'X_train': X_train, 'y_train': y_train,\n          'X_val': X_val, 'y_val': y_val,\n          'X_test': X_test_pad}\n    \n    # Run entire procedure and get predictions, best threshold and best f1 score\n    # Here we use the concatenated embedding with a dimension of 900!\n    test_preds, val_preds, thresh, f1 = train_val_pred(dataset, hparam, conc_embedding)\n    \n    print(\"len(test_preds) = {}, len(val_preds) = {}, thresh = {} at f1 = {}\".format(len(test_preds), \n                                                                                     len(val_preds), \n                                                                                     thresh, \n                                                                                     f1))\n    # Save into results\n    new_result = {'name': 'fold-' + str(fold), \n                  'test_preds': test_preds, \n                  'val_preds': val_preds, \n                  'thresh': thresh, \n                  'f1': f1}\n    results.append(new_result)\n    \n    # Memory management!\n    import gc; gc.collect()\n    \ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed8db104e5799a55a3e00a2a8b4aae4bd82a1424"},"cell_type":"markdown","source":"Get some statistics about our results"},{"metadata":{"trusted":false,"_uuid":"fd219f6651e1b25e3f24045f4fcef409a28bead3"},"cell_type":"code","source":"t0 = time.time()\n\nprint(\"Got {} number of results!\".format(len(results)))\navg_thresh = 0\nfor result in results:\n    print(\"{} gave f1 score {} with thresh {}\".format(result['name'], result['f1'], result['thresh']))\n    avg_thresh += result['thresh']\n\navg_thresh = avg_thresh / len(results)\nprint(\"Got an average threshold at {}\".format(avg_thresh))\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3930fc04200ea2ca2a8c2566cbb7854df01ddcd0"},"cell_type":"markdown","source":"Calculate the threshold that gives the best combined f1 score of all predictions"},{"metadata":{"trusted":false,"_uuid":"8a44cedc387157176b2a8c5205f6e4e28fb5c8e5"},"cell_type":"code","source":"'''\nt0 = time.time()\n\npred_val_y = None\nfactor = int(1.0 / len(results))\n\nfor result in results:\n    pred_val_y += factor * result['val_preds']\n\nbest_f1_combined = -1\nbest_thresh_combined = -1\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(dataset['y_val'], (pred_val_y > thresh).astype(int))\n    if f1 > best_f1_combined:\n        best_f1_combined = f1\n        best_thresh_combined = thresh\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    \nprint(\"Best f1 score = {} at tresh {}\".format(best_f1_combined, best_thresh_combined))\n\ntimeSince(t0)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a6c46b5c97c6c2dd98942ac0aac1feff7518f56"},"cell_type":"markdown","source":"Predict and save results!"},{"metadata":{"trusted":false,"_uuid":"28f9e73bc20104cf17b9ff40eb5c4fdcf72e08da"},"cell_type":"code","source":"t0 = time.time()\n\nprint(\"Using treshold {}\".format(avg_thresh))\n\nfactor = 1.0 / len(results)\npred_test_y = results[0]['test_preds'] * factor\n\nprint(\"Using factor: \", factor)\n\nfor i in range(1, len(results)):\n    pred_test_y += factor * results[i]['test_preds']\n    \n\npred_test_y_res = (pred_test_y > avg_thresh).astype(int)\n\nresults_dict = {'qid':df_test['qid'].values, 'prediction':[]}\n\nfor prediction in pred_test_y_res:\n    results_dict['prediction'].append(prediction[0])\n    \nprint(results_dict['qid'][:15])\nprint(results_dict['prediction'][:15])\n    \n# Save results\ndf = pd.DataFrame(data=results_dict)\ndf.to_csv('submission.csv', index=False)\nprint(\"Saved csv to disk!\")\n\ntimeSince(t0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b3b9095d83ae84c9bbc269ae1f206a648322691"},"cell_type":"markdown","source":"Pad the question per batch size instead of padding every question to the same length!\nThis is done to save runtime!\n\nThis was an attempt at padding on batch leve. Did not get it to work with the keras LSTM model so I'm skipping this for now and continuing with cross validation and statistical features. I will probably come back to this later."},{"metadata":{"trusted":false,"_uuid":"9442e4b8690021a34678b5d64672980a510767e4"},"cell_type":"code","source":"'''\nt0 = time.time()\n\ndef pad_per_batch(X, batch_size):\n    X_pad = []\n    print(f\"Length is {len(X)} and using batch size {batch_size}\")\n    e = 0\n    \n    max_found = 0\n    max_p_found = 0 # Keep stats for now\n    \n    for i in range(batch_size,len(X), batch_size):\n        # start and end for this minibatch\n        s = i-batch_size\n        e = i\n        batch = X[s:e]\n        \n        # calculate 98th percentile batch length\n        a = np.array([len(x) for x in batch])\n        p = int(np.percentile(a, 98)) # Get 98th percentile of all lengths on this batch!\n        m = int(np.max(a))\n        \n        # track stats for now\n        if m > max_found:\n            max_found = m\n        if p > max_p_found:\n            max_p_found = p\n        \n        padded_batch = pad_sequences(batch, maxlen=p)\n        for vec in padded_batch:\n            X_pad.append(vec)\n    \n    # Get the last batch as well!\n    last_batch = X[e:]\n    a = np.array([len(x) for x in last_batch])\n    p = int(np.percentile(a, 98)) # Get 98th percentile of all lengths on this batch!\n    \n    padded_batch = pad_sequences(last_batch, maxlen=p)\n    for vec in padded_batch:\n        X_pad.append(vec)\n        \n    print(f\"max p = {max_p_found} and max = {max_found}\")\n    print(f\"X_pad length = {len(X_pad)}, X length = {len(X)}\")\n    return np.asarray(X_pad), max_p_found\n        \n# Pad the questions per batch size\nX_train_pad, max_pad_len1 = pad_per_batch(X_train_token, hparam['MINIBATCH_SIZE'])\nX_val_pad, max_pad_len2 = pad_per_batch(X_val_token, hparam['MINIBATCH_SIZE'])\nX_test_pad, max_pad_len3 = pad_per_batch(X_test_token, hparam['MINIBATCH_SIZE'])\n\nmax_pad_len = max(max_pad_len1, max(max_pad_len2, max_pad_len3))\nhparam['PAD_LENGTH'] = max_pad_len\n\n# pack into placeholder\ndataset = {'X_train': X_train_pad, 'y_train': y_train,\n          'X_val': X_val_pad, 'y_val': y_val,\n          'X_test': X_test_pad}\n\nprint(\"X_train_pad.shape: {}\".format(X_train_pad.shape))\nprint(\"X_train_pad[3333].shape: {}\".format(X_train_pad[3333].shape))\nprint(\"y_train.shape: {}\".format(y_train.shape))\nprint(\"X_val_pad.shape: {}\".format(X_val_pad.shape))\nprint(\"y_val.shape: {}\".format(y_val.shape))\nprint(\"X_test_pad.shape: {}\".format(X_test_pad.shape))\n\nprint(X_train[3333])\nprint(X_train_token[3333])\nprint(X_train_pad[3333])\nprint(len(tokenizer.word_counts))\nprint(len(tokenizer.word_index))\n\ntimeSince(t0)\n'''","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}