{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Embedding, Dense, Dropout, Concatenate, Lambda, Flatten\nfrom keras.layers import GlobalMaxPool1D\nfrom keras.models import Model\n\n\nimport tqdm\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df7215ff9c344e3aa28bc8ff35518e5425da19e4"},"cell_type":"markdown","source":"# Combinations\nThis kernel would contain a combination of previousle tested models. For example, it may be useful to combine pretrained embeddings with ones that were trained on this particular datase."},{"metadata":{"_uuid":"0d4287da9f0fd92df1144c52c69484125726da6c"},"cell_type":"markdown","source":"# Embeddings"},{"metadata":{"trusted":true,"_uuid":"a205fab8b96c97dd55ae127bd06808a4248c88f7"},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 75000\nEMBEDDINGS_TRAINED_DIMENSIONS = 100\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a434bf33e5c0523dc1c904bb109df387bfe1360b"},"cell_type":"markdown","source":"## Custom\nTrain our own embeddings on the training data"},{"metadata":{"trusted":true,"_uuid":"30a9ed60c284d80fe4cb7cdfc97a3067700be7e3"},"cell_type":"code","source":"import gensim, logging\nfrom nltk.tokenize import sent_tokenize\n\nlogging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n\nclass SentenceGenerator(object):\n    def __init__(self, texts):\n        self.texts = texts\n    def __iter__(self):\n        for text in self.texts:\n            sentences = sent_tokenize(text)\n            for sent in sentences:\n                yield sent\n \n\ndef train_w2v(texts, epochs=5):\n    sent_gen = SentenceGenerator(texts)\n    model_path = \"quora_w2v\" +\\\n        f\"_{EMBEDDINGS_TRAINED_DIMENSIONS}dimenstions\" +\\\n        f\"_{str(epochs)}epochs\" +\\\n        f\"_{MAX_WORDS}words\" +\\\n        \".model\"\n\n    if (os.path.isfile(model_path)):\n        model = gensim.models.Word2Vec.load(model_path)\n        print(\"Word2Vec loaded from \" + model_path)\n    else:\n        model = gensim.models.Word2Vec(sent_gen, size=EMBEDDINGS_TRAINED_DIMENSIONS, workers=4, max_final_vocab=MAX_WORDS, iter=epochs)\n        model.save(model_path)\n        print(\"Word2Vec saved to \" + model_path)\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c2aac8432c093a9e141eed3fd72e37c9397c2a"},"cell_type":"markdown","source":"## Pretrained\nLoad (one of) the embeddings"},{"metadata":{"trusted":true,"_uuid":"0651b1104f3aec8da85cdadac71442dd83617a8f"},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file) as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for line in f if len(line)>100)\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1fad516489612efbaae7a7885dfec3173d32e0e"},"cell_type":"markdown","source":"# NLP Features\nFind Part of Speech tags and named entities in the questions. Tokenize them and use them later in the model."},{"metadata":{"trusted":true,"_uuid":"8460f2146cb1f0a693a068319a27815ff2572465"},"cell_type":"code","source":"import spacy\n\nnlp = spacy.load(\"en_core_web_sm\", disable=['parser'])\nnlp.add_pipe(nlp.create_pipe('sentencizer'))\nprint(f\"spaCy pipes: {nlp.pipe_names}\")\n\n\n# Find POS and NER tags\n# Entity types from https://spacy.io/api/annotation#named-entities\npos_tags = nlp.tokenizer.vocab.morphology.tag_map.keys()\npos_tags_count = len(pos_tags)\nentity_types = [\"PERSON\", \"NORP\", \"FAC\", \"ORG\", \"LOC\", \"PRODUCT\", \"EVENT\", \"WORK_OF_ART\", \"LAW\", \"LANGUAGE\", \"DATE\",\n                \"TIME\", \"PERCENT\", \"MONEY\", \"QUANTITY\", \"ORDINAL\", \"CARDINAL\"]\nentity_types_count = len(entity_types)\n\n\npos_tokenizer = Tokenizer(num_words=pos_tags_count, lower=False)\npos_tokenizer.fit_on_texts(pos_tags)\ndefault_filter_without_underscore = '!\"#$%&()*+,-./:;<=>?@[\\]^`{|}~'\nentity_tokenizer = Tokenizer(num_words=entity_types_count,\n                             lower=False, oov_token='0',\n                             filters=default_filter_without_underscore)\nentity_tokenizer.fit_on_texts(list(entity_types))\nentity_types_count = len(entity_tokenizer.index_word) + 1\n\ndef token_encoded_pos_getter(token):\n    if token.tag_ in pos_tokenizer.word_index:\n        return pos_tokenizer.word_index[token.tag_]\n    else:\n        return 0\n\ndef token_encoded_ent_getter(token):\n    if token.ent_type_ in entity_tokenizer.word_index:\n        return entity_tokenizer.word_index[token.ent_type_]\n    else:\n        return 0\n\nspacy.tokens.token.Token.set_extension('encoded_pos', force=True, getter=token_encoded_pos_getter)\nspacy.tokens.token.Token.set_extension('encoded_ent', force=True, getter=token_encoded_ent_getter)\nspacy.tokens.doc.Doc.set_extension('encoded_pos', force=True, getter=lambda doc: [token._.encoded_pos for token in doc])\nspacy.tokens.doc.Doc.set_extension('encoded_ent', force=True, getter=lambda doc: [token._.encoded_ent for token in doc])\n\ndef make_nlp_features(texts):\n    '''\n    A simple greedy function that generates one-hot encodings for the NLP features of each word in each question.\n    '''\n    pos_encodings = []\n    ent_encodings = []\n    for doc in tqdm.tqdm(nlp.pipe(texts, batch_size=100, n_threads=4), total=len(texts)):\n        pos_encodings.append(doc._.encoded_pos)\n        ent_encodings.append(doc._.encoded_ent)\n\n    pos_encodings = np.array(pos_encodings)\n    pos_encodings = pad_sequences(pos_encodings, maxlen=MAX_SEQUENCE_LENGTH)\n\n    ent_encodings = np.array(ent_encodings)\n    ent_encodings = pad_sequences(ent_encodings, maxlen=MAX_SEQUENCE_LENGTH)\n\n    return pos_encodings, ent_encodings","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"738d1e89d9f157aa954a1eb335139d0e907182dd"},"cell_type":"markdown","source":"# Data\nLoad the data."},{"metadata":{"trusted":true,"_uuid":"c6c3339bb66c31947ac0f19a31d2e8afaf512d3e"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20dd051d411545c5d9f4fef74a281985bdcb04c4"},"cell_type":"code","source":"BATCH_SIZE = 512\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\ntest_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"631ba595856aac8ddde49c4a0964bc131136a064"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=MAX_WORDS)\ntokenizer.fit_on_texts(list(df_train[\"question_text\"].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1ebb68e362da3d7018d13417e9de4ab6532bc9c"},"cell_type":"code","source":"custom_embeddings = train_w2v(question_texts, epochs=5)\npretrained_embeddings = load_embeddings(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75d212d515a7abec924b779cdde966f7655fbc00"},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        if word not in embeddings:\n            not_embedded[word] = not_embedded[word] + 1\n            continue\n        embedding_vector = embeddings[word]\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    print(sorted(not_embedded, key=not_embedded.get)[:10])\n    return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a16cebc3748a38b26726ecd5f41fa1fe732d1e5"},"cell_type":"code","source":"custom_emb_weights = create_embedding_weights(tokenizer, custom_embeddings, EMBEDDINGS_TRAINED_DIMENSIONS)\npretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a04e769dee4e3a0ebe350c9082ea5d38687a917c"},"cell_type":"markdown","source":"# Model\nConstruct the model to use, e.g. a simple NN"},{"metadata":{"trusted":true,"_uuid":"1b6d82892554ba2f276e19ce066a3c6cad3209ff"},"cell_type":"code","source":"from keras.layers import Conv1D, Conv2D, Reshape, MaxPool1D, MaxPool2D\n\nfilter_size = 5\nnum_filters = 45\n\ndef create_model(embeddings_weights):\n    tok_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tok_input\")\n    ent_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"ent_input\", dtype='uint8')\n    pos_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"pos_input\", dtype='uint8')\n\n    trained = Embedding(MAX_WORDS,\n                        EMBEDDINGS_TRAINED_DIMENSIONS,\n                        weights=[custom_emb_weights],\n                        trainable=True)(tok_input)\n    pretrained = Embedding(MAX_WORDS,\n                          EMBEDDINGS_LOADED_DIMENSIONS,\n                          weights=[pretrained_emb_weights],\n                          trainable=True)(tok_input)\n    \n    trained = GlobalMaxPool1D()(trained)\n    trained = Dropout(0.7)(trained)\n    trained = Dense(10)(trained)\n    pretrained = GlobalMaxPool1D()(pretrained)\n    pretrained = Dropout(0.7)(pretrained)\n    pretrained = Dense(10)(pretrained)\n    \n\n    x_ent = Lambda(\n        keras.backend.one_hot,\n        arguments={\"num_classes\": entity_types_count},\n        output_shape = (MAX_SEQUENCE_LENGTH, entity_types_count, 1))(ent_input)\n    x_ent = Reshape((MAX_SEQUENCE_LENGTH, entity_types_count, 1))(x_ent)\n    conv_0 = Conv2D(num_filters, kernel_size=(filter_size, entity_types_count), kernel_initializer='he_normal', activation='tanh')(x_ent)\n    maxpool_0 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_size + 1, 1))(conv_0)\n    x_ent = Flatten()(x_ent)\n    x_ent = Dropout(0.1)(x_ent)\n    x_ent = Dense(10)(x_ent)\n    \n    x_pos = Lambda(\n        keras.backend.one_hot,\n        arguments={\"num_classes\": pos_tags_count},\n        output_shape = (MAX_SEQUENCE_LENGTH, pos_tags_count))(pos_input)\n    x_pos = Reshape((MAX_SEQUENCE_LENGTH, pos_tags_count, 1))(x_pos)\n    conv_0 = Conv2D(num_filters, kernel_size=(filter_size, pos_tags_count), kernel_initializer='he_normal', activation='tanh')(x_pos)\n    maxpool_0 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_size + 1, 1))(conv_0)\n    x_pos = Flatten()(x_pos)\n    x_pos = Dropout(0.1)(x_pos)\n    x_pos = Dense(10)(x_pos)\n    \n    x = Concatenate()([trained, pretrained, x_pos, x_ent])\n    x = Dropout(0.7)(x)\n    x = Dense(10)(x)\n    x = Dropout(0.5)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n\n    model = Model(inputs=[tok_input, ent_input, pos_input], outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d5235d72f2cb1fdc24a81db9e2e3408e8fec1df"},"cell_type":"markdown","source":"# Model evaluation\n\n\n"},{"metadata":{"trusted":true,"_uuid":"790dc08dcbb0c65fdf840819ee4b61c96fd100d2"},"cell_type":"code","source":"import sklearn\nimport keras\nimport matplotlib.pyplot as plt\n\nTHRESHOLD = 0.35\n\nclass F1EpochCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        \n    def on_epoch_end(self, batch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = sklearn.metrics.f1_score(targets, predictions)\n        print(f\"validation_f1: {f1}\")\n        self.f1s.append(f1)\n        return\n    \ndef display_model_history(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper right')\n    plt.show()\n\ndef display_model_f1(f1_callback):\n    plt.plot(f1_callback.f1s)\n    plt.title('F1')\n    plt.ylabel('F1')\n    plt.xlabel('Epoch')\n    plt.legend(['F1 score'], loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e37b7b24d69b340fd238c8b4fa16c0e6561eab5b"},"cell_type":"markdown","source":"# Training\nTrain the model. Also, experiment with different versions"},{"metadata":{"_uuid":"89360ffd20dd998f54eb6c4e638201caf21df8a2"},"cell_type":"markdown","source":"## Prepare the data first\nE.g. the tokenized words as well as the nlp features"},{"metadata":{"trusted":true,"_uuid":"24a65f899ac175ac793c1398448ce52130b520c6"},"cell_type":"code","source":"(pos_encodings, ent_encodings) = make_nlp_features(question_texts)\n\ntrain_X = pad_sequences(tokenizer.texts_to_sequences(question_texts),\n                        maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9f6a55d479ac67bbb05e0267de04922b9fe4f34"},"cell_type":"code","source":"%%time\nmodel = create_model(custom_emb_weights)\nf1_callback = F1EpochCallback()\n#     x={\"pos_input\": pos_encodings, \"ent_input\": ent_encodings, \"tok_input\": train_X},\nhistory = model.fit(\n    x=[train_X, ent_encodings, pos_encodings],\n    y=question_targets,\n    batch_size=512, epochs=15, validation_split=0.015) #callbacks=[f1_callback],\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"515129fee79a56565b1963a792849ad887a08251"},"cell_type":"code","source":"display_model_history(history)\n# display_model_f1(f1_callback)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2dbe359ff06904c47f9438a79145936fe8347fba"},"cell_type":"markdown","source":"# Results"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0691949cc07aadc8baaf11a0e3018853e87f9073"},"cell_type":"code","source":"(test_pos_encodings, test_ent_encodings) = make_nlp_features(test_texts)\n\ntest_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts),\n                       maxlen=MAX_SEQUENCE_LENGTH)\n\npred_test = model.predict([test_word_tokens, test_ent_encodings, test_pos_encodings], batch_size=1024, verbose=1)\npred_test = (pred_test > THRESHOLD).astype(int)\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = pred_test\ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f130554d307b31a68422e03716498eacf52306c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}