{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# os.environ[\"KERAS_BACKEND\"] = \"plaidml.keras.backend\"\n\nimport keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Embedding, Dense, Dropout, Concatenate, Lambda, Flatten\nfrom keras.layers import GlobalMaxPool1D\nfrom keras.models import Model\n\n\nimport tqdm\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df7215ff9c344e3aa28bc8ff35518e5425da19e4"},"cell_type":"markdown","source":"# Combinations\nThis kernel would contain a combination of previousle tested models. For example, it may be useful to combine pretrained embeddings with ones that were trained on this particular datase."},{"metadata":{"_uuid":"0d4287da9f0fd92df1144c52c69484125726da6c"},"cell_type":"markdown","source":"# Embeddings"},{"metadata":{"trusted":true,"_uuid":"a205fab8b96c97dd55ae127bd06808a4248c88f7"},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 75000\nEMBEDDINGS_TRAINED_DIMENSIONS = 100\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a434bf33e5c0523dc1c904bb109df387bfe1360b"},"cell_type":"markdown","source":"## Custom\nTrain our own embeddings on the training data"},{"metadata":{"trusted":true,"_uuid":"30a9ed60c284d80fe4cb7cdfc97a3067700be7e3"},"cell_type":"code","source":"import gensim, logging\nfrom nltk.tokenize import sent_tokenize\n\nlogging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n\nclass SentenceGenerator(object):\n    def __init__(self, texts):\n        self.texts = texts\n    def __iter__(self):\n        for text in self.texts:\n            sentences = sent_tokenize(text)\n            for sent in sentences:\n                yield sent\n \n\ndef train_w2v(texts, epochs=5):\n    sent_gen = SentenceGenerator(texts)\n    model_path = \"quora_w2v\" +\\\n        f\"_{EMBEDDINGS_TRAINED_DIMENSIONS}dimenstions\" +\\\n        f\"_{str(epochs)}epochs\" +\\\n        f\"_{MAX_WORDS}words\" +\\\n        \".model\"\n\n    if (os.path.isfile(model_path)):\n        model = gensim.models.Word2Vec.load(model_path)\n        print(\"Word2Vec loaded from \" + model_path)\n    else:\n        model = gensim.models.Word2Vec(sent_gen, size=EMBEDDINGS_TRAINED_DIMENSIONS, workers=4, max_final_vocab=MAX_WORDS, iter=epochs)\n        model.save(model_path)\n        print(\"Word2Vec saved to \" + model_path)\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c2aac8432c093a9e141eed3fd72e37c9397c2a"},"cell_type":"markdown","source":"## Pretrained\nLoad (one of) the embeddings"},{"metadata":{"trusted":true,"_uuid":"0651b1104f3aec8da85cdadac71442dd83617a8f"},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file) as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for line in f if len(line)>100)\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"738d1e89d9f157aa954a1eb335139d0e907182dd"},"cell_type":"markdown","source":"# Data\nLoad the data."},{"metadata":{"trusted":true,"_uuid":"c6c3339bb66c31947ac0f19a31d2e8afaf512d3e"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20dd051d411545c5d9f4fef74a281985bdcb04c4"},"cell_type":"code","source":"BATCH_SIZE = 512\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\ntest_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"631ba595856aac8ddde49c4a0964bc131136a064"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=MAX_WORDS)\ntokenizer.fit_on_texts(list(df_train[\"question_text\"].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1ebb68e362da3d7018d13417e9de4ab6532bc9c"},"cell_type":"code","source":"custom_embeddings = train_w2v(question_texts, epochs=5)\npretrained_embeddings = load_embeddings(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75d212d515a7abec924b779cdde966f7655fbc00"},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        if word not in embeddings:\n            not_embedded[word] = not_embedded[word] + 1\n            continue\n        embedding_vector = embeddings[word]\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    print(sorted(not_embedded, key=not_embedded.get)[:10])\n    return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a16cebc3748a38b26726ecd5f41fa1fe732d1e5"},"cell_type":"code","source":"custom_emb_weights = create_embedding_weights(tokenizer, custom_embeddings, EMBEDDINGS_TRAINED_DIMENSIONS)\npretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a04e769dee4e3a0ebe350c9082ea5d38687a917c"},"cell_type":"markdown","source":"# Model\nConstruct the model to use, e.g. a simple NN"},{"metadata":{"trusted":true,"_uuid":"1b6d82892554ba2f276e19ce066a3c6cad3209ff"},"cell_type":"code","source":"from keras.layers import Conv2D, Reshape, MaxPool2D\n\nfilter_sizes = [1,2,3,5]\nnum_filters = 42\n\ndef create_model():\n    tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tokenized_input\")\n\n    trained = Embedding(MAX_WORDS,\n                        EMBEDDINGS_TRAINED_DIMENSIONS,\n                        weights=[custom_emb_weights],\n                        trainable=False)(tokenized_input)\n    \n    pretrained = Embedding(MAX_WORDS,\n                           EMBEDDINGS_LOADED_DIMENSIONS,\n                           weights=[pretrained_emb_weights],\n                           trainable=False)(tokenized_input)\n\n    trained = Reshape((MAX_SEQUENCE_LENGTH, EMBEDDINGS_TRAINED_DIMENSIONS, 1))(trained)\n    conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], EMBEDDINGS_TRAINED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(trained)\n    conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], EMBEDDINGS_TRAINED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(trained)\n    conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], EMBEDDINGS_TRAINED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(trained)\n    conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], EMBEDDINGS_TRAINED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(trained)\n    maxpool_0 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[0] + 1, 1))(conv_0)\n    maxpool_1 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[1] + 1, 1))(conv_1)\n    maxpool_2 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[2] + 1, 1))(conv_2)\n    maxpool_3 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[3] + 1, 1))(conv_3)\n    trained = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])\n    trained = Flatten()(trained)\n\n    pretrained = Reshape((MAX_SEQUENCE_LENGTH, EMBEDDINGS_LOADED_DIMENSIONS, 1))(pretrained)\n    conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], EMBEDDINGS_LOADED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(pretrained)\n    conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], EMBEDDINGS_LOADED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(pretrained)\n    conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], EMBEDDINGS_LOADED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(pretrained)\n    conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], EMBEDDINGS_LOADED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(pretrained)\n    maxpool_0 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[0] + 1, 1))(conv_0)\n    maxpool_1 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[1] + 1, 1))(conv_1)\n    maxpool_2 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[2] + 1, 1))(conv_2)\n    maxpool_3 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_sizes[3] + 1, 1))(conv_3)\n    pretrained = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])\n    pretrained = Flatten()(pretrained)\n    \n    x = Concatenate(axis=1)([pretrained, trained])\n    x = Dropout(0.7)(x)\n    out = Dense(1, activation=\"sigmoid\")(x)\n\n    model = Model(inputs=[tokenized_input], outputs=out)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d5235d72f2cb1fdc24a81db9e2e3408e8fec1df"},"cell_type":"markdown","source":"# Model evaluation\n\n\n"},{"metadata":{"trusted":true,"_uuid":"790dc08dcbb0c65fdf840819ee4b61c96fd100d2"},"cell_type":"code","source":"import sklearn\nimport keras\nimport matplotlib.pyplot as plt\n\nTHRESHOLD = 0.35\n\nclass F1EpochCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        \n    def on_epoch_end(self, batch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = sklearn.metrics.f1_score(targets, predictions)\n        print(f\"validation_f1: {f1}\")\n        self.f1s.append(f1)\n        return\n    \ndef display_model_history(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper right')\n    plt.show()\n\ndef display_model_f1(f1_callback):\n    plt.plot(f1_callback.f1s)\n    plt.title('F1')\n    plt.ylabel('F1')\n    plt.xlabel('Epoch')\n    plt.legend(['F1 score'], loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e37b7b24d69b340fd238c8b4fa16c0e6561eab5b"},"cell_type":"markdown","source":"# Training\nTrain the model. Also, experiment with different versions"},{"metadata":{"_uuid":"89360ffd20dd998f54eb6c4e638201caf21df8a2"},"cell_type":"markdown","source":"## Prepare the data first\nE.g. the tokenized words as well as the nlp features"},{"metadata":{"trusted":true,"_uuid":"24a65f899ac175ac793c1398448ce52130b520c6"},"cell_type":"code","source":"train_X = pad_sequences(tokenizer.texts_to_sequences(question_texts),\n                        maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9f6a55d479ac67bbb05e0267de04922b9fe4f34"},"cell_type":"code","source":"# %%time\nmodel = create_model()\nf1_callback = F1EpochCallback()\nhistory = model.fit(\n    x=[train_X],\n    y=question_targets,\n    batch_size=512, epochs=20, callbacks=[f1_callback], validation_split=0.015,\n    verbose=2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"515129fee79a56565b1963a792849ad887a08251"},"cell_type":"code","source":"display_model_history(history)\ndisplay_model_f1(f1_callback)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2dbe359ff06904c47f9438a79145936fe8347fba"},"cell_type":"markdown","source":"# Results"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0691949cc07aadc8baaf11a0e3018853e87f9073"},"cell_type":"code","source":"test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts),\n                       maxlen=MAX_SEQUENCE_LENGTH)\n\npred_test = model.predict([test_word_tokens], batch_size=1024, verbose=1)\npred_test = (pred_test > THRESHOLD).astype(int)\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = pred_test\ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f130554d307b31a68422e03716498eacf52306c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}