{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn import metrics\n\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 45000\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80c188d28dac370261b2fba477f6f85f58756a94"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1485d03a810b6f433541f7cf8ab373be895bd139"},"cell_type":"code","source":"BATCH_SIZE = 512\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\ntest_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adc1bbdf0874d110e69a484191ad5becc34e09bc"},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file) as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for line in f)\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings\n\n%time pretrained_embeddings = load_embeddings(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac5ea1dbf060a2618c9bab850e6daabd4473ba5"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=MAX_WORDS)\n\n%time tokenizer.fit_on_texts(list(df_train[\"question_text\"].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c559b21937fd5eeef51b4f367b781bb6638de40a"},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        embedding_vector = embeddings.get(word)\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    return embeddings_matrix\n\npretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7fdf8d375e2fe22bf30aff536d8b753a073049"},"cell_type":"code","source":"THRESHOLD = 0.35\n\nclass EpochMetricsCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        self.precisions = []\n        self.recalls = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = metrics.f1_score(targets, predictions)\n        precision = metrics.precision_score(targets, predictions)\n        recall = metrics.recall_score(targets, predictions)\n\n        print(\" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}\"\n              .format(f1, precision, recall))\n        self.f1s.append(f1)\n        self.precisions.append(precision)\n        self.recalls.append(recall)\n        return\n    \ndef display_model_history(history):\n    data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})\n    ax = sns.lineplot(data=data, palette=\"pastel\", linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')\n    sns.despine()\n    plt.show()\n\ndef display_model_epoch_metrics(epoch_callback):   \n    data = pd.DataFrame(data = {\n        'F1': epoch_callback.f1s,\n        'Precision': epoch_callback.precisions,\n        'Recall': epoch_callback.recalls})\n    sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\n    sns.despine()\n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8b71f90c1a7a0e05d028ad83a29c99d1ae3f9f9"},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\n\n%time X = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)\n%time Y = question_targets\n\n%time test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b28f785b5a819fd7bec6a9a20f08946df068d470"},"cell_type":"code","source":"from keras.layers import Input, Embedding, Dense, Dropout, SpatialDropout1D, Flatten\nfrom keras.layers import LSTM, Bidirectional, GRU, BatchNormalization\nfrom keras.models import Model\n\ndef make_model():\n    tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tokenized_input\")\n    embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,\n                          weights=[pretrained_emb_weights],\n                          trainable=False)(tokenized_input)\n    \n    d0 = SpatialDropout1D(0.1)(embedding)\n    lstm = Bidirectional(LSTM(128, return_sequences=True))(d0)\n    lstm = Bidirectional(LSTM(64, return_sequences=False))(lstm)\n    d1 = Dropout(0.15)(lstm)\n    d1 = Dense(64)(d1)\n    b = BatchNormalization()(d1)\n    out = Dense(1, activation='sigmoid')(b)\n    \n    model = Model(inputs=[tokenized_input], outputs=out)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8358a2ed2c7b15931d6d133734a6ea41c34d2a4b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)\n\nepoch_callback = EpochMetricsCallback()\nmodel = make_model()\nhistory = model.fit(x=train_X, y=train_Y, validation_split=0.015,\n                    batch_size=BATCH_SIZE, epochs=4, verbose=2,\n                    callbacks=[epoch_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"238b056a0c1abc0cedbd0bd8a2285717eafb1275"},"cell_type":"code","source":"display_model_history(history)\ndisplay_model_epoch_metrics(epoch_callback)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a65ccd27d3b955cbde4e96c96c4329dff5bce350"},"cell_type":"code","source":"test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH)\nkaggle_predictions = (model.predict([test_word_tokens], batch_size=1024, verbose=2))\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (kaggle_predictions > THRESHOLD).astype(int) \ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}