{"cells":[{"metadata":{"_uuid":"a0bf941fc0d4edbe5ad2bbadef80cbb4d5ac152a"},"cell_type":"markdown","source":"# Average\nThis kernel is based on the model(s) used in the other kernels for this competition - CNNs over several words - but with averaging. E.g. each CNN architecture is used as a separate model for prediction and then all predictions are averaged.\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Embedding, Dense, Dropout, Concatenate, Lambda, Flatten\nfrom keras.layers import GlobalMaxPool1D\nfrom keras.models import Model\n\n\nimport tqdm\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d4287da9f0fd92df1144c52c69484125726da6c"},"cell_type":"markdown","source":"# Embeddings"},{"metadata":{"_uuid":"a205fab8b96c97dd55ae127bd06808a4248c88f7","trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 45000\nEMBEDDINGS_TRAINED_DIMENSIONS = 100\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c2aac8432c093a9e141eed3fd72e37c9397c2a"},"cell_type":"markdown","source":"## Pretrained\nLoad (one of) the embeddings"},{"metadata":{"_uuid":"0651b1104f3aec8da85cdadac71442dd83617a8f","trusted":true},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file) as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for line in f)\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2258c40afb58bf7a45cf3226d5d73d77ba2149c"},"cell_type":"code","source":"pretrained_embeddings = load_embeddings(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"738d1e89d9f157aa954a1eb335139d0e907182dd"},"cell_type":"markdown","source":"# Data\nLoad the data."},{"metadata":{"_uuid":"c6c3339bb66c31947ac0f19a31d2e8afaf512d3e","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20dd051d411545c5d9f4fef74a281985bdcb04c4","trusted":true},"cell_type":"code","source":"BATCH_SIZE = 512\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\ntest_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"631ba595856aac8ddde49c4a0964bc131136a064","trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=MAX_WORDS)\ntokenizer.fit_on_texts(list(df_train[\"question_text\"].values))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75d212d515a7abec924b779cdde966f7655fbc00","trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        if word not in embeddings:\n            not_embedded[word] = not_embedded[word] + 1\n            continue\n        embedding_vector = embeddings[word]\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    print(sorted(not_embedded, key=not_embedded.get)[:10])\n    return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a16cebc3748a38b26726ecd5f41fa1fe732d1e5","trusted":true},"cell_type":"code","source":"pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a04e769dee4e3a0ebe350c9082ea5d38687a917c"},"cell_type":"markdown","source":"# Model\nConstruct the model to use, e.g. a simple NN"},{"metadata":{"_uuid":"7d5235d72f2cb1fdc24a81db9e2e3408e8fec1df"},"cell_type":"markdown","source":"# Model evaluation\n\n\n"},{"metadata":{"_uuid":"790dc08dcbb0c65fdf840819ee4b61c96fd100d2","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import metrics\n\nTHRESHOLD = 0.35\n\nclass EpochMetricsCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        self.precisions = []\n        self.recalls = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = metrics.f1_score(targets, predictions)\n        precision = metrics.precision_score(targets, predictions)\n        recall = metrics.recall_score(targets, predictions)\n\n        print(\" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}\"\n              .format(f1, precision, recall))\n        self.f1s.append(f1)\n        self.precisions.append(precision)\n        self.recalls.append(recall)\n        return\n    \ndef display_model_history(history):\n    data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})\n    ax = sns.lineplot(data=data, palette=\"pastel\", linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')\n    sns.despine()\n    plt.show()\n\ndef display_model_epoch_metrics(epoch_callback):\n    data = pd.DataFrame(data = {\n        'F1': epoch_callback.f1s,\n        'Precision': epoch_callback.precisions,\n        'Recall': epoch_callback.recalls})\n    sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\n    sns.despine()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e37b7b24d69b340fd238c8b4fa16c0e6561eab5b"},"cell_type":"markdown","source":"# Training\nTrain the model. Also, experiment with different versions"},{"metadata":{"_uuid":"89360ffd20dd998f54eb6c4e638201caf21df8a2"},"cell_type":"markdown","source":"## Prepare the data first\nE.g. the tokenized words"},{"metadata":{"_uuid":"24a65f899ac175ac793c1398448ce52130b520c6","trusted":true},"cell_type":"code","source":"X = pad_sequences(tokenizer.texts_to_sequences(question_texts),\n                        maxlen=MAX_SEQUENCE_LENGTH)\nY = question_targets\n\ntest_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts),\n                       maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"568f3bdff2829145383c4e261275f362a2bd5217"},"cell_type":"markdown","source":"## Alternative models"},{"metadata":{"trusted":true,"_uuid":"e2cf90badb9f66d302e9592875dc477aad3712d7"},"cell_type":"code","source":"from keras.layers import Conv1D, Conv2D, Reshape, MaxPool1D, MaxPool2D, BatchNormalization\n\ndef make_model(filter_size, num_filters):\n    tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tokenized_input\")\n    \n    pretrained = Embedding(MAX_WORDS,\n                           EMBEDDINGS_LOADED_DIMENSIONS,\n                           weights=[pretrained_emb_weights],\n                           trainable=False)(tokenized_input)\n\n    pretrained = Reshape((MAX_SEQUENCE_LENGTH, EMBEDDINGS_LOADED_DIMENSIONS, 1))(pretrained)\n    pretrained = Dropout(0.1)(pretrained)\n    conv_0 = Conv2D(num_filters, kernel_size=(filter_size, EMBEDDINGS_LOADED_DIMENSIONS),\n                    kernel_initializer='he_normal', activation='tanh')(pretrained)\n    maxpool_0 = MaxPool2D(pool_size=(MAX_SEQUENCE_LENGTH - filter_size + 1, 1))(conv_0)\n\n    d0 = Dropout(0.15)(maxpool_0)\n    d0 = Dense(10)(d0)\n\n    x = Flatten()(d0)\n    x = Dropout(0.3)(x)\n    x = BatchNormalization()(x)\n    out = Dense(1, activation=\"sigmoid\")(x)\n\n    model = Model(inputs=[tokenized_input], outputs=out)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9f6a55d479ac67bbb05e0267de04922b9fe4f34","trusted":true,"scrolled":true},"cell_type":"code","source":"import random\nfrom sklearn.model_selection import train_test_split\n\nfilter_sizes = [1, 2, 3, 5]\nnum_filters = 45\n\ntest_predictions = []\nkaggle_predictions = []\n\ntrain_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.025)\n\nfor f in filter_sizes:\n    print(\"CNN MODEL WITH FILTER OF SIZE {0}\".format(f))\n    epoch_callback = EpochMetricsCallback()\n    model = make_model(f, num_filters)\n    \n    x, val_x, y, val_y = train_test_split(train_X, train_Y, test_size=0.01)\n    history = model.fit(\n        x=x, y=y, validation_data=(val_x, val_y),\n        batch_size=512, epochs=23, callbacks=[epoch_callback], verbose=2)\n    display_model_history(history)\n    display_model_epoch_metrics(epoch_callback)\n    \n    kaggle_predictions.append(model.predict([test_word_tokens], batch_size=1024, verbose=2))\n    test_predictions.append(model.predict([test_X]))\n    \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2dbe359ff06904c47f9438a79145936fe8347fba"},"cell_type":"markdown","source":"# Results"},{"metadata":{"_uuid":"0691949cc07aadc8baaf11a0e3018853e87f9073","scrolled":true,"trusted":true},"cell_type":"code","source":"avg = np.average(kaggle_predictions, axis=0)\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (avg > THRESHOLD).astype(int) \ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f130554d307b31a68422e03716498eacf52306c","trusted":true},"cell_type":"code","source":"# Adjust the threshold\n\navg = np.average(test_predictions, axis=0)\nf1s = []\nprecisions = []\nrecalls = []\n\nTs = [x * 0.01 for x in range(0, 50)]\nfor t in Ts:\n    pred = (avg > t).astype(int)\n    f1s.append(metrics.f1_score(test_Y, pred))\n    precisions.append(metrics.precision_score(test_Y, pred))\n    recalls.append(metrics.recall_score(test_Y, pred))\n\n\nplt.plot(Ts, f1s)\nplt.plot(Ts, precisions)\nplt.plot(Ts, recalls)\nplt.title('Threshold levels')\nplt.ylabel('Value')\nplt.xlabel('Threshold')\nplt.legend(['F1', 'Precision', 'Recall'])\nplt.show()\n\nthresh = Ts[np.argmax(f1s)]\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))\n\nthresh = THRESHOLD\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33a18293ead982bd7a96b0fbd29669d2f8aff0c6"},"cell_type":"code","source":"\n\n# avg = np.average(kaggle_predictions, axis=0)\n# df_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\n# df_out['prediction'] = (avg > thresh).astype(int)\n# df_out.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"120148f98c59900b58d3e468ab99d95781e6bca4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}