{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "47f4c14e-ce61-8508-98dc-3476e71e459d"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "11f68ab6-9ffa-fec5-3f76-2d351c236692"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "5fa9104a-6208-4724-27dc-dd7ad4b48272"
      },
      "source": [
        "## Read Data ##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eca0de69-55e1-1cd8-137a-537e4d2d4c10"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv('../input/train.csv', encoding='utf-8')\n",
        "df_train['id'] = df_train['id'].apply(str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5c77b206-7b3c-c59b-e6d9-f84cb1eed3c5"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv('../input/test.csv', encoding='utf-8')\n",
        "df_test['test_id'] = df_test['test_id'].apply(str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "55e0097d-08c1-5bcf-42f4-32240ee34fbb"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train, df_test))\n",
        "df_all['question1'].fillna('', inplace=True)\n",
        "df_all['question2'].fillna('', inplace=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "94aaae1c-c081-b41d-d1fc-e8091406c9f6"
      },
      "source": [
        "## Create Vocab ##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eb60bcdc-0345-d72b-ebe2-611a964b8727"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import itertools"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0e2695cb-4e6f-e7af-4087-82acfd8229b2"
      },
      "outputs": [],
      "source": [
        "counts_vectorizer = CountVectorizer(max_features=10000-1).fit(\n",
        "    itertools.chain(df_all['question1'], df_all['question2']))\n",
        "other_index = len(counts_vectorizer.vocabulary_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f1114921-8527-437c-c980-30a04a0a9108"
      },
      "source": [
        "##Prep Data##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ca2f2087-8f6c-787e-5042-c5e379c1a131"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f22db32e-2d45-65fb-6847-59ad7020f854"
      },
      "outputs": [],
      "source": [
        "words_tokenizer = re.compile(counts_vectorizer.token_pattern)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1bafb51f-d0c1-1342-dc80-9225b028beec"
      },
      "outputs": [],
      "source": [
        "def create_padded_seqs(texts, max_len=10):\n",
        "    seqs = texts.apply(lambda s: \n",
        "        [counts_vectorizer.vocabulary_[w] if w in counts_vectorizer.vocabulary_ else other_index\n",
        "         for w in words_tokenizer.findall(s.lower())])\n",
        "    return pad_sequences(seqs, maxlen=max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8afdab6e-f05c-4e6c-0d6d-e395861b5d4e"
      },
      "outputs": [],
      "source": [
        "X1_train, X1_val, X2_train, X2_val, y_train, y_val = \\\n",
        "    train_test_split(create_padded_seqs(df_all[df_all['id'].notnull()]['question1']), \n",
        "                     create_padded_seqs(df_all[df_all['id'].notnull()]['question2']),\n",
        "                     df_all[df_all['id'].notnull()]['is_duplicate'].values,\n",
        "                     stratify=df_all[df_all['id'].notnull()]['is_duplicate'].values,\n",
        "                     test_size=0.3, random_state=1989)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "7e5e7c17-a66a-acd8-3bc2-a5b4c643f5b4"
      },
      "source": [
        "##Training##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b9da33a1-8f1c-c6a2-e1e7-40656be2cbe7"
      },
      "outputs": [],
      "source": [
        "import keras.layers as lyr\n",
        "from keras.models import Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1becec26-2df1-b38c-afc5-6a053ab47382"
      },
      "outputs": [],
      "source": [
        "input1_tensor = lyr.Input(X1_train.shape[1:])\n",
        "input2_tensor = lyr.Input(X2_train.shape[1:])\n",
        "\n",
        "words_embedding_layer = lyr.Embedding(X1_train.max() + 1, 100)\n",
        "seq_embedding_layer = lyr.LSTM(256, activation='tanh')\n",
        "\n",
        "seq_embedding = lambda tensor: seq_embedding_layer(words_embedding_layer(tensor))\n",
        "\n",
        "merge_layer = lyr.multiply([seq_embedding(input1_tensor), seq_embedding(input2_tensor)])\n",
        "\n",
        "dense1_layer = lyr.Dense(16, activation='sigmoid')(merge_layer)\n",
        "ouput_layer = lyr.Dense(1, activation='sigmoid')(dense1_layer)\n",
        "\n",
        "model = Model([input1_tensor, input2_tensor], ouput_layer)\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam')\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "852640d6-6318-2cf6-1b1e-6409f36ec951"
      },
      "outputs": [],
      "source": [
        "model.fit([X1_train, X2_train], y_train, \n",
        "          validation_data=([X1_val, X2_val], y_val), \n",
        "          batch_size=128, epochs=6, verbose=2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b5d7e836-cd1a-05cb-7741-d967a2bf34a1"
      },
      "source": [
        "##Extract Features From Model##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "37dbfa10-9aa3-fe0d-95f2-8b1f2aeda32f"
      },
      "outputs": [],
      "source": [
        "features_model = Model([input1_tensor, input2_tensor], merge_layer)\n",
        "features_model.compile(loss='mse', optimizer='adam')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a59d052f-1eca-7693-cafe-ff4b899e469b"
      },
      "outputs": [],
      "source": [
        "F_train = features_model.predict([X1_train, X2_train], batch_size=128)\n",
        "F_val = features_model.predict([X1_val, X2_val], batch_size=128)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d47649ca-f55a-1528-86a1-bc465b85189d"
      },
      "source": [
        "##Train XGBoost##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a9e938f7-aa89-a0e0-298c-721763c198ae"
      },
      "outputs": [],
      "source": [
        "import xgboost as xgb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "46f28c7b-318e-fb0c-49d3-fa91b1967086"
      },
      "outputs": [],
      "source": [
        "dTrain = xgb.DMatrix(F_train, label=y_train)\n",
        "dVal = xgb.DMatrix(F_val, label=y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "36498229-1969-8424-79c9-6e7e9ddd7ba3"
      },
      "outputs": [],
      "source": [
        "xgb_params = {\n",
        "    'objective': 'binary:logistic',\n",
        "    'booster': 'gbtree',\n",
        "    'eval_metric': 'logloss',\n",
        "    'eta': 0.1, \n",
        "    'max_depth': 9,\n",
        "    'subsample': 0.9,\n",
        "    'colsample_bytree': 1 / F_train.shape[1]**0.5,\n",
        "    'min_child_weight': 5,\n",
        "    'silent': 1\n",
        "}\n",
        "bst = xgb.train(xgb_params, dTrain, 1000,  [(dTrain,'train'), (dVal,'val')], \n",
        "                verbose_eval=10, early_stopping_rounds=10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "eb0f298c-7513-d3df-0947-aa8ca6d5bd87"
      },
      "source": [
        "##Predict Test##"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a45ad89e-d81d-26a5-48e3-4c7e7487dd5a"
      },
      "outputs": [],
      "source": [
        "X1_test = create_padded_seqs(df_all[df_all['test_id'].notnull()]['question1'])\n",
        "X2_test = create_padded_seqs(df_all[df_all['test_id'].notnull()]['question2'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7ebf03ed-1ccf-22f5-c296-1b307adb5de8"
      },
      "outputs": [],
      "source": [
        "F_test = features_model.predict([X1_test, X2_test], batch_size=128)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "226069f1-e7e6-2559-593f-80b7326e7c1c"
      },
      "outputs": [],
      "source": [
        "dTest = xgb.DMatrix(F_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "832684e5-e1d1-235b-bcfc-52e0c19832fa"
      },
      "outputs": [],
      "source": [
        "df_sub = pd.DataFrame({\n",
        "        'test_id': df_all[df_all['test_id'].notnull()]['test_id'].values,\n",
        "        'is_duplicate': bst.predict(dTest, ntree_limit=bst.best_ntree_limit)\n",
        "    }).set_index('test_id')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "119ad0de-8aea-b97a-7816-10b671d8fbc9"
      },
      "outputs": [],
      "source": [
        "df_sub.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "89c55e55-46f4-8728-bd4e-7abff5ea9b9a"
      },
      "outputs": [],
      "source": [
        "df_sub['is_duplicate'].hist(bins=100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f48ad5db-a1d7-b97a-8052-269e7ce43160"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}