{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "345614ff-11bc-3491-73f8-368565bc374f"
      },
      "source": [
        "**I found that the train dataset is very noisy. How should we tackle this problem ?**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b6a11b7b-8e3c-4ce8-5dee-862aec54dd8f"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import stem\n",
        "from sklearn.externals.joblib import Parallel, delayed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e26ed6c4-24ea-fb8a-6d1e-fd1ec1cdac62"
      },
      "outputs": [],
      "source": [
        "def tanimoto_coefficient(words1, words2):\n",
        "    try:\n",
        "        res = len(words1 & words2) / (len(words1) + len(words2) - len(words1 & words2))\n",
        "    except ZeroDivisionError:\n",
        "        res = 0\n",
        "    return res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a1874528-3f2f-0c0d-7f7b-fa3f94e7bce3"
      },
      "outputs": [],
      "source": [
        "stops = set(stopwords.words(\"english\"))\n",
        "stemmer = stem.LancasterStemmer()\n",
        "lemmatizer = stem.wordnet.WordNetLemmatizer()\n",
        "def st2words(st):\n",
        "    st = str(st)\n",
        "    st = re.sub(r'\\?|\\.|\\,|\\(|\\)|\uff0d|\\'|\\\"', \" \", st)\n",
        "    words = [w for w in st.split() if w != \"\"]\n",
        "    words = [stemmer.stem(w) if w != w.upper() else w for w in words]\n",
        "    words = [w.lower()  for w in st.split()]\n",
        "    words = [lemmatizer.lemmatize(w) for w in words]\n",
        "    words = [w for w in words if w not in stops]\n",
        "    return set(words)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "64cd819f-9f06-34c6-7fd0-39206a45be17"
      },
      "outputs": [],
      "source": [
        "def tanimoto_coefficient_from_st(st1, st2):\n",
        "    words1 = st2words(st1)\n",
        "    words2 = st2words(st2)\n",
        "    return tanimoto_coefficient(words1, words2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "096e2ab0-9715-1344-e76c-d17e076b818e"
      },
      "outputs": [],
      "source": [
        "train = pd.read_csv(\"../input/train.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "48499015-29ea-b82f-217d-55cb3b069829"
      },
      "outputs": [],
      "source": [
        "train[\"R\"] = Parallel(n_jobs=-1, verbose=2)([delayed(tanimoto_coefficient_from_st)(row[0], row[1]) for row in train[[\"question1\", \"question2\"]].values])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0c176989-5f65-5df7-da26-81ee5aebc6ff"
      },
      "outputs": [],
      "source": [
        "train[train.is_duplicate == 0].sort([\"R\"], ascending=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "1bc9eda4-34fb-2a45-a4c9-9f0d098dbdf4"
      },
      "source": [
        "This is the result of sorting unduplicated records with tanimoto coefficient. I think there are many duplicated records but is_duplicate = 0. It is mentioned that the dataset is not 100% accurate but how should we tackle this problem ?"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}