{"cells":[{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":478},"colab_type":"code","id":"lXDaiuSn5sQg","outputId":"8e0d25f6-47f8-436d-cdef-3dcdb40c6632","trusted":false},"cell_type":"code","source":"!pip3 install spacy\n!pip3 install nltk","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":95,"resources":{"http://localhost:8080/nbextensions/google.colab/files.js":{"data":"Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=","headers":[["content-type","application/javascript"]],"ok":true,"status":200,"status_text":"OK"}}},"colab_type":"code","id":"BSD1-fDH6UP8","outputId":"7e3ee679-9384-4346-c9a4-75fa0ae8d015","trusted":false},"cell_type":"code","source":"\nfrom google.colab import files\nfiles.upload()","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":239},"colab_type":"code","id":"L7WB_jPn_lWf","outputId":"3f485d9e-a1aa-441e-9555-db3b5258cbe9","trusted":false},"cell_type":"code","source":"!pip3 install kaggle","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"WKcbmoU9_v0M","trusted":false},"cell_type":"code","source":"!mkdir -p ~/.kaggle\n!cp kaggle.json ~/.kaggle/","execution_count":0,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":276},"colab_type":"code","id":"xiZS8LI3ArSl","outputId":"eb4c8fdc-9cfa-469c-949d-90cedb34eb10","trusted":false},"cell_type":"code","source":"!kaggle competitions download -c quora-insincere-questions-classification","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":135},"colab_type":"code","id":"kwzMHhHuXTcH","outputId":"3a2e63cb-7a0e-497e-c387-1a6baab1a889","trusted":false},"cell_type":"code","source":"chmod 600 /root/.kaggle/kaggle.json","execution_count":0,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"rz0K-PMfbDfq","trusted":false},"cell_type":"code","source":"zip=ZipFile('train.csv.zip', 'r')\nzip.extractall()","execution_count":0,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"cxt-zXTggXTi","trusted":false},"cell_type":"code","source":"zip=ZipFile('test.csv.zip', 'r')\nzip.extractall()","execution_count":0,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"z7C3glhY6D0D","trusted":true},"cell_type":"code","source":"import spacy\nimport nltk\nnlp=spacy.load('en')\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport string\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport re\nfrom collections import Counter\nfrom zipfile import ZipFile\nimport gensim\nfrom gensim import corpora, models, similarities\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix","execution_count":26,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"ulCiiKCmBwDu","trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/train.csv')\ntest=pd.read_csv('../input/test.csv')","execution_count":27,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"colab_type":"code","id":"ir8mxS7GI99G","outputId":"e92d6009-a137-49aa-8f13-4e8b5e5cc9a3","trusted":true},"cell_type":"code","source":"train.head()","execution_count":28,"outputs":[{"output_type":"execute_result","execution_count":28,"data":{"text/plain":" qid ... target\n0 00002165364db923c7e6 ... 0\n1 000032939017120e6e44 ... 0\n2 0000412ca6e4628ce2cf ... 0\n3 000042bf85aa498cd78e ... 0\n4 0000455dfa3e01eae3af ... 0\n\n[5 rows x 3 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidquestion_texttarget
000002165364db923c7e6How did Quebec nationalists see their province...0
1000032939017120e6e44Do you have an adopted dog, how would you enco...0
20000412ca6e4628ce2cfWhy does velocity affect time? Does velocity a...0
3000042bf85aa498cd78eHow did Otto von Guericke used the Magdeburg h...0
40000455dfa3e01eae3afCan I convert montra helicon D to a mountain b...0
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"0Urra0FfsWZP"},"cell_type":"markdown","source":"\n##Preprocessing"},{"metadata":{"colab":{},"colab_type":"code","id":"t3oGx7J2aU13","trusted":true},"cell_type":"code","source":"from sklearn.utils import resample\n\nsincere = train[train.target == 0]\ninsincere = train[train.target == 1]\n\ntrain = pd.concat([resample(sincere,\n replace = False,\n n_samples = len(insincere)), insincere])","execution_count":29,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"CCIOe_UxaU8Q","trusted":true},"cell_type":"code","source":"contractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n def replace(match):\n return contractions[match.group(0)]\n return c_re.sub(replace, text)","execution_count":30,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":127},"colab_type":"code","id":"2eVfZtOBaU-u","outputId":"2f0ad42c-debc-4f62-b61e-ea70b1203974","trusted":true},"cell_type":"code","source":"from gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n strip_tags, # remove html tags\n strip_punctuation, # replace punctuation with space\n strip_multiple_whitespaces,# remove repeating whitespaces\n strip_non_alphanum, # remove non-alphanumeric characters\n strip_numeric, # remove numbers\n remove_stopwords,# remove stopwords\n strip_short # remove words less than minsize=3 characters long\n ]\n\ndef gensim_preprocess(docs, logging=True):\n docs = [expandContractions(doc) for doc in docs]\n docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n texts_out = []\n for doc in docs:\n # https://spacy.io/usage/processing-pipelines\n doc = nlp((\" \".join(doc)), # doc = text to tokenize => creates doc\n # disable parts of the language processing pipeline we don't need here to speed up processing\n disable=['ner', # named entity recognition\n 'tagger', # part-of-speech tagger\n 'textcat', # document label categorizer\n ])\n texts_out.append([tok.lemma_ for tok in doc if tok.lemma_ != '-PRON-'])\n return pd.Series(texts_out)\n\ngensim_preprocess(train.question_text.iloc[10:15])","execution_count":31,"outputs":[{"output_type":"execute_result","execution_count":31,"data":{"text/plain":"0 [old, old, finance, economics, phd]\n1 [well, procedure, learn, web, development]\n2 [sociopath, know, sociopath, think, normal]\n3 [nice, thing, customer, say]\n4 [machine, use, insulator]\ndtype: object"},"metadata":{}}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":54},"colab_type":"code","id":"mOt6cafBaVBu","outputId":"5ecaf1f3-6ca0-494a-cb5e-7b33faa64460","trusted":true},"cell_type":"code","source":"%time train_corpus = gensim_preprocess(train.question_text)","execution_count":32,"outputs":[{"output_type":"stream","text":"CPU times: user 8min 11s, sys: 1.09 s, total: 8min 13s\nWall time: 8min 12s\n","name":"stdout"}]},{"metadata":{"colab_type":"text","id":"7p8NP-qOs5t5"},"cell_type":"markdown","source":"## N-Gramming"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":55},"colab_type":"code","id":"toj4-nVcaVSL","outputId":"36034f07-0718-4c02-9e94-b4af421c8dcb","trusted":true},"cell_type":"code","source":"# create ngrams\nngram_phraser = models.Phrases(train_corpus, threshold=1)\nngram = models.phrases.Phraser(ngram_phraser)\n#print example\nprint(ngram[train_corpus[0]])\n\n# apply model to corpus\ntexts = [ngram[token] for token in train_corpus]","execution_count":33,"outputs":[{"output_type":"stream","text":"['halfway', 'point', 'short', 'length', 'possible', 'size', 'observable', 'universe']\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":276},"colab_type":"code","id":"N9tBYU-AaVVW","outputId":"9263d011-2f63-4f7c-d997-642d66ffd406","trusted":true},"cell_type":"code","source":"# preparing ngrams for modeling\ntexts = [' '.join(text) for text in texts]\ntrain['ngrams'] = texts\ntrain.head()","execution_count":34,"outputs":[{"output_type":"execute_result","execution_count":34,"data":{"text/plain":" qid ... ngrams\n857763 a80f381e7bf5a2eb7189 ... halfway point short length possible size obser...\n79319 0f87a3371bc5c1369076 ... mark_zuckerberg vote_donald trump\n471047 5c3c9746832a7ad8f500 ... get_rank jee_main well_option look nit\n522474 6646d843d517c6e9356d ... like esrb\n1058289 cf5f171bd38b76ef025b ... important guy_girl beauty brilliance\n\n[5 rows x 4 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidquestion_texttargetngrams
857763a80f381e7bf5a2eb7189What is the halfway point between the shortest...0halfway point short length possible size obser...
793190f87a3371bc5c1369076Why did Mark Zuckerberg vote for Donald Trump?0mark_zuckerberg vote_donald trump
4710475c3c9746832a7ad8f500I've got a rank of 15k in JEE Mains 2017. What...0get_rank jee_main well_option look nit
5224746646d843d517c6e9356dWhat don't you like about the ESRB?0like esrb
1058289cf5f171bd38b76ef025bWhat's important for a guy in his girl - beaut...0important guy_girl beauty brilliance
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"3oIRCvRFszT4"},"cell_type":"markdown","source":"##Logistic Regression Baseline Model"},{"metadata":{"colab":{},"colab_type":"code","id":"Xxrx2DR5aU5j","trusted":true},"cell_type":"code","source":"# represent features as BOW\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer = CountVectorizer()\nvectorizer.fit(train.ngrams)\n\n# split into test and train sets\nX_train, X_test, y_train, y_test = train_test_split(train.ngrams, train.target, test_size=0.2)","execution_count":35,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":92},"colab_type":"code","id":"K-9WmDVigWPq","outputId":"4afb9609-bbbc-47b3-d496-7dd5c1ebe0d7","trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlr = LogisticRegression()\nlr.fit(vectorizer.transform(X_train), y_train)\n\nprint('Logistic Regression Score: ', lr.score(vectorizer.transform(X_test), y_test))","execution_count":36,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n FutureWarning)\n","name":"stderr"},{"output_type":"stream","text":"Logistic Regression Score: 0.8695396609330528\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":182},"colab_type":"code","id":"rmTK0pVasFjE","outputId":"03f93c78-5301-4eda-99a3-f61727ca4b5f","trusted":true},"cell_type":"code","source":"y_ = lr.predict(vectorizer.transform(X_test))\n\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test, y_))","execution_count":37,"outputs":[{"output_type":"stream","text":" precision recall f1-score support\n\n 0 0.86 0.89 0.87 16180\n 1 0.88 0.85 0.87 16144\n\n micro avg 0.87 0.87 0.87 32324\n macro avg 0.87 0.87 0.87 32324\nweighted avg 0.87 0.87 0.87 32324\n\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"colab_type":"code","id":"LPjlgQS9sFwg","outputId":"d3530dbe-78cd-4332-b889-e124d6ad7001","trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\npd.DataFrame(confusion_matrix(y_test, y_))","execution_count":38,"outputs":[{"output_type":"execute_result","execution_count":38,"data":{"text/plain":" 0 1\n0 14392 1788\n1 2429 13715","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01
0143921788
1242913715
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"uJ2HiKiasMEH"},"cell_type":"markdown","source":"##Bernoulli Naive Bayes Model"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":72},"colab_type":"code","id":"NCOSIJV6vBSc","outputId":"825da444-ebef-4b21-b2b2-5b797fe1be14","trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import BernoulliNB\n\nbnb = BernoulliNB()\n%time bnb.fit(vectorizer.transform(X_train), y_train)\n\nprint('Naive Bayes Score: ', bnb.score(vectorizer.transform(X_test), y_test))","execution_count":39,"outputs":[{"output_type":"stream","text":"CPU times: user 1.19 s, sys: 0 ns, total: 1.19 s\nWall time: 1.2 s\nNaive Bayes Score: 0.865796312337582\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":219},"colab_type":"code","id":"4ioCR5vQvCzI","outputId":"c5e74a98-91f9-4d29-98b1-24c761471729","trusted":true},"cell_type":"code","source":"%time bnb_y_ = bnb.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, bnb_y_))","execution_count":40,"outputs":[{"output_type":"stream","text":"CPU times: user 304 ms, sys: 0 ns, total: 304 ms\nWall time: 305 ms\n precision recall f1-score support\n\n 0 0.88 0.85 0.86 16180\n 1 0.85 0.89 0.87 16144\n\n micro avg 0.87 0.87 0.87 32324\n macro avg 0.87 0.87 0.87 32324\nweighted avg 0.87 0.87 0.87 32324\n\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"colab_type":"code","id":"FOVS1mN3sF40","outputId":"7c48b4e0-0899-43b9-8220-ebdbd2402988","trusted":true},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, bnb_y_))","execution_count":41,"outputs":[{"output_type":"execute_result","execution_count":41,"data":{"text/plain":" 0 1\n0 13681 2499\n1 1839 14305","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01
0136812499
1183914305
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"sKQmuj96swFm"},"cell_type":"markdown","source":"##XG Boost"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"colab_type":"code","id":"5gb5rh_rsvEc","outputId":"0d6ee213-8df9-4d50-b2c8-19dd90c64544","trusted":true},"cell_type":"code","source":"import xgboost as xgb\n\nxgb_model = xgb.XGBClassifier().fit(vectorizer.transform(X_train), y_train)\n\nprint('XGBoost Score: ', xgb_model.score(vectorizer.transform(X_test), y_test))\n\n","execution_count":42,"outputs":[{"output_type":"stream","text":"XGBoost Score: 0.7015530256156416\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":182},"colab_type":"code","id":"CY-WY2o3su3p","outputId":"c1c45a2d-f195-4e23-c5f8-763ca1909579","trusted":true},"cell_type":"code","source":"xgb_y_ = xgb_model.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, xgb_y_))","execution_count":43,"outputs":[{"output_type":"stream","text":" precision recall f1-score support\n\n 0 0.64 0.93 0.76 16180\n 1 0.87 0.48 0.61 16144\n\n micro avg 0.70 0.70 0.70 32324\n macro avg 0.75 0.70 0.69 32324\nweighted avg 0.75 0.70 0.69 32324\n\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"colab_type":"code","id":"dnLI57rPsuoD","outputId":"1ee35366-8885-47bf-c656-1237f038477d","trusted":true},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, xgb_y_))","execution_count":44,"outputs":[{"output_type":"execute_result","execution_count":44,"data":{"text/plain":" 0 1\n0 14987 1193\n1 8454 7690","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01
0149871193
184547690
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"LxhRbMTbtB_U"},"cell_type":"markdown","source":"## Ensemble Model"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":129},"colab_type":"code","id":"nEem9kM9suJL","outputId":"811c55a9-d226-4083-c91e-5f3e6293ad4f","trusted":true},"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\n\n#create submodels\nestimators = []\n\nmodel1 = lr\nmodel2 = bnb\nmodel3 = xgb_model\n\n\nestimators.append(('logistic', model1))\nestimators.append(('bernoulli', model2))\nestimators.append(('xgboost', model3))\n\n\n# create ensemble model\n%time ensemble = VotingClassifier(estimators).fit(vectorizer.transform(X_train), y_train)\nprint('Ensemble Score: ', ensemble.score(vectorizer.transform(X_test), y_test))","execution_count":45,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n FutureWarning)\n","name":"stderr"},{"output_type":"stream","text":"CPU times: user 27 s, sys: 52 ms, total: 27.1 s\nWall time: 27.1 s\nEnsemble Score: 0.8671575300086622\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":182},"colab_type":"code","id":"NB2ogXF8sF_i","outputId":"a1d35c03-2699-41f1-ed31-aa48979d33e9","trusted":true},"cell_type":"code","source":"ensemble_y_ = ensemble.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, ensemble_y_))\n","execution_count":46,"outputs":[{"output_type":"stream","text":" precision recall f1-score support\n\n 0 0.85 0.89 0.87 16180\n 1 0.89 0.84 0.86 16144\n\n micro avg 0.87 0.87 0.87 32324\n macro avg 0.87 0.87 0.87 32324\nweighted avg 0.87 0.87 0.87 32324\n\n","name":"stdout"}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"colab_type":"code","id":"KpLtD225tRfM","outputId":"b4fedae8-2ba2-4310-aab6-7fb1114ae84a","trusted":true},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, ensemble_y_))","execution_count":47,"outputs":[{"output_type":"execute_result","execution_count":47,"data":{"text/plain":" 0 1\n0 14445 1735\n1 2559 13585","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
01
0144451735
1255913585
\n
"},"metadata":{}}]},{"metadata":{"colab_type":"text","id":"auVcui52v6Rc"},"cell_type":"markdown","source":"##Submission"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":243},"colab_type":"code","id":"KrxXzDbDtSBs","outputId":"90ab260c-ed7d-4b24-aae5-477d101308ee","trusted":true},"cell_type":"code","source":"# preprocessing/lemmatizing/stemming test data\n%time test_corpus = gensim_preprocess(test.question_text)\ntest_texts = [ngram[token] for token in test_corpus]\n\ntest_texts = [' '.join(text) for text in test_texts]\ntest['ngrams'] = test_texts\ntest.head()","execution_count":48,"outputs":[{"output_type":"stream","text":"CPU times: user 18min 29s, sys: 2.79 s, total: 18min 31s\nWall time: 18min 31s\n","name":"stdout"},{"output_type":"execute_result","execution_count":48,"data":{"text/plain":" qid ... ngrams\n0 0000163e3ea7c7a74cd7 ... woman rude_arrogant little_bite wealth power\n1 00002bd4fb5d505b9161 ... apply_college engineer bms college_engineer wa...\n2 00007756b4a147d2b0b3 ... like nurse practitioner\n3 000086e4b7e1c7146103 ... entrepreneur\n4 0000c4c3fbe8785a3090 ... education make good people nowadays\n\n[5 rows x 3 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidquestion_textngrams
00000163e3ea7c7a74cd7Why do so many women become so rude and arroga...woman rude_arrogant little_bite wealth power
100002bd4fb5d505b9161When should I apply for RV college of engineer...apply_college engineer bms college_engineer wa...
200007756b4a147d2b0b3What is it really like to be a nurse practitio...like nurse practitioner
3000086e4b7e1c7146103Who are entrepreneurs?entrepreneur
40000c4c3fbe8785a3090Is education really making good people nowadays?education make good people nowadays
\n
"},"metadata":{}}]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":263},"colab_type":"code","id":"s62VHSn8wC8c","outputId":"ed5c2cd4-4be3-4220-d901-95c9497d7bc0","trusted":true},"cell_type":"code","source":"#ensemble on test data\nensemble.fit(vectorizer.transform(train.ngrams), train.target)\nprediction = ensemble.predict(vectorizer.transform(test.ngrams))\n\nsubmission = pd.DataFrame({'qid':test.qid, 'prediction':prediction})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":49,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n FutureWarning)\n","name":"stderr"},{"output_type":"execute_result","execution_count":49,"data":{"text/plain":" qid prediction\n0 0000163e3ea7c7a74cd7 1\n1 00002bd4fb5d505b9161 0\n2 00007756b4a147d2b0b3 0\n3 000086e4b7e1c7146103 0\n4 0000c4c3fbe8785a3090 0","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidprediction
00000163e3ea7c7a74cd71
100002bd4fb5d505b91610
200007756b4a147d2b0b30
3000086e4b7e1c71461030
40000c4c3fbe8785a30900
\n
"},"metadata":{}}]}],"metadata":{"colab":{"collapsed_sections":[],"name":"Quora.ipynb","provenance":[],"toc_visible":true,"version":"0.3.2"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}