{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"_kg_hide-input":false},"cell_type":"code","source":"# NLTK\nimport nltk\nfrom collections import Counter\nnltk.download('punkt')\nnltk.download('averaged_perceptron_tagger')\nnltk.download('wordnet')\nnltk.download('brown')\nnltk.download('names')\n\n\n# Tokeenization\nfrom nltk import sent_tokenize, word_tokenize\nfrom nltk.corpus import stopwords\nfrom string import punctuation\nfrom nltk.corpus import wordnet\nfrom nltk import pos_tag, pos_tag_sents\nfrom nltk.stem import WordNetLemmatizer\n\n# Vectorizer\nfrom io import StringIO\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n# Utilities\nfrom collections import OrderedDict\nimport pickle\nfrom tqdm import tqdm\nimport operator \n\n# Plotting\nfrom matplotlib import pyplot as plt\n\n# Classifier\nfrom sklearn.naive_bayes import MultinomialNB\n\n# Word Embedding\nfrom gensim.models.keyedvectors import KeyedVectors\n\n# Spell check\n\nimport re\nimport gc\n\n# Sklearn\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import classification_report, accuracy_score, recall_score, f1_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.naive_bayes import GaussianNB, MultinomialNB\n\nfrom gensim.matutils import unitvec\n\nimport logging\nimport normalise\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5921292c0c2a2c4a49249ebdbb8a2be78e5dac2c"},"cell_type":"code","source":"training_dataset = pd.read_csv(\"../input/train.csv\")\ntesting_dataset = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fe86f445dfc5a9c10d224ffd526a39e83aab939"},"cell_type":"code","source":"stopwords_en = set(stopwords.words('english'))\nstopwords_en_withpunct = stopwords_en.union(set(punctuation))\n#stopwords_json = {\"en\":[\"a\",\"a's\",\"able\",\"about\",\"above\",\"according\",\"accordingly\",\"across\",\"actually\",\"after\",\"afterwards\",\"again\",\"against\",\"ain't\",\"all\",\"allow\",\"allows\",\"almost\",\"alone\",\"along\",\"already\",\"also\",\"although\",\"always\",\"am\",\"among\",\"amongst\",\"an\",\"and\",\"another\",\"any\",\"anybody\",\"anyhow\",\"anyone\",\"anything\",\"anyway\",\"anyways\",\"anywhere\",\"apart\",\"appear\",\"appreciate\",\"appropriate\",\"are\",\"aren't\",\"around\",\"as\",\"aside\",\"ask\",\"asking\",\"associated\",\"at\",\"available\",\"away\",\"awfully\",\"b\",\"be\",\"became\",\"because\",\"become\",\"becomes\",\"becoming\",\"been\",\"before\",\"beforehand\",\"behind\",\"being\",\"believe\",\"below\",\"beside\",\"besides\",\"best\",\"better\",\"between\",\"beyond\",\"both\",\"brief\",\"but\",\"by\",\"c\",\"c'mon\",\"c's\",\"came\",\"can\",\"can't\",\"cannot\",\"cant\",\"cause\",\"causes\",\"certain\",\"certainly\",\"changes\",\"clearly\",\"co\",\"com\",\"come\",\"comes\",\"concerning\",\"consequently\",\"consider\",\"considering\",\"contain\",\"containing\",\"contains\",\"corresponding\",\"could\",\"couldn't\",\"course\",\"currently\",\"d\",\"definitely\",\"described\",\"despite\",\"did\",\"didn't\",\"different\",\"do\",\"does\",\"doesn't\",\"doing\",\"don't\",\"done\",\"down\",\"downwards\",\"during\",\"e\",\"each\",\"edu\",\"eg\",\"eight\",\"either\",\"else\",\"elsewhere\",\"enough\",\"entirely\",\"especially\",\"et\",\"etc\",\"even\",\"ever\",\"every\",\"everybody\",\"everyone\",\"everything\",\"everywhere\",\"ex\",\"exactly\",\"example\",\"except\",\"f\",\"far\",\"few\",\"fifth\",\"first\",\"five\",\"followed\",\"following\",\"follows\",\"for\",\"former\",\"formerly\",\"forth\",\"four\",\"from\",\"further\",\"furthermore\",\"g\",\"get\",\"gets\",\"getting\",\"given\",\"gives\",\"go\",\"goes\",\"going\",\"gone\",\"got\",\"gotten\",\"greetings\",\"h\",\"had\",\"hadn't\",\"happens\",\"hardly\",\"has\",\"hasn't\",\"have\",\"haven't\",\"having\",\"he\",\"he's\",\"hello\",\"help\",\"hence\",\"her\",\"here\",\"here's\",\"hereafter\",\"hereby\",\"herein\",\"hereupon\",\"hers\",\"herself\",\"hi\",\"him\",\"himself\",\"his\",\"hither\",\"hopefully\",\"how\",\"howbeit\",\"however\",\"i\",\"i'd\",\"i'll\",\"i'm\",\"i've\",\"ie\",\"if\",\"ignored\",\"immediate\",\"in\",\"inasmuch\",\"inc\",\"indeed\",\"indicate\",\"indicated\",\"indicates\",\"inner\",\"insofar\",\"instead\",\"into\",\"inward\",\"is\",\"isn't\",\"it\",\"it'd\",\"it'll\",\"it's\",\"its\",\"itself\",\"j\",\"just\",\"k\",\"keep\",\"keeps\",\"kept\",\"know\",\"known\",\"knows\",\"l\",\"last\",\"lately\",\"later\",\"latter\",\"latterly\",\"least\",\"less\",\"lest\",\"let\",\"let's\",\"like\",\"liked\",\"likely\",\"little\",\"look\",\"looking\",\"looks\",\"ltd\",\"m\",\"mainly\",\"many\",\"may\",\"maybe\",\"me\",\"mean\",\"meanwhile\",\"merely\",\"might\",\"more\",\"moreover\",\"most\",\"mostly\",\"much\",\"must\",\"my\",\"myself\",\"n\",\"name\",\"namely\",\"nd\",\"near\",\"nearly\",\"necessary\",\"need\",\"needs\",\"neither\",\"never\",\"nevertheless\",\"new\",\"next\",\"nine\",\"no\",\"nobody\",\"non\",\"none\",\"noone\",\"nor\",\"normally\",\"not\",\"nothing\",\"novel\",\"now\",\"nowhere\",\"o\",\"obviously\",\"of\",\"off\",\"often\",\"oh\",\"ok\",\"okay\",\"old\",\"on\",\"once\",\"one\",\"ones\",\"only\",\"onto\",\"or\",\"other\",\"others\",\"otherwise\",\"ought\",\"our\",\"ours\",\"ourselves\",\"out\",\"outside\",\"over\",\"overall\",\"own\",\"p\",\"particular\",\"particularly\",\"per\",\"perhaps\",\"placed\",\"please\",\"plus\",\"possible\",\"presumably\",\"probably\",\"provides\",\"q\",\"que\",\"quite\",\"qv\",\"r\",\"rather\",\"rd\",\"re\",\"really\",\"reasonably\",\"regarding\",\"regardless\",\"regards\",\"relatively\",\"respectively\",\"right\",\"s\",\"said\",\"same\",\"saw\",\"say\",\"saying\",\"says\",\"second\",\"secondly\",\"see\",\"seeing\",\"seem\",\"seemed\",\"seeming\",\"seems\",\"seen\",\"self\",\"selves\",\"sensible\",\"sent\",\"serious\",\"seriously\",\"seven\",\"several\",\"shall\",\"she\",\"should\",\"shouldn't\",\"since\",\"six\",\"so\",\"some\",\"somebody\",\"somehow\",\"someone\",\"something\",\"sometime\",\"sometimes\",\"somewhat\",\"somewhere\",\"soon\",\"sorry\",\"specified\",\"specify\",\"specifying\",\"still\",\"sub\",\"such\",\"sup\",\"sure\",\"t\",\"t's\",\"take\",\"taken\",\"tell\",\"tends\",\"th\",\"than\",\"thank\",\"thanks\",\"thanx\",\"that\",\"that's\",\"thats\",\"the\",\"their\",\"theirs\",\"them\",\"themselves\",\"then\",\"thence\",\"there\",\"there's\",\"thereafter\",\"thereby\",\"therefore\",\"therein\",\"theres\",\"thereupon\",\"these\",\"they\",\"they'd\",\"they'll\",\"they're\",\"they've\",\"think\",\"third\",\"this\",\"thorough\",\"thoroughly\",\"those\",\"though\",\"three\",\"through\",\"throughout\",\"thru\",\"thus\",\"to\",\"together\",\"too\",\"took\",\"toward\",\"towards\",\"tried\",\"tries\",\"truly\",\"try\",\"trying\",\"twice\",\"two\",\"u\",\"un\",\"under\",\"unfortunately\",\"unless\",\"unlikely\",\"until\",\"unto\",\"up\",\"upon\",\"us\",\"use\",\"used\",\"useful\",\"uses\",\"using\",\"usually\",\"uucp\",\"v\",\"value\",\"various\",\"very\",\"via\",\"viz\",\"vs\",\"w\",\"want\",\"wants\",\"was\",\"wasn't\",\"way\",\"we\",\"we'd\",\"we'll\",\"we're\",\"we've\",\"welcome\",\"well\",\"went\",\"were\",\"weren't\",\"what\",\"what's\",\"whatever\",\"when\",\"whence\",\"whenever\",\"where\",\"where's\",\"whereafter\",\"whereas\",\"whereby\",\"wherein\",\"whereupon\",\"wherever\",\"whether\",\"which\",\"while\",\"whither\",\"who\",\"who's\",\"whoever\",\"whole\",\"whom\",\"whose\",\"why\",\"will\",\"willing\",\"wish\",\"with\",\"within\",\"without\",\"won't\",\"wonder\",\"would\",\"wouldn't\",\"x\",\"y\",\"yes\",\"yet\",\"you\",\"you'd\",\"you'll\",\"you're\",\"you've\",\"your\",\"yours\",\"yourself\",\"yourselves\",\"z\",\"zero\"]}\n#stopwords_combined = set.union(set(stopwords_json['en']), stopwords_en_withpunct)\nstopwords = stopwords_en_withpunct\nstopwords = stopwords.union(set(['']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e926ac78a4d32781a03237d706ed469bd51494a"},"cell_type":"code","source":"wnl = WordNetLemmatizer()\n\ndef penn2morphy(penntag):\n    \"\"\" Converts Penn Treebank tags to WordNet. \"\"\"\n    morphy_tag = {'NN':'n', 'JJ':'a',\n                  'VB':'v', 'RB':'r'}\n    try:\n        return morphy_tag[penntag[:2]]\n    except:\n        return 'n' \n    \ndef lemmatize_sent(text): \n    # Text input is string, returns lowercased strings.\n    return [wnl.lemmatize(word.lower(), pos=penn2morphy(tag)) \n            for word, tag in pos_tag(word_tokenize(text))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"221c1723b9fb9096171f8e45501ae3dcab9ad78f"},"cell_type":"code","source":"\ndef preprocess_text(text):\n    # Input: str, i.e. document/sentence\n    # Output: list(str) , i.e. list of lemmas\n    \n    tokens =  [word for word in lemmatize_sent(text) \n            if word not in stopwords\n            and not word.isdigit()]\n    try: \n        tokens = normalise.normalise(text=tokens, user_abbrevs=custom_dictionary, verbose=False)\n    except:\n        result = []\n        for text in tokens:\n            try:\n                result.append(normalise.normalise(texts, verbose=False))\n            except:\n                result.append(text)\n        tokens = result\n    tokens = [word for word in tokens\n          if word not in stopwords]\n    return tokens;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomVectorizer(TfidfVectorizer):\n    def build_analyzer(self):\n        stop_words = self.get_stop_words()\n    \n        def analyser(doc):\n            if (self.lowercase == True):\n                doc = doc.lower()\n            tokens = preprocess_text(doc)\n            \n            return(self._word_ngrams(tokens, stop_words))\n        return (analyser)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_features_TFIDF = 5000\nngram_range_TFIDF = (1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_vectorizer_preprocessing = CustomVectorizer(stop_words=stopwords,\n                                    ngram_range=ngram_range_TFIDF,\n                                    max_features=max_features_TFIDF,\n                                    encoding='utf-8',\n                                    decode_error='strict',\n                                    strip_accents = None,\n                                    lowercase=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_vectors = tfidf_vectorizer_preprocessing.fit_transform(training_dataset['question_text'])\ntest_vectors = tfidf_vectorizer_preprocessing.transform(testing_dataset['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d14e4cce3790ad561bdc86cafb3a8fe20006887"},"cell_type":"code","source":"randomForestClassifier = RandomForestClassifier(n_estimators=30, \n                       max_depth=120,\n                       min_samples_leaf=2,\n                       max_features=\"auto\",\n                       n_jobs=-1)\nX_training = train_vectors\nY_training = training_dataset['target']\nrandomForestClassifier.fit(X_training, Y_training)\nX_test = test_vectors\nY_predict = randomForestClassifier.predict(X_test)\ndata_submission = pd.DataFrame({'qid': testing_dataset.qid, 'prediction': Y_predict})\ndata_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}