{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport spacy\nimport re\nfrom gensim import corpora, models, similarities\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"np.random.seed(27)\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import resample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere = train[train.target == 0]\ninsincere = train[train.target == 1]\ntrain = pd.concat([resample(sincere,\n                     replace = False,\n                     n_samples = len(insincere)), insincere])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions_re = re.compile('(%s)' % '|'.join(contractions.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions_re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expandContractions(text, contractions_re=contractions_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return contractions_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  strip_numeric, # remove numbers\n                  remove_stopwords,# remove stopwords\n                  strip_short # remove words less than minsize=3 characters long\n                 ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nlp = spacy.load('en')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gensim_preprocess(docs, logging=True):\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    texts_out = []\n    for doc in docs:\n    # https://spacy.io/usage/processing-pipelines\n        doc = nlp((\" \".join(doc)),  # doc = text to tokenize => creates doc\n                  # disable parts of the language processing pipeline we don't need here to speed up processing\n                  disable=['ner', # named entity recognition\n                           'tagger', # part-of-speech tagger\n                           'textcat', # document label categorizer\n                          ])\n        texts_out.append([tok.lemma_ for tok in doc if tok.lemma_ != '-PRON-'])\n    return pd.Series(texts_out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gensim_preprocess(train.question_text.iloc[10:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = gensim_preprocess(train.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ngram_phraser = models.Phrases(train_data, threshold=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ngram = models.phrases.Phraser(ngram_phraser)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(ngram[train_data[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts = [ngram[token] for token in train_data]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts = [' '.join(text) for text in texts]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ngrams'] = texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = CountVectorizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer.fit(train.ngrams)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(train.ngrams, train.target, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import BernoulliNB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bnb = BernoulliNB()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time bnb.fit(vectorizer.transform(x_train), y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Naive Bayes Score: ', bnb.score(vectorizer.transform(x_test), y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = bnb.predict(vectorizer.transform(x_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Classification Report\")\nprint(metrics.classification_report(y_test, predictions))\nprint(\"\")\nprint(\"Confusion Matrix\")\nprint(metrics.confusion_matrix(y_test, predictions))\nprint(\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_test = bnb.predict(vectorizer.transform(test.question_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'qid':test['qid'],'prediction':predict_test})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}