{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:43:07.530853Z","start_time":"2018-11-27T02:43:07.100919Z"},"trusted":true,"_uuid":"c7eb65ab392a382557629d83854ffbb1f489e58c"},"cell_type":"code","source":"%matplotlib inline \nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport re\nimport random\n\nfrom nltk import word_tokenize, sent_tokenize\n\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import confusion_matrix, precision_recall_fscore_support, classification_report\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.dummy import DummyClassifier","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:43:07.562298Z","start_time":"2018-11-27T02:43:07.531752Z"},"trusted":true,"_uuid":"dae9caf1429ca94d1415f1e7fd8ffabea9f1fecc"},"cell_type":"code","source":"# Lematization\nfrom nltk.stem import WordNetLemmatizer\nwordnet_lemmatizer = WordNetLemmatizer()\nfrom nltk import wordpunct_tokenize\nfrom nltk.corpus import stopwords","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:43:07.577932Z","start_time":"2018-11-27T02:43:07.563272Z"},"trusted":true,"_uuid":"76d62051c83fe562c4b475c7ee2c48ef4676634b"},"cell_type":"code","source":"stoplist = stopwords.words(\"english\")\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T03:06:16.353671Z","start_time":"2018-11-27T03:06:14.787477Z"},"trusted":true,"_uuid":"072c846012328c3e7d23e3a370560a7f208baa14"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:43:36.759216Z","start_time":"2018-11-27T02:43:36.630764Z"},"trusted":true,"_uuid":"80b1dc64321d241b02e83180571dde8c10d05665"},"cell_type":"code","source":"train = train[train.qid.isin(['527aac2ce6f12f789fe5','7cd3188b81cdec72ad91'])!=True]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcc2d5889d28fe6c627a7933c9650ee6f2fa4fdf"},"cell_type":"markdown","source":"### Normalization Text"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:46:32.319775Z","start_time":"2018-11-27T02:43:38.437232Z"},"trusted":true,"_uuid":"abf763c8159a2257e83fcd5d30a51b6286a1160f"},"cell_type":"code","source":"\n# Insincere question\nqcorpus = [] # corpus\n\nfor text in train.question_text:\n    tokens = []\n    for token in word_tokenize(text):\n        if token.isalpha() and token not in stoplist: # delete  stopwords and non characters\n            token = wordnet_lemmatizer.lemmatize(token.lower()) # lematization token\n            tokens.append(token)\n    \n    qcorpus.append(\" \".join(tokens))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:46:32.892236Z","start_time":"2018-11-27T02:46:32.754116Z"},"trusted":false,"_uuid":"fd7f2f749130a6c8c1c0bd69f995f4eae3f3c931"},"cell_type":"code","source":"train['question_text'] = qcorpus","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:46:40.786450Z","start_time":"2018-11-27T02:46:33.330846Z"},"trusted":false,"_uuid":"9d64a11095b5ab4c653972cb40371ea39c7a585f"},"cell_type":"code","source":"\n# Insincere question\nqcorpus = [] # corpus\n\nfor text in test.question_text:\n    tokens = []\n    for token in word_tokenize(text):\n        if token.isalpha() and token not in stoplist: # delete  stopwords and non characters\n            token = wordnet_lemmatizer.lemmatize(token.lower()) # lematization token\n            tokens.append(token)\n    \n    qcorpus.append(\" \".join(tokens))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:46:40.830310Z","start_time":"2018-11-27T02:46:40.826111Z"},"trusted":false,"_uuid":"eacd67b3b8255c6f80c8fd4a048954beda3f564b"},"cell_type":"code","source":"test['question_text'] = qcorpus","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19ce3d4dbeda2ebc8ac04cb670ac252751bd06d4"},"cell_type":"markdown","source":"### Features ( N-gramas)"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:49:48.354547Z","start_time":"2018-11-27T02:49:23.686381Z"},"trusted":false,"_uuid":"ceebfc06dd5d44de203dcd4313f8f1eb4754a475"},"cell_type":"code","source":"# incluyo bigramas (aclaracion:si aparece \"best friend\" tambien va a contar para \"best\" y para \"friend\")\ncount_vect = CountVectorizer(min_df=5,stop_words=\"english\",ngram_range=(1,2))\nX_train_ngrams = count_vect.fit_transform(train['question_text'] ) # cuenta frecuencia de tokens y define el diccionario\nX_test_ngrams = count_vect.transform(test['question_text'] ) # cuenta frecuencia de tokens existentes en el diccionario\nprint(\"numero de features=\",X_train_ngrams.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fda47cf766838d285f69d30867a0dff3e760d5f9"},"cell_type":"markdown","source":"# Naive Bayes"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:52:52.213292Z","start_time":"2018-11-27T02:52:52.211124Z"},"trusted":false,"_uuid":"8ee321e70872589b216dcefe1a58c846f111c8e8"},"cell_type":"code","source":"\n\n\nclf = MultinomialNB(alpha=1)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:53:27.513948Z","start_time":"2018-11-27T02:53:27.298013Z"},"trusted":false,"_uuid":"9d08ab083804eea946c763d68c3785f2cee821d5"},"cell_type":"code","source":"clf.fit(X_train_ngrams, train.target)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:55:50.937442Z","start_time":"2018-11-27T02:55:50.912207Z"},"trusted":false,"_uuid":"57986232972d39c86007f07e20fb0071f839f4cb"},"cell_type":"code","source":"test['prediction'] = clf.predict(X_test_ngrams)\n","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T02:59:09.399140Z","start_time":"2018-11-27T02:59:09.394076Z"},"trusted":false,"_uuid":"7340e155c5e8128b04c209a705db2680b942c535"},"cell_type":"code","source":"submission = test[['qid','prediction']]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T03:01:01.702730Z","start_time":"2018-11-27T03:01:01.622028Z"},"trusted":true,"_uuid":"0f2ba8f046bb054b5379658f2149f42df004ed5d","_kg_hide-output":true,"_kg_hide-input":false},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False,sep=\",\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"varInspector":{"cols":{"lenName":16,"lenType":16,"lenVar":40},"kernels_config":{"python":{"delete_cmd_postfix":"","delete_cmd_prefix":"del ","library":"var_list.py","varRefreshCmd":"print(var_dic_list())"},"r":{"delete_cmd_postfix":") ","delete_cmd_prefix":"rm(","library":"var_list.r","varRefreshCmd":"cat(var_dic_list()) "}},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"window_display":false}},"nbformat":4,"nbformat_minor":1}