{"cells":[{"metadata":{"trusted":true,"_uuid":"df13779f9f39a515d43825d1fe82223e3e7abb29"},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5376f7d018a350dbcd0f6d5a02351e404ce8947"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f05afa5a931ea9389f5bb307448f883d4fb69156"},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb497ddd0e1638a063e303c7b0d3435f47ffe505"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93cddcfcb2854f7e18635a2a4e424ac1d6309716"},"cell_type":"code","source":"train_df.target.value_counts()/train_df.target.value_counts().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80a33180c6c9a86121a685523bb1ac3a6418639e"},"cell_type":"code","source":"train_df['Number_of_words'] = train_df.question_text.apply(lambda x: len(str(x).split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08c71616d0e832c2edf8de2b24c6dcb5a7b12d73"},"cell_type":"code","source":"train_df['Char_count'] = train_df['question_text'].str.len()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05f3d64b395eb85fc698d8948197ea5fe9c9e334"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2362c337c3c64e8e569bb37329b54a6fc19fa7d6"},"cell_type":"code","source":"def avg_word(sentence):\n    words = sentence.split()\n    return (sum(len(word) for word in words)/len(words))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e43aa4164aea55b9f964b15a5bd166c5de82544"},"cell_type":"code","source":" train_df['avg_word'] = train_df['question_text'].apply(lambda x:avg_word(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad9c48eda5eb8dee8bae44c8d7b3304db818eb73"},"cell_type":"code","source":"from nltk.corpus import stopwords\nstop = stopwords.words('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"330650c6ed3da21044c8f348b60f17355422eb4c"},"cell_type":"code","source":" train_df['stopwords'] = train_df['question_text'].apply(lambda x: len([x for x in x.split() if x in stop]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c2c4ed294a798652ecb6fe96c16a6a706bba89b"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a912f996303450e38ab1f8c633e64d436791e91"},"cell_type":"markdown","source":"<b> Basic Preprocessing </b>"},{"metadata":{"trusted":true,"_uuid":"01489a359ae26ca209d51b6c8c5f845f2af62ee4"},"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\ntest_df['question_text'] = test_df['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d5ffe69fa2f0e15a32ba3b77cfe9de78c1433c8"},"cell_type":"code","source":" train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dfd6ff871913e55dce3706d648d3dcf529fbd06"},"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].str.replace('[^\\w\\s]','')\ntest_df['question_text'] = test_df['question_text'].str.replace('[^\\w\\s]','')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aed60c8b420f826b1733a7688a7b7d240d81ef5d"},"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\ntest_df['question_text'] = test_df['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2592fa96a14abf88b6b306e3e32e623cb3b070b"},"cell_type":"code","source":"freq_train = pd.Series(' '.join(train_df['question_text']).split()).value_counts()[:30]\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: ' '.join([i for i in x.split() if i not in freq_train]))\nfreq_train = pd.Series(' '.join(train_df['question_text']).split()).value_counts()[-30:]\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: ' '.join([i for i in x.split() if i not in freq_train]))\nfreq_test = pd.Series(' '.join(test_df['question_text']).split()).value_counts()[:30]\ntest_df['question_text'] = test_df['question_text'].apply(lambda x: ' '.join([i for i in x.split() if i not in freq_test]))\nfreq_test = pd.Series(' '.join(test_df['question_text']).split()).value_counts()[-30:]\ntest_df['question_text'] = test_df['question_text'].apply(lambda x: ' '.join([i for i in x.split() if i not in freq_test]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3987dc62a0cbdf6a0df1d3bfa806352c3f97ef6a"},"cell_type":"code","source":"train_df['Number_of_words'] = train_df.question_text.apply(lambda x: len(str(x).split()))\ntrain_df['Char_count'] = train_df.question_text.str.len()\ntest_df['Number_of_words'] = test_df.question_text.apply(lambda x: len(str(x).split()))\ntest_df['Char_count'] = test_df.question_text.str.len()\ntrain_df.drop(['avg_word','stopwords'],axis=1,inplace=True)\ntrain_df[train_df['target']==1].head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99c93ddafbd7b149062c448301e2420bf1f292b"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.model_selection import train_test_split    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b372b011c9d3a7ac054ac9a11a964ed172337529"},"cell_type":"code","source":"y = train_df['target']\nX = train_df['question_text']\nX_train,X_test,y_train,y_test = train_test_split(X,y,random_state=1,test_size=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632b36b5e6d4e7d963900afb7ec6710cf387f02b"},"cell_type":"code","source":"train_df['question_text'].apply(lambda x: len(x.split(' '))).sum()\ntest_df['question_text'].apply(lambda x: len(x.split(' '))).sum()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6775a18ae4c192dc6326e6952530e5647271dbb4"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfTransformer,CountVectorizer\n\nnb = Pipeline([('vect', CountVectorizer()),\n               ('tfidf', TfidfTransformer()),\n               ('clf', MultinomialNB()),\n              ])\nnb.fit(X_train, y_train)\nfrom sklearn.metrics import classification_report,accuracy_score\ny_pred = nb.predict(X_test)\n\nprint('accuracy %s' % accuracy_score(y_pred, y_test))\nprint(classification_report(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8fc5ba4b8d6bc6311622827e85a9dc40c107414"},"cell_type":"markdown","source":"**Linear SVM**"},{"metadata":{"trusted":true,"_uuid":"9819ddc5983fcd69b710ee0c48464eed4a5b7df1"},"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\n\nsgd = Pipeline([('vect', CountVectorizer()),\n                ('tfidf', TfidfTransformer()),\n                ('clf', SGDClassifier(loss='hinge', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None)),\n               ])\nsgd.fit(X_train, y_train)\ny_pred = sgd.predict(X_test)\n\nprint('accuracy %s' % accuracy_score(y_pred, y_test))\nprint(classification_report(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"5f8a63015e15d30f605e182361a05af68f538568"},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlogreg = Pipeline([('vect', CountVectorizer()),\n                ('tfidf', TfidfTransformer()),\n                ('clf', LogisticRegression(C=1e3)),\n               ])\nlogreg.fit(X_train, y_train)\n\ny_pred = logreg.predict(X_test)\n\nprint('accuracy %s' % accuracy_score(y_pred, y_test))\nprint(classification_report(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5e50fc619cfec2a9de7a560f1332266fac6fd68"},"cell_type":"code","source":"# from tqdm import tqdm\n# tqdm.pandas(desc=\"progress-bar\")\n# from gensim.models import Doc2Vec\n# from sklearn import utils\n# import gensim\n# from gensim.models.doc2vec import TaggedDocument\n# import re\n\n# def label_sentences(corpus, label_type):\n#     \"\"\"\n#     Gensim's Doc2Vec implementation requires each document/paragraph to have a label associated with it.\n#     We do this by using the TaggedDocument method. The format will be \"TRAIN_i\" or \"TEST_i\" where \"i\" is\n#     a dummy index of the post.\n#     \"\"\"\n#     labeled = []\n#     for i, v in enumerate(corpus):\n#         label = label_type + '_' + str(i)\n#         labeled.append(TaggedDocument(v.split(), [label]))\n#     return labeled\n# X_train, X_test, y_train, y_test = train_test_split(train_df['question_text'], train_df['target'], random_state=0, test_size=0.3)\n# X_train = label_sentences(X_train, 'Train')\n# X_test = label_sentences(X_test, 'Test')\n# all_data = X_train + X_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87a141df5b505b400a8947ca73c10d0bdf7f149d"},"cell_type":"code","source":"# model_dbow = Doc2Vec(dm=0, vector_size=300, negative=5, min_count=1, alpha=0.065, min_alpha=0.065)\n# model_dbow.build_vocab([x for x in tqdm(all_data)])\n\n# for epoch in range(30):\n#     model_dbow.train(utils.shuffle([x for x in tqdm(all_data)]), total_examples=len(all_data), epochs=1)\n#     model_dbow.alpha -= 0.002\n#     model_dbow.min_alpha = model_dbow.alpha","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54599d87323d67fd44923e30f654e3c19f20e09b"},"cell_type":"code","source":"# import  numpy as np\n# def get_vectors(model, corpus_size, vectors_size, vectors_type):\n#     \"\"\"\n#     Get vectors from trained doc2vec model\n#     :param doc2vec_model: Trained Doc2Vec model\n#     :param corpus_size: Size of the data\n#     :param vectors_size: Size of the embedding vectors\n#     :param vectors_type: Training or Testing vectors\n#     :return: list of vectors\n#     \"\"\"\n#     vectors = np.zeros((corpus_size, vectors_size))\n#     for i in range(0, corpus_size):\n#         prefix = vectors_type + '_' + str(i)\n#         vectors[i] = model.docvecs[prefix]\n#     return vectors\n    \n# train_vectors_dbow = get_vectors(model_dbow, len(X_train), 300, 'Train')\n# test_vectors_dbow = get_vectors(model_dbow, len(X_test), 300, 'Test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"591fe81019694b749c120ae2bb2eff6cb7d1afef"},"cell_type":"code","source":"# logreg = LogisticRegression(n_jobs=1, C=1e5)\n# logreg.fit(train_vectors_dbow, y_train)\n# logreg = logreg.fit(train_vectors_dbow, y_train)\n# y_pred = logreg.predict(test_vectors_dbow)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0ae4e9a6e67a742c7a592d301a00fd8f5708654a"},"cell_type":"code","source":"# print('accuracy %s' % accuracy_score(y_pred, y_test))\n# print(classification_report(y_test, y_pred))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c72a17a601e8a3513bd81a5fe98bc9fa99a0b7eb"},"cell_type":"code","source":"y_pred1 = logreg.predict(test_df['question_text'])\ny_pred1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a43572b98de790f1648b90a18d5eea7cb18707f"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub['prediction']=y_pred1\nsub.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}