{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# importing the required libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport nltk\nimport numpy as np \nimport pandas as pd \nprint(os.listdir(\"../input\"))\nfrom nltk.corpus import stopwords\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\nfrom sklearn.naive_bayes import GaussianNB,MultinomialNB,ComplementNB,BernoulliNB","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# a small function to see all the columns","metadata":{}},{"cell_type":"code","source":"def display_all(df):\n    with pd.option_context(\"display.max_rows\", 1000, \"display.max_columns\", 1000): \n        display(df)","metadata":{"_uuid":"e0422554da95f756b2252af69e87c710d9575fae","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# getting the files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest  = pd.read_csv('../input/test.csv')","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# checking the files","metadata":{}},{"cell_type":"code","source":"display_all(train.head())","metadata":{"_uuid":"0b39c34363c9ab869585dab837ac29a0433d957c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].value_counts()","metadata":{"_uuid":"cdc5349f5c50b230a73220dec5397db0c07a2e9c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text = train['question_text']\ntest_text = test['question_text']\ntrain_target = train['target']\nall_text = train_text.append(test_text)","metadata":{"_uuid":"889a808295cbcdc1d208c6849e9b6c2179bd5f49","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TDIF Vectorizing","metadata":{}},{"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer()\ntfidf_vectorizer.fit(all_text)\n\ncount_vectorizer = CountVectorizer()\ncount_vectorizer.fit(all_text)\n\ntrain_text_features_cv = count_vectorizer.transform(train_text)\ntest_text_features_cv = count_vectorizer.transform(test_text)\n\ntrain_text_features_tf = tfidf_vectorizer.transform(train_text)\ntest_text_features_tf = tfidf_vectorizer.transform(test_text)","metadata":{"_uuid":"b57afc7c489358f6f57713328bce1a7d831bf44d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"train_text.head()","metadata":{"_uuid":"c31e69f5bbd7a87bfb086053d43b0fa8c5f3a535","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = KFold(n_splits = 5, shuffle = True, random_state = 2018)\ntest_preds = 0\noof_preds = np.zeros([train.shape[0],])\n\nfor i, (train_idx,valid_idx) in enumerate(kfold.split(train)):\n    x_train, x_valid = train_text_features_tf[train_idx,:], train_text_features_tf[valid_idx,:]\n    y_train, y_valid = train_target[train_idx], train_target[valid_idx]\n    classifier = LogisticRegression()\n    print('fitting.......')\n    classifier.fit(x_train,y_train)\n    print('predicting......')\n    print('\\n')\n    oof_preds[valid_idx] = classifier.predict_proba(x_valid)[:,1]\n    test_preds += 0.2*classifier.predict_proba(test_text_features_tf)[:,1]","metadata":{"_uuid":"b74c8f07ea61f13c980a5b0ef23d00785219964b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_train = (oof_preds > .25).astype(np.int)\nf1_score(train_target, pred_train)","metadata":{"_uuid":"20ec786a41e629db0f3ca06c722dfe98efea7c56","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"submission1 = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission1['prediction'] = (test_preds>0.25).astype(np.int)\nsubmission1.to_csv('submission.csv', index=False)\nsubmission1['prediction'] = (test_preds>0.25)","metadata":{"_uuid":"53d728e51940326983d37573de0a2c0ed7345844","trusted":true},"execution_count":null,"outputs":[]}]}