{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\n#from wordcloud import WordCloud as wc   # not needed\nfrom nltk.corpus import stopwords\nimport matplotlib.pylab as pylab\nimport matplotlib.pyplot as plt\nfrom pandas import get_dummies\nimport matplotlib as mpl\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport warnings\nimport sklearn\nimport string\nimport scipy\nimport numpy\nimport nltk\nimport json\nimport sys\nimport csv\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\ntrain_large = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_large = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n\ntrain = train_large[:100000]\ntest = test_large[:100]\n\ntrain.head()\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape of train:',train.shape)\nprint('Shape of test:',test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_words']=train['question_text'].apply(lambda x : len(str(x).split()))\ntest['num_words']=test['question_text'].apply(lambda x : len(str(x).split()))\nprint('Max number of words in a question train dataset:',np.mean(train['num_words']))\nprint('Max number of words in a question test dataset:',test['num_words'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_unique_words']=train['question_text'].apply(lambda x : len(set(str(x).split())))\ntest['num_unique_words']=test['question_text'].apply(lambda x : len(set(str(x).split())))\n\n\nprint('maximum of num_unique_words in train',train[\"num_unique_words\"].max())\n\nprint(\"maximum of num_unique_words in test\",test[\"num_unique_words\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eng_stopwords=set(stopwords.words('english'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_stopwords']=train['question_text'].apply(lambda x : len([i for i in str(x).split() if i in eng_stopwords]))\ntest['num_stopwords']=test['question_text'].apply(lambda x : len([i for i in str(x).split() if i in eng_stopwords]))\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_punctuations']=train['question_text'].apply(lambda x : len([i for i in str(x).split() if i in string.punctuation]))\ntest['num_punctuations']=test['question_text'].apply(lambda x : len([i for i in str(x).split() if i in string.punctuation]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom nltk import pos_tag\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\nfrom nltk.corpus import wordnet as wn\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn import model_selection, naive_bayes, svm\nfrom sklearn.metrics import accuracy_score\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain['question_text'] = [entry.lower() for entry in train['question_text']]\n\ntest['question_text'] = [entry.lower() for entry in test['question_text']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain['question_text']= [word_tokenize(entry) for entry in train['question_text']]\n\ntest['question_text']= [word_tokenize(entry) for entry in test['question_text']]\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# or adjective etc. By default it is set to Noun\ntag_map = defaultdict(lambda : wn.NOUN)\ntag_map['J'] = wn.ADJ\ntag_map['V'] = wn.VERB\ntag_map['R'] = wn.ADV\n# the tag_map would map any tag to 'N' (Noun) except\n# Adjective to J, Verb -> v, Adverb -> R\n# that means if you get a Pronoun then it would still be mapped to Noun\n\n\nfor index,entry in enumerate(train['question_text']):\n    # Declaring Empty List to store the words that follow the rules for this step\n    Final_words = []\n    \n    # Initializing WordNetLemmatizer()\n    word_Lemmatized = WordNetLemmatizer()\n    \n    # pos_tag function below will provide the 'tag' \n    # i.e if the word is Noun(N) or Verb(V) or something else.\n    for word, tag in pos_tag(entry):\n        # Below condition is to check for Stop words and consider only \n        # alphabets\n        if word not in stopwords.words('english') and word.isalpha():\n            word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])\n            Final_words.append(word_Final)\n            \n    # The final processed set of words for each iteration will be stored \n    # in 'question_text_final'\n    train.loc[index,'question_text_final'] = str(Final_words)  \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for index,entry in enumerate(test['question_text']):\n    # Declaring Empty List to store the words that follow the rules for this step\n    Final_words_test = []\n    \n    # Initializing WordNetLemmatizer()\n    word_Lemmatized = WordNetLemmatizer()\n    \n    # pos_tag function below will provide the 'tag' \n    # i.e if the word is Noun(N) or Verb(V) or something else.\n    for word, tag in pos_tag(entry):\n        # Below condition is to check for Stop words and consider only \n        # alphabets\n        if word not in stopwords.words('english') and word.isalpha():\n            word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])\n            Final_words_test.append(word_Final)\n            \n    # The final processed set of words for each iteration will be stored \n    # in 'question_text_final'\n    test.loc[index,'question_text_final'] = str(Final_words_test) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Tfidf_vect = TfidfVectorizer()\nTfidf_vect.fit(train['question_text_final'])\n\nTrain_X_Tfidf = Tfidf_vect.transform(train['question_text_final'])\n\nTest_X_Tfidf = Tfidf_vect.transform(test['question_text_final'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr=LogisticRegression()\nlr.fit(x_train,y_train)\nypred=lr.predict(x_test)\nprint(accuracy_score(ypred,y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train,x_test,y_train,y_test=train_test_split(Train_X_Tfidf,train['target'],test_size=0.1,random_state=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max=count=0\nfor i in range(1,20):\n    knn=KNeighborsClassifier(n_neighbors=i)\n    knn.fit(x_train,y_train)\n    ypred=knn.predict(x_test)\n    acc=accuracy_score(ypred,y_test)\n    if acc>max:\n        max=acc\n        count=i\n    print('For i = ', i ,\":\" ,acc )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn=KNeighborsClassifier(n_neighbors=count)\nknn.fit(x_train,y_train)\nypred=knn.predict(x_test)\nacc=accuracy_score(ypred,y_test)\nprint(acc)\n#count= 6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max=id=0\nmax1=id1=0\nfor i in range(1,20):\n    dt = DecisionTreeClassifier(criterion='gini',max_depth=i)\n    dt.fit(x_train,y_train )\n    ypred=dt.predict(x_test )\n    print(accuracy_score(ypred,y_test))\n    if accuracy_score(ypred,y_test) > max:\n        max=accuracy_score(ypred,y_test)\n        id=i\n    \n    dt = DecisionTreeClassifier(criterion='entropy',max_depth=i)\n    dt.fit(x_train,y_train )\n    ypred=dt.predict(x_test )\n    print(accuracy_score(ypred,y_test))\n    if accuracy_score(ypred,y_test) > max:\n        max1=accuracy_score(ypred,y_test)\n        id1=i\n        \nprint(\"----------------\")\nprint(max ,\":\", id)\nprint(max1 , \":\", id1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.svm import SVC ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"svm=SVC(kernel='linear')\nsvm.fit(x_train,y_train)\nypred=svm.predict(x_test)\nprint(accuracy_score(y_test,ypred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10,201,10):\n    dt = RandomForestClassifier(n_estimators=i)\n    dt.fit(x_train,y_train )\n    ypred=dt.predict(x_test )\n    print(\"For n = \", i , \" :\",accuracy_score(ypred,y_test))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}