{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing Basic packages...!!!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport nltk\nfrom sklearn.preprocessing import LabelBinarizer\nfrom nltk.corpus import stopwords\nfrom nltk.stem.porter import PorterStemmer\nfrom wordcloud import WordCloud,STOPWORDS\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize,sent_tokenize\nfrom bs4 import BeautifulSoup\nimport re,string,unicodedata\nfrom keras.preprocessing import text, sequence\nfrom nltk.tokenize.toktok import ToktokTokenizer\nfrom sklearn.metrics import classification_report,confusion_matrix,accuracy_score\nfrom sklearn.model_selection import train_test_split\nfrom string import punctuation\nfrom nltk import pos_tag\nfrom nltk.corpus import wordnet\nimport keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Embedding,LSTM,Dropout,CuDNNLSTM,GlobalMaxPool1D,Bidirectional","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nsubmission = pd.read_csv(\"../input/quora-insincere-questions-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"importing test id's for futher use","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"id = test['qid']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are {} rows and {} columns in train'.format(train.shape[0],train.shape[1]))\nprint('There are {} rows and {} columns in test'.format(test.shape[0],test.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x=train.target.value_counts()\nsns.barplot(x.index,x)\nplt.gca().set_ylabel('samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Text Preprocessing...!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rename(columns={'question_text' : 'text'},inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here the target is classified in two section\n\n* 1 : Insincere(An insincere question is defined as a question intended to make a statement rather than look for helpful answers)\n\n* 0 : sincere(Its the question which is helpful","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"now we will look at moissing values...","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ahhyeah....their is none of missing values present","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"here column named 'qid' doesn't play any significant role in training the model....so,we will drop it","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop('qid',inplace = True,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now, we will apply the same step to the test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.rename(columns={'question_text' : 'text'},inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.drop('qid',inplace = True,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now let's begin to preprocess out data through which one can train it or test it efficiently","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"for data preprocessing...:\nreference : ##https://www.kaggle.com/saranyashalya/getting-started-with-text-preprocessing-nlp##","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"importing some basic Inbuild functions ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stop = set(stopwords.words('english'))\npunctuation = list(string.punctuation)\nstop.update(punctuation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def strip_html(text):\n    soup = BeautifulSoup(text, \"html.parser\")\n    return soup.get_text()\n\n#Removing the square brackets\ndef remove_between_square_brackets(text):\n    return re.sub('\\[[^]]*\\]', '', text)\n# Removing URL's\ndef remove_urls(text):\n    return re.sub(r'http\\S+', '', text)\n# Removing hashtags\ndef remove_hash(text):\n    text = \" \".join(word.strip() for word in re.split('#|_', text))\n    return text\n#Removing the stopwords from text\ndef remove_stopwords(text):\n    final_text = []\n    for i in text.split():\n        if i.strip().lower() not in stop:\n            final_text.append(i.strip())\n    return \" \".join(final_text)\n#Removing the noisy text\ndef denoise_text(text):\n    text = strip_html(text)\n    text = remove_between_square_brackets(text)\n    text = remove_urls(text)\n    text = remove_hash(text)\n    text = remove_stopwords(text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now applying this user - defined fuction to our dataset (i.e train and test set)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['text']=train['text'].apply(denoise_text)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['text']=test['text'].apply(denoise_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now we will break the sentences into meaningful tokens...!!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"but first we will set basic parameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"max_features = 20000\nmaxlen = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(train.text)\ntokenized_train = tokenizer.texts_to_sequences(train.text)\nX = sequence.pad_sequences(tokenized_train, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenized_test = tokenizer.texts_to_sequences(test.text)\nsub_test = sequence.pad_sequences(tokenized_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Text Modelling","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"now we will split the dataset by train_test_split","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train , X_test ,y_train ,y_test = train_test_split(X,train['target'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"fiting the dataset to a logit model...!! ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nlogit = LogisticRegression(max_iter = 1000)\nlogit.fit(X_train,y_train)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now its time to check the model score...!!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"score = logit.score(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"AAAHyeah.....93% train accuracy...have we overfitted our model....to check for it we will test our model to unknown dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"y = logit.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nprint(\"accuracy of X_test data is : \",accuracy_score(y_test,y))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"nice..!! we have a accuracy of more than 93% ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#classification report\nfrom sklearn.metrics import classification_report\nprint(\"classification_report of X_test data is : \",classification_report(y_test,y))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**If you like my kernal please upvote...!!!!**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"final = logit.predict(sub_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\n        \"qid\": id,\n        \"prediction\": final\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"samplesubmission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('samplesubmission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}