{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"comments= pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comments.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Missing value treatment ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"comments.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Heat map to check if any missing values are present in the dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sb\n\nfrom matplotlib import pyplot as plt\n\nsb.heatmap(comments.isnull(),cbar=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of classes present in each target variable\nfor i,j in comments[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].items():\n    print(i,j.value_counts(),'',sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val = comments[['toxic','severe_toxic','obscene','threat','insult','identity_hate']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize']=[20,8]\nfig =plt.figure()\n\nplt.subplot(2,2,1)\nplot = plt.hist(val.toxic)\nplt.title('toxic')\nplt.subplot(2,2,2)\nplt.hist(val.severe_toxic,color='red')\nplt.title('severe_toxic')\nplt.subplot(2,2,3)\nplt.hist(val.obscene,color='orange')\nplt.title('obscene')\nplt.subplot(2,2,4)\nplt.hist(val.threat,color='green')\nplt.title('threat')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize']=[20,8]\nfig =plt.figure()\n\nplt.subplot(2,2,1)\nplt.hist(val.insult,color='purple')\nplt.title('insult')\nplt.subplot(2,2,2)\nplt.hist(val.identity_hate,color='black')\nplt.title('identity_hate')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom string import punctuation\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stop_nltk = stopwords.words(\"english\")\n\nstop_updated = stop_nltk + [\"...\",\"..\",\"\\n\",\"\\t\",\"==\",\"=\",\"//\",\"'\",'D',',','en wikipedia org','https en wikipedia','wikipedia org wiki'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions_dict = { \"ain't\": \"are not\",\"'s\":\" is\",\"aren't\": \"are not\",\n                     \"can't\": \"cannot\",\"can't've\": \"cannot have\",\n                     \"'cause\": \"because\",\"could've\": \"could have\",\"couldn't\": \"could not\",\n                     \"couldn't've\": \"could not have\", \"didn't\": \"did not\",\"doesn't\": \"does not\",\n                     \"don't\": \"do not\",\"hadn't\": \"had not\",\"hadn't've\": \"had not have\",\n                     \"hasn't\": \"has not\",\"haven't\": \"have not\",\"he'd\": \"he would\",\n                     \"he'd've\": \"he would have\",\"he'll\": \"he will\", \"he'll've\": \"he will have\",\n                     \"how'd\": \"how did\",\"how'd'y\": \"how do you\",\"how'll\": \"how will\",\n                     \"I'd\": \"I would\", \"I'd've\": \"I would have\",\"I'll\": \"I will\",\n                     \"I'll've\": \"I will have\",\"I'm\": \"I am\",\"I've\": \"I have\", \"isn't\": \"is not\",\n                     \"it'd\": \"it would\",\"it'd've\": \"it would have\",\"it'll\": \"it will\",\n                     \"it'll've\": \"it will have\", \"let's\": \"let us\",\"ma'am\": \"madam\",\n                     \"mayn't\": \"may not\",\"might've\": \"might have\",\"mightn't\": \"might not\", \n                     \"mightn't've\": \"might not have\",\"must've\": \"must have\",\"mustn't\": \"must not\",\n                     \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n                     \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\"oughtn't\": \"ought not\",\n                     \"oughtn't've\": \"ought not have\",\"shan't\": \"shall not\",\"sha'n't\": \"shall not\",\n                     \"shan't've\": \"shall not have\",\"she'd\": \"she would\",\"she'd've\": \"she would have\",\n                     \"she'll\": \"she will\", \"she'll've\": \"she will have\",\"should've\": \"should have\",\n                     \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\"so've\": \"so have\",\n                     \"that'd\": \"that would\",\"that'd've\": \"that would have\", \"there'd\": \"there would\",\n                     \"there'd've\": \"there would have\", \"they'd\": \"they would\",\n                     \"they'd've\": \"they would have\",\"they'll\": \"they will\",\n                     \"they'll've\": \"they will have\", \"they're\": \"they are\",\"they've\": \"they have\",\n                     \"to've\": \"to have\",\"wasn't\": \"was not\",\"we'd\": \"we would\",\n                     \"we'd've\": \"we would have\",\"we'll\": \"we will\",\"we'll've\": \"we will have\",\n                     \"we're\": \"we are\",\"we've\": \"we have\", \"weren't\": \"were not\",\"what'll\": \"what will\",\n                     \"what'll've\": \"what will have\",\"what're\": \"what are\", \"what've\": \"what have\",\n                     \"when've\": \"when have\",\"where'd\": \"where did\", \"where've\": \"where have\",\n                     \"who'll\": \"who will\",\"who'll've\": \"who will have\",\"who've\": \"who have\",\n                     \"why've\": \"why have\",\"will've\": \"will have\",\"won't\": \"will not\",\n                     \"won't've\": \"will not have\", \"would've\": \"would have\",\"wouldn't\": \"would not\",\n                     \"wouldn't've\": \"would not have\",\"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n                     \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n                     \"y'all've\": \"you all have\", \"you'd\": \"you would\",\"you'd've\": \"you would have\",\n                     \"you'll\": \"you will\",\"you'll've\": \"you will have\", \"you're\": \"you are\",\n                     \"you've\": \"you have\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\n\ncontractions_re=re.compile('(%s)' % '|'.join(contractions_dict.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for expanding contractions\ndef expand_contractions(text,contractions_dict=contractions_dict):\n def replace(match):\n    return contractions_dict[match.group(0)]\n return contractions_re.sub(replace, text)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for removing the stopwords  and punctuations\nlemm = WordNetLemmatizer()\ndef clean_text(sent):\n    tokens = word_tokenize(sent.lower())\n    lemmed = [lemm.lemmatize(term) for term in tokens \\\n               if term not in stop_updated and \\\n                term not in list(punctuation) and\\\n               len(term) > 2] \n    res = \" \".join(lemmed)\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Expanding Contractions in the comments\n\ncomments['clean_text']=comments['comment_text'].apply(lambda x:expand_contractions(x))\n\ncomments['clean_text']=comments['clean_text'].apply(lambda x: re.sub('\\w*\\d\\w*','', x))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comments['clean_text'] = comments.clean_text.apply(clean_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comments.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = comments.clean_text.values\ntarget = comments.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, target, train_size = 0.75,random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf_vectorizer = TfidfVectorizer(max_features=3500)\nX_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\nX_test_tfidf = tfidf_vectorizer.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier = MultinomialNB()\nclassifier.fit(X_train_tfidf, y_train)\ntraget_prediction = classifier.predict(X_test_tfidf)\nprint('Accuaracy score for toxic comment classification:',accuracy_score(y_test,traget_prediction)*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traget_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}