{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tr_1=pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv',usecols=['comment_text','toxic'])\ntr_2=pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv',usecols=['comment_text','toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_text=pd.concat((tr_1,tr_2),axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/a/47091490/4084039\nimport re\n\ndef decontracted(phrase):\n    # specific\n    phrase = re.sub(r\"won't\", \"will not\", phrase)\n    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n\n    # general\n    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n    return phrase","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://gist.github.com/sebleier/554280\n# we are removing the words from the stop words list: 'no', 'nor', 'not'\nstopwords= ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\\\n            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \\\n            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\\\n            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', \\\n            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \\\n            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \\\n            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\\\n            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\\\n            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\\\n            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \\\n            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', \\\n            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\\\n            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\\\n            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", \\\n            'won', \"won't\", 'wouldn', \"wouldn't\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combining all the above statemennts \nfrom tqdm import tqdm\npreprocessed_essays = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_text['comment_text'].values):\n    sent = decontracted(sentance)\n    sent = sent.replace('\\\\r', ' ')\n    sent = sent.replace('\\\\\"', ' ')\n    sent = sent.replace('\\\\n', ' ')\n        # https://gist.github.com/sebleier/554280\n    sent = ' '.join(e for e in sent.split() if e not in stopwords)\n    preprocessed_essays.append(sent.lower().strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_text['clean']=preprocessed_essays","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def partition(x):\n    if x < 0.5:\n        return 0\n    return 1\n\n#changing reviews with score less than 3 to be positive and vice-versa\nactualScore = x_text['toxic']\npositiveNegative = actualScore.map(partition) \nx_text['toxic'] = positiveNegative\nprint(\"Number of data points in our data\", x_text.shape)\nx_text.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a=set(x_text['toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_v=pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_v.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\npreprocessed_essays = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_v['comment_text'].values):\n    sent = decontracted(sentance)\n    sent = sent.replace('\\\\r', ' ')\n    sent = sent.replace('\\\\\"', ' ')\n    sent = sent.replace('\\\\n', ' ')\n        # https://gist.github.com/sebleier/554280\n    sent = ' '.join(e for e in sent.split() if e not in stopwords)\n    preprocessed_essays.append(sent.lower().strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_v['clean']=preprocessed_essays","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test=pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\npreprocessed_essays = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_test['content'].values):\n    sent = decontracted(sentance)\n    sent = sent.replace('\\\\r', ' ')\n    sent = sent.replace('\\\\\"', ' ')\n    sent = sent.replace('\\\\n', ' ')\n        # https://gist.github.com/sebleier/554280\n    sent = ' '.join(e for e in sent.split() if e not in stopwords)\n    preprocessed_essays.append(sent.lower().strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test['clean']=preprocessed_essays","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# we use count vectorizer to convert the values into one hot encoded features\nfrom sklearn.feature_extraction.text import CountVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We are considering only the words which appeared in at least 10 documents(rows or projects).\nvectorizer = CountVectorizer(min_df=10)\ntext_bow_tr = vectorizer.fit_transform(x_text['clean'])\ntext_bow_cv=vectorizer.transform(x_v['clean'])\ntext_bow_test=vectorizer.transform(x_test['clean'])\nprint(\"Shape of matrix after one hot encodig \",text_bow_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport nltk\nimport string\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_curve, auc\nfrom nltk.stem.porter import PorterStemmer\n\nimport re\n# Tutorial about Python regular expressions: https://pymotw.com/2/re/\nimport string\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom gensim.models import Word2Vec\nfrom gensim.models import KeyedVectors\nimport pickle\n\nfrom tqdm import tqdm\nimport os\n\n\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Final Data matrix\")\nprint(\"train matrix=>\",X_tr.shape, y_train.shape)\n\nprint(\"test matrix=>\",X_te.shape, y_test.shape)\nprint(\"=\"*100)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train=x_text['toxic']\ny_cv=x_v['toxic']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.datasets import *\nfrom sklearn.linear_model import LogisticRegression","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logistic=LogisticRegression(class_weight='balanced')\nlogistic.fit(text_bow_tr, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = logistic.predict(text_bow_cv)\nacc = accuracy_score(y_cv, pred, normalize=True)*float(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}