{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import random, os, time, math, re\n\nimport string\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Detect hardware, return appropriate distribution strategy\n# try:\n#     # TPU detection. No parameters necessary if TPU_NAME environment variable is\n#     # set: this is always the case on Kaggle.\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#     print('Running on TPU ', tpu.master())\n# except ValueError:\n#     tpu = None\n\n# if tpu:\n#     tf.config.experimental_connect_to_cluster(tpu)\n#     tf.tpu.experimental.initialize_tpu_system(tpu)\n#     strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# else:\n#     # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n#     strategy = tf.distribute.get_strategy()\n\n# print(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1              = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n# train1              = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv\")\n# train1                = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n# train1              = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv\")\n                          \n# sample_submission   = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\n# validation          = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\n# test                = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\n\n# validation2         = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv\")\n# test2               = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv\")    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train1[[\"comment_text\",\"toxic\"]][0:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)\ntrain.head(50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize import sent_tokenize,word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer \nfrom nltk.stem import WordNetLemmatizer ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"toxic\"]= (train[\"toxic\"] > 0.4).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.barplot( data=train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### remove stop words and punctuation"},{"metadata":{"trusted":true},"cell_type":"code","source":"punctuations = string.punctuation\nstop_words = set(stopwords.words(\"english\"))\n\ndef remove_stop_words(words):\n    return [w for w in words if not w in stop_words]\n\ndef remove_empty(words):\n    return [w for w in words if not len(w.strip())<2]\n\ndef words_to_lowercase(words):\n    return [w.lower() for w in words]\n\ndef remove_punctuation_from_word(word):\n    return ''.join(c for c in word if c not in punctuations)\n\ndef remove_punctuation_from_words(words):\n    return [remove_punctuation_from_word(w) for w in words]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned = train.copy()\n\ntrain_cleaned['comment_text'] = train['comment_text'].apply(word_tokenize).apply(words_to_lowercase).apply(remove_stop_words).apply(remove_punctuation_from_words).apply(remove_empty)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Detection of emojis\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Installing emot library\n!pip install emot\n#Importing libraries\nimport re\nfrom emot.emo_unicode import UNICODE_EMO, EMOTICONS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for converting emojis into word\ndef convert_emojis(text):\n    for emot in UNICODE_EMO:\n        text = text.replace(emot, \"_\".join(UNICODE_EMO[emot].replace(\",\",\"\").replace(\":\",\"\").split()))\n    return text\n# Example\ntext1 = \"Hilarious 😂. The feeling of making a sale 😎, The feeling of actually fulfilling orders 😒\"\nconvert_emojis(text1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Detection of abreviation \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install scispacy\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scispacy.abbreviation import AbbreviationDetector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"abbreviation_pipe=AbbreviationDetector(nlp)\nnlp.add_pipe(abbreviation_pipe)\ndef replace_acronyms(text):\n    doc=nlp(txt)\n    altered_tok=[tok.text for tok in doc]\n    print(doc._.abbreviations)\n    for abrv in doc._.abbreviations:\n        altered_tok[abrv.start]=str(abrv._.long_form)\n    return(\" \"join(altered_tok))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### stemming"},{"metadata":{"trusted":true},"cell_type":"code","source":"ps = PorterStemmer() \n\ndef stem_words(words):\n    return [ps.stem(w) for w in words]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_cleaned['comment_text'] = train_cleaned['comment_text'].apply(stem_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Lemmatization"},{"metadata":{"trusted":true},"cell_type":"code","source":"lemmatizer = WordNetLemmatizer() \n\ndef lemmatize_words(words):\n    return [lemmatizer.lemmatize(w) for w in words]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned['comment_text'] = train_cleaned['comment_text'].apply(lemmatize_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vectorisation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned['comment_text'] = train_cleaned['comment_text'].apply(\" \".join)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Count vectorization"},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vectorizer = CountVectorizer() \ncount_vectorizer.fit(train_cleaned['comment_text']) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Vocabulary: \", count_vectorizer.vocabulary_) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" count_vector = count_vectorizer.transform(train_cleaned['comment_text']) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TF-IDF"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_vector = tfidf_vectorizer.fit_transform(train_cleaned['comment_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Vocabulary: \", tfidf_vectorizer.vocabulary_) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> ## Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_count = MultinomialNB()\nclf_tfidf = MultinomialNB()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction using Count vectorization"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_df = count_vector\ny_df = train_cleaned.toxic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(x_df, y_df, test_size=0.10, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf_regression = LogisticRegression().fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_regression.score(x_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_count.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_count.score(x_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_comment_example = \"haha you are losers\"\nnon_toxic_comment_example = \"thank you for your response\"\n\ntoxic_comment_example_vect = count_vectorizer.transform([toxic_comment_example]).toarray()\nnon_toxic_comment_example_vect = count_vectorizer.transform([non_toxic_comment_example]).toarray()\n\ntoxic_comment_example_prediction = clf_count.predict(toxic_comment_example_vect)\nnon_toxic_comment_example_prediction = clf_count.predict(non_toxic_comment_example_vect)\n\nprint(\"prediction for toxic comment example : \",toxic_comment_example_prediction[0])\nprint(\"prediction for non toxic comment example : \",non_toxic_comment_example_prediction[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction using TF-IDF"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_df = tfidf_vector\ny_df = train_cleaned.toxic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(x_df, y_df, test_size=0.10, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_tfidf.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf_tfidf.score(x_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_comment_example = \"haha you are losers\"\nnon_toxic_comment_example = \"thank you for your response\"\n\ntoxic_comment_example_vect = tfidf_vectorizer.transform([toxic_comment_example])\nnon_toxic_comment_example_vect = tfidf_vectorizer.transform([non_toxic_comment_example])\n\ntoxic_comment_example_prediction = clf_tfidf.predict(toxic_comment_example_vect)\nnon_toxic_comment_example_prediction = clf_tfidf.predict(non_toxic_comment_example_vect)\n\nprint(\"prediction for toxic comment example : \",toxic_comment_example_prediction[0])\nprint(\"prediction for non toxic comment example : \",non_toxic_comment_example_prediction[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_comment_example = \"haha you are losers\"\nnon_toxic_comment_example = \"thank you for your response\"\n\ntoxic_comment_example_vect = count_vectorizer.transform([toxic_comment_example]).toarray()\nnon_toxic_comment_example_vect = count_vectorizer.transform([non_toxic_comment_example]).toarray()\n\ntoxic_comment_example_prediction = clf_regression.predict(toxic_comment_example_vect)\nnon_toxic_comment_example_prediction = clf_regression.predict(non_toxic_comment_example_vect)\n\nprint(\"prediction for toxic comment example : \",toxic_comment_example_prediction[0])\nprint(\"prediction for non toxic comment example : \",non_toxic_comment_example_prediction[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install transformers","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}