{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import nltk\nimport random\nimport pandas as pd\nimport re","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Getting Datasets"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"])\ndf2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"])\n\n# Merging the training csv files\ntrain = pd.concat([df1, df2], axis = 0, sort = False).reset_index(drop=True)\n\ntest_translated = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\ntest = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\nvalid = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\nsubmission = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_translated.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_shape = train.shape[0]\ntest_shape = test.shape[0]\nsum = train_shape + test_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"    :    train    :    test\")\nprint(\"rows:    {} :      {}\".format(train_shape, test_shape))\nprint(\"perc:    {} :      {}\".format(train_shape*100/sum,test_shape*100/sum))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = [\"toxic\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cleaning(sen):\n    sen = re.sub(r\"what's\",\"what is\",sen)\n    sen = re.sub(r\"\\'s'\",\" \",sen)\n    sen = re.sub(r\"\\'ve'\",\" have \",sen)\n    sen = re.sub(r\"can't\",\"cannot\",sen)\n    sen = re.sub(r\"n't\",\" not \",sen)\n    sen = re.sub(r\"i'm\",\"i am \",sen)\n    sen = re.sub(r\"\\'re'\",\" are \",sen)\n    sen = re.sub(r\"\\'d\",\" would \",sen)\n    sen = re.sub(r\"\\'ll\",\"will\",sen)\n    sen = re.sub(r\"\\'scuse\", \"excuse\",sen)\n    sen = re.sub(\"\\W\",\" \",sen)\n    sen = re.sub(r\"\\s+\",\" \",sen)\n    sen = sen.strip(' ')\n    return sen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cleaned_training_data = []\nfor i in range(len(train)):\n    cleaned_comment = cleaning(train[\"comment_text\"][i])\n    cleaned_training_data.append(cleaned_comment)\ntrain[\"comment_text\"] = pd.Series(cleaned_training_data).astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.comment_text\ny = train[\"toxic\"]\nX_translated = test_translated.translated\nX_multi = test.content","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nvector = TfidfVectorizer(max_features=2000, min_df=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_dtm = vector.fit_transform(X_train)\nX_test_dtm = vector.fit_transform(X_test)\nX_translated_dtm = vector.fit_transform(X_translated)\nX_multi_dtm = vector.fit_transform(X_multi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\nmodel_log = LogisticRegression(C=6.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = y_train.astype('int')\ny_test = y_test.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_prob = pd.DataFrame()\nfor label in targets:\n    print(\"...On the label:{}\".format(label))\n    model_log.fit(X_train_dtm, y_train)\n    predictions = model_log.predict(X_train_dtm)\n    print(\"Training accuracy: {}\".format(accuracy_score(y_train, predictions)))\n    probability_train = model_log.predict_proba(X_train_dtm)[:,1]\n    print(classification_report(y_train, predictions))\n    print()\n    print(confusion_matrix(y_train, predictions))\n    training_prob[label] = probability_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_prob.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Testing accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"testing_prob = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for label in targets:\n    print(\"...On the label:{}\".format(label))\n    predictions = model_log.predict(X_test_dtm)\n    print(\"Testing accuracy: {}\".format(accuracy_score(y_test, predictions)))\n    probability_test = model_log.predict_proba(X_test_dtm)[:,1]\n    print(classification_report(y_test, predictions))\n    print()\n    print(confusion_matrix(y_test, predictions))\n    testing_prob[label] = probability_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testing_prob.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Multilingual dataset predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_prob = submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for label in targets:\n    predictions = model_log.predict(X_multi_dtm)\n    probability_multi = model_log.predict_proba(X_multi_dtm)[:,1]\n    multi_prob[label] = probability_multi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_prob.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Translated dataset predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"trans_prob = submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for label in targets:\n    predictions = model_log.predict(X_translated_dtm)\n    probability_trans = model_log.predict_proba(X_translated_dtm)[:,1]\n    trans_prob[label] = probability_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trans_prob.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_prob.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}