{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport sqlite3\nimport pandas as pd\nimport numpy as np\nimport nltk\nimport string\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_curve, auc\nfrom nltk.stem.porter import PorterStemmer\n\nimport re\n# Tutorial about Python regular expressions: https://pymotw.com/2/re/\nimport string\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom gensim.models import Word2Vec\nfrom gensim.models import KeyedVectors\nimport pickle\n\nfrom tqdm import tqdm\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Importing Data : \n* Data is imported from a different source as the original dataset contains comments in languages other than english as well and since we are implementing Logistic Regression with lots of feature engineering, it is better to work with a uniform and consistent data. \n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train1  = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\").dropna()\ntrain2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\").dropna()\nvalid = pd.read_csv('../input/jigsaw-translate-en/validation_en.csv')\ntest = pd.read_csv('../input/jigsaw-translate-en/test_en.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nprint(\"Number of data points in training set 1: \", train1.shape[0])\nprint(\"Number of data points in training set 2: \", train2.shape[0])\nprint(\"Number of data points in total training set 1: \", (train1.shape[0]+train2.shape[0]))\nprint(\"Number of data points in validation set: \", valid.shape[0])\nprint(\"Number of data points in test set : \", test.shape[0])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print([print('{} \\n'.format(i)) for i in train2.comment_text[(train2.toxic>0.4) & (train2.toxic <0.5)][:7]])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- As we can see that comments with a toxicity score of more than 0.4 but less than 0.5 can also be considered as toxic, It might be wise to set the threshold to >0.4 or >0.35 when we are rounding it off to 0 or 1 for a classification problem. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train2.toxic = train2.toxic.apply(lambda x: 1 if x>0.4 else 0)\nvalid = valid.drop(columns = ['comment_text', 'id', 'lang'])\ntest = test.drop(columns = ['content', 'id', 'lang'])\ntrain = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']]\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.toxic.value_counts().plot(kind = 'bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- We can observe from the plot above that there is a significant class imbalance between toxic and non toxic comments in our training dataset.\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Undersampling:\n- since there is a significant difference between toxic and non-toxic comments, it makes sense to undersample our model to not be susceptible to any bias ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import resample\ntoxic = train[train.toxic ==1]\nnot_toxic = train[train.toxic == 0]\n\ndownsampled = resample(not_toxic,\n                       replace = False, # sample without replacement\n                       n_samples = len(toxic), # match minority n\n                       random_state = 10) # reproducible results\ntrain = pd.concat([downsampled, toxic])\ntrain.toxic.value_counts().plot(kind = 'bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = train['comment_text']\nx_valid = valid['content_en']\nx_test = test['content_en']\ny_train = train['toxic']\ny_valid = valid['toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/a/47091490/4084039\nimport re\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm\ndef decontracted(phrase):\n    # specific\n    phrase = re.sub(r\"won't\", \"will not\", phrase)\n    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n\n    # general\n    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n    return phrase\nstopwords= set(['br', 'the', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\\\n            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \\\n            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\\\n            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', \\\n            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \\\n            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \\\n            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\\\n            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\\\n            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\\\n            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \\\n            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', \\\n            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\\\n            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\\\n            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", \\\n            'won', \"won't\", 'wouldn', \"wouldn't\"])\npreprocessed_comments_train = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_train.values):\n    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n    sentance = decontracted(sentance)\n    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n    # https://gist.github.com/sebleier/554280\n    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n    preprocessed_comments_train.append(sentance.strip())\n\npreprocessed_comments_valid = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_valid.values):\n    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n    sentance = decontracted(sentance)\n    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n    # https://gist.github.com/sebleier/554280\n    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n    preprocessed_comments_valid.append(sentance.strip())\n\npreprocessed_comments_test = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(x_test.values):\n    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n    sentance = decontracted(sentance)\n    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n    # https://gist.github.com/sebleier/554280\n    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords)\n    preprocessed_comments_test.append(sentance.strip())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### BagOfWords :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# We are considering only the words which appeared in at least 10 comments .\nvectorizer = CountVectorizer(min_df=10)\nvectorizer.fit(preprocessed_comments_train)\nx_train_comments_bow = vectorizer.transform(preprocessed_comments_train)\nx_valid_comments_bow = vectorizer.transform(preprocessed_comments_valid)\nx_test_comments_bow = vectorizer.transform(preprocessed_comments_test)\n\nfeature_names_comments_bow_one_hot = vectorizer.get_feature_names()\nprint(\"Shape of matrix after one hot encodig \",x_train_comments_bow.shape)\nprint(\"Shape of matrix after one hot encodig \",x_valid_comments_bow.shape)\nprint(\"Shape of matrix after one hot encodig \",x_test_comments_bow.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TFIDF :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nvectorizer = TfidfVectorizer(min_df=10)\nvectorizer.fit_transform(preprocessed_comments_valid)\nx_train_tfidf_comment = vectorizer.transform(preprocessed_comments_train)\nx_valid_tfidf_comment = vectorizer.transform(preprocessed_comments_valid)\nx_test_tfidf_comment = vectorizer.transform(preprocessed_comments_test)\nfeature_names_tfidf_comment = vectorizer.get_feature_names()\nprint(\"Shape of matrix after one hot encodig \",x_train_tfidf_comment.shape)\nprint(\"Shape of matrix after one hot encodig \",x_valid_tfidf_comment.shape)\nprint(\"Shape of matrix after one hot encodig \",x_test_tfidf_comment.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training the w2v Model with X_train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def cleanPunctuation(sentence):                   #https://stackoverflow.com/questions/13237533/find-different-realization-of-a-word-in-a-sentence-string-python   \n    exclude = set(string.punctuation)\n    return ''.join(ch for ch in sentence if ch not in exclude)\n\nfrom bs4 import BeautifulSoup\ni = 0 \nlist_of_sent = []\nfor sent1 in tqdm(preprocessed_comments_train):\n    filtered_sent = []\n    soup = BeautifulSoup(sent1, 'html.parser')\n#     soup_packtpage = BeautifulSoup(sent,\"html.parser\")\n    sentt = soup.get_text()\n    for w in sentt.split():\n        for clean_words in cleanPunctuation(w).split():\n            filtered_sent.append(clean_words.lower())\n        else:\n            continue\n    list_of_sent.append(filtered_sent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport gensim\nw2v_model = gensim.models.Word2Vec(list_of_sent, min_count=10,size=50,workers=-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Average w2v on comments :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_avg_w2v_vectors_essay = []; # the avg-w2v for each sentence/comment is stored in this list\nglove_words = set(w2v_model.wv.vocab)\nfor sentence in tqdm(preprocessed_comments_train): # for each comment/sentence\n    vector = np.zeros(50) # as word vectors are of zero length\n    cnt_words =0; # num of wordswith a valid vector in the sentence/comment  \n    for word in sentence.split(): # for each word in a comment \n        if word in glove_words:\n            vector += w2v_model.wv[word]\n            cnt_words += 1\n        else:\n            pass\n    if cnt_words != 0:\n        vector /= cnt_words\n    x_train_avg_w2v_vectors_essay.append(vector)\n\nprint(len(x_train_avg_w2v_vectors_essay))\nprint(len(x_train_avg_w2v_vectors_essay[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid_avg_w2v_vectors_essay = []; # the avg-w2v for each sentence/comment is stored in this list\nglove_words = set(w2v_model.wv.vocab)\nfor sentence in tqdm(preprocessed_comments_valid): # for each comment/sentence\n    vector = np.zeros(50) # as word vectors are of zero length\n    cnt_words =0; # num of wordswith a valid vector in the sentence/comment  \n    for word in sentence.split(): # for each word in a comment \n        if word in glove_words:\n            vector += w2v_model.wv[word]\n            cnt_words += 1\n        else:\n            pass\n    if cnt_words != 0:\n        vector /= cnt_words\n    x_valid_avg_w2v_vectors_essay.append(vector)\n\nprint(len(x_valid_avg_w2v_vectors_essay))\nprint(len(x_valid_avg_w2v_vectors_essay[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_avg_w2v_vectors_essay = []; # the avg-w2v for each sentence/comment is stored in this list\nglove_words = set(w2v_model.wv.vocab)\nfor sentence in tqdm(preprocessed_comments_test): # for each comment/sentence\n    vector = np.zeros(50) # as word vectors are of zero length\n    cnt_words =0; # num of wordswith a test vector in the sentence/comment  \n    for word in sentence.split(): # for each word in a comment \n        if word in glove_words:\n            vector += w2v_model.wv[word]\n            cnt_words += 1\n        else:\n            pass\n    if cnt_words != 0:\n        vector /= cnt_words\n    x_test_avg_w2v_vectors_essay.append(vector)\n\nprint(len(x_test_avg_w2v_vectors_essay))\nprint(len(x_test_avg_w2v_vectors_essay[0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacking up feature Vectors:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# from numpy import hstack \nfrom scipy.sparse import hstack\n\nx_train_stack = hstack([\n    x_train_comments_bow, \n    x_train_tfidf_comment, \n    x_train_avg_w2v_vectors_essay\n])\n\n\nx_valid_stack = hstack([\n    x_valid_comments_bow, \n    x_valid_tfidf_comment, \n    x_valid_avg_w2v_vectors_essay\n])\n\n\nx_test_stack = hstack([\n    x_test_comments_bow, \n    x_test_tfidf_comment, \n    x_test_avg_w2v_vectors_essay\n])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying Logistic Regression:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/a/44149119\nfrom sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, roc_curve\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\n\ndef ROC_CM(y_train_true, y_train_prob, y_test_true, y_test_prob):\n    '''\n    a funciton to plot the ROC curve for train labels and test labels.\n    Use the best threshold found in train set to classify items in test set.\n    '''\n    fpr_train, tpr_train, thresholds_train = roc_curve(y_train_true, y_train_prob, pos_label =True)\n    sum_sensitivity_specificity_train = tpr_train + (1-fpr_train)\n    best_threshold_id_train = np.argmax(sum_sensitivity_specificity_train)\n    best_threshold = thresholds_train[best_threshold_id_train]\n    best_fpr_train = fpr_train[best_threshold_id_train]\n    best_tpr_train = tpr_train[best_threshold_id_train]\n    y_train = y_train_prob > best_threshold\n\n    cm_train = confusion_matrix(y_train_true, y_train)\n    acc_train = accuracy_score(y_train_true, y_train)\n    auc_train = roc_auc_score(y_train_true, y_train)\n\n    print('Train Accuracy: %s ' %acc_train)\n    print('Train AUC: %s ' %auc_train)\n#     print('Train Confusion Matrix:')\n#     print(cm_train)\n\n    fig = plt.figure(figsize=(15,13))\n    ax = fig.add_subplot(221)\n    curve1 = ax.plot(fpr_train, tpr_train)\n    curve2 = ax.plot([0, 1], [0, 1], color='navy', linestyle='--')\n    dot = ax.plot(best_fpr_train, best_tpr_train, marker='o', color='black')\n    ax.text(best_fpr_train, best_tpr_train, s = '(%.3f,%.3f)' %(best_fpr_train, best_tpr_train))\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.0])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('ROC curve (Train), AUC = %.4f'%auc_train)\n\n    fpr_test, tpr_test, thresholds_test = roc_curve(y_test_true, y_test_prob, pos_label =True)\n\n    y_test = y_test_prob > best_threshold\n\n    cm_test = confusion_matrix(y_test_true, y_test)\n    acc_test = accuracy_score(y_test_true, y_test)\n    auc_test = roc_auc_score(y_test_true, y_test)\n\n    print('Test Accuracy: %s ' %acc_test)\n    print('Test AUC: %s ' %auc_test)\n#     print('Test Confusion Matrix:')\n#     print(cm_test)\n\n    tpr_score = float(cm_test[1][1])/(cm_test[1][1] + cm_test[1][0])\n    fpr_score = float(cm_test[0][1])/(cm_test[0][0]+ cm_test[0][1])\n\n    ax2 = fig.add_subplot(222)\n    curve1 = ax2.plot(fpr_test, tpr_test)\n    curve2 = ax2.plot([0, 1], [0, 1], color='navy', linestyle='--')\n    dot = ax2.plot(fpr_score, tpr_score, marker='o', color='black')\n    ax2.text(fpr_score, tpr_score, s = '(%.3f,%.3f)' %(fpr_score, tpr_score))\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.0])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('ROC curve (Test), AUC = %.4f'%auc_test)\n    plt.savefig('ROC', dpi = 500)\n\n    df_cm_train = pd.DataFrame(cm_train, index = [i for i in [\"True Positive\",\"False Positive\"]],\n                         columns = [i for i in [\"False Negative\", \"True Negative\"]])\n    ax3 = fig.add_subplot(223)\n    plt.title(\"Train Confusion Matrix\")\n    sns.heatmap(df_cm_train, annot=True,fmt=\"d\")\n    df_cm_test = pd.DataFrame(cm_test, index = [i for i in [\"True Positive\",\"False Positive\"]],\n                         columns = [i for i in [\"False Negative\", \"True Negative\"]])\n    ax3 = fig.add_subplot(224)\n    \n    plt.title(\"Test Confusion Matrix\")\n    sns.heatmap(df_cm_test, annot=True,fmt=\"d\")\n    \n    plt.show()\n    \n    return best_threshold\n# ROC_CM(Y_train, Y_train_pred, Y_test, Y_test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import SGDClassifier\n\n\nclassifier = SGDClassifier(loss = 'log',class_weight='balanced')\nparameters = {'alpha':[0.001,0.005, 0.01, 0.015, 0.020, 0.025, .030, 0.035, 0.040]} \nclf = GridSearchCV(classifier, \n                   parameters, \n                   cv=3, \n                   scoring='roc_auc',\n                   return_train_score=True)\nclf.fit(x_train_stack, y_train)\n\ntrain_auc= clf.cv_results_['mean_train_score']\ntrain_auc_std= clf.cv_results_['std_train_score']\ncv_auc = clf.cv_results_['mean_test_score'] \ncv_auc_std= clf.cv_results_['std_test_score']\n\nplt.plot(parameters['alpha'], train_auc, label='Train AUC')\n# this code is copied from here: https://stackoverflow.com/a/48803361/4084039\nplt.gca().fill_between(parameters['alpha'],train_auc - train_auc_std,train_auc + train_auc_std,alpha=0.2,color='darkblue')\n\nplt.plot(parameters['alpha'], cv_auc, label='CV AUC')\n# this code is copied from here: https://stackoverflow.com/a/48803361/4084039\nplt.gca().fill_between(parameters['alpha'],cv_auc - cv_auc_std,cv_auc + cv_auc_std,alpha=0.2,color='darkorange')\n\nplt.scatter(parameters['alpha'], train_auc, label='Train AUC points')\nplt.scatter(parameters['alpha'], cv_auc, label='CV AUC points')\n\n\nplt.legend()\nplt.xlabel(\"log(Alpha): hyperparameter\")\nplt.ylabel(\"AUC\")\nplt.title(\"ERROR PLOTS\")\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_alpha = clf.best_params_['alpha']\n# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_curve.html#sklearn.metrics.roc_curve\nfrom sklearn.metrics import roc_curve, auc\n\nLR = SGDClassifier(loss = 'log',alpha = best_alpha)\n\nLR.fit(x_train_stack, y_train)\n# roc_auc_score(Y_true, Y_score) the 2nd parameter should be probability estimates of the positive class\n# not the predicted outputs\n\n# Y_train_pred = batch_predict(LR, x_train_stack)    \n# Y_test_pred = batch_predict(LR, x_cv_stack)\n\nY_train_pred = LR.predict_proba(x_train_stack)    \nY_valid_pred = LR.predict_proba(x_valid_stack)\n\ntrain_fpr, train_tpr, tr_thresholds = roc_curve(y_train, Y_train_pred[:,1])\ntest_fpr, test_tpr, te_thresholds = roc_curve(y_valid, Y_valid_pred[:,1])\n\nplt.plot(train_fpr, train_tpr, label=\"Train AUC =\"+str(auc(train_fpr, train_tpr)))\nplt.plot(test_fpr, test_tpr, label=\"Test AUC =\"+str(auc(test_fpr, test_tpr)))\nplt.legend()\nplt.xlabel(\"True Positive Rate(TPR)\")\nplt.ylabel(\"False Positive Rate(FPR)\")\nplt.title(\"AUC\")\nplt.grid()\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROC_CM(y_train, Y_train_pred[:,1],y_valid, Y_valid_pred[:,1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- As we can see that a simple linear regression model with very miniscule hyperparamater tuning results in significantly satisfactory results. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_predictions = LR.predict_proba(x_test_stack)\nsub['toxic'] = submission_predictions[:,1]\nsub.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}