{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-20T10:18:20.414160Z","iopub.execute_input":"2021-05-20T10:18:20.414500Z","iopub.status.idle":"2021-05-20T10:18:20.424423Z","shell.execute_reply.started":"2021-05-20T10:18:20.414472Z","shell.execute_reply":"2021-05-20T10:18:20.423154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:20.428862Z","iopub.execute_input":"2021-05-20T10:18:20.429442Z","iopub.status.idle":"2021-05-20T10:18:20.441486Z","shell.execute_reply.started":"2021-05-20T10:18:20.429389Z","shell.execute_reply":"2021-05-20T10:18:20.440349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load training data\ntrain_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:20.451666Z","iopub.execute_input":"2021-05-20T10:18:20.452163Z","iopub.status.idle":"2021-05-20T10:18:25.369930Z","shell.execute_reply.started":"2021-05-20T10:18:20.452130Z","shell.execute_reply":"2021-05-20T10:18:25.369043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.371126Z","iopub.execute_input":"2021-05-20T10:18:25.371541Z","iopub.status.idle":"2021-05-20T10:18:25.381468Z","shell.execute_reply.started":"2021-05-20T10:18:25.371510Z","shell.execute_reply":"2021-05-20T10:18:25.380427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.383211Z","iopub.execute_input":"2021-05-20T10:18:25.383486Z","iopub.status.idle":"2021-05-20T10:18:25.399719Z","shell.execute_reply.started":"2021-05-20T10:18:25.383460Z","shell.execute_reply":"2021-05-20T10:18:25.398547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.401896Z","iopub.execute_input":"2021-05-20T10:18:25.402397Z","iopub.status.idle":"2021-05-20T10:18:25.662888Z","shell.execute_reply.started":"2021-05-20T10:18:25.402306Z","shell.execute_reply":"2021-05-20T10:18:25.661940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rename the *question_text* column for convenience\ntrain_df = train_df.rename({'question_text': 'question'}, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.665402Z","iopub.execute_input":"2021-05-20T10:18:25.665778Z","iopub.status.idle":"2021-05-20T10:18:25.702345Z","shell.execute_reply.started":"2021-05-20T10:18:25.665745Z","shell.execute_reply":"2021-05-20T10:18:25.701384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.703481Z","iopub.execute_input":"2021-05-20T10:18:25.703781Z","iopub.status.idle":"2021-05-20T10:18:25.830179Z","shell.execute_reply.started":"2021-05-20T10:18:25.703753Z","shell.execute_reply":"2021-05-20T10:18:25.829113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.831305Z","iopub.execute_input":"2021-05-20T10:18:25.831617Z","iopub.status.idle":"2021-05-20T10:18:25.966788Z","shell.execute_reply.started":"2021-05-20T10:18:25.831588Z","shell.execute_reply":"2021-05-20T10:18:25.965717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.969224Z","iopub.execute_input":"2021-05-20T10:18:25.969520Z","iopub.status.idle":"2021-05-20T10:18:25.980706Z","shell.execute_reply.started":"2021-05-20T10:18:25.969491Z","shell.execute_reply":"2021-05-20T10:18:25.979751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:25.982988Z","iopub.execute_input":"2021-05-20T10:18:25.983388Z","iopub.status.idle":"2021-05-20T10:18:26.008330Z","shell.execute_reply.started":"2021-05-20T10:18:25.983345Z","shell.execute_reply":"2021-05-20T10:18:26.007298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.ticker as ticker\n\nncount = train_df.shape[0]\n\nplt.figure(figsize=(7, 5))\n\nax = sns.countplot(data=train_df, x='target')\nplt.title('Portion of Questions')\nplt.xlabel('Number of Axles')\n\n# Make twin axis\nax2=ax.twinx()\n\n# Switch so count axis is on right, frequency on left\nax2.yaxis.tick_left()\nax.yaxis.tick_right()\n\n# Also switch the labels over\nax.yaxis.set_label_position('right')\nax2.yaxis.set_label_position('left')\n\nax2.set_ylabel('Frequency [%]')\n\nfor p in ax.patches:\n    x=p.get_bbox().get_points()[:,0]\n    y=p.get_bbox().get_points()[1,1]\n    ax.annotate('{:.1f}%'.format(100.*y/ncount), (x.mean(), y), \n            ha='center', va='bottom') # set the alignment of the text\n\n# Use a LinearLocator to ensure the correct number of ticks\nax.yaxis.set_major_locator(ticker.LinearLocator(11))\n\n# Fix the frequency range to 0-100\nax2.set_ylim(0,100)\nax.set_ylim(0,ncount)\n\n# And use a MultipleLocator to ensure a tick spacing of 10\nax2.yaxis.set_major_locator(ticker.MultipleLocator(10))\n\n# Need to turn the grid on ax2 off, otherwise the gridlines end up on top of the bars\nax2.grid(None)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:26.009719Z","iopub.execute_input":"2021-05-20T10:18:26.010265Z","iopub.status.idle":"2021-05-20T10:18:26.374497Z","shell.execute_reply.started":"2021-05-20T10:18:26.010223Z","shell.execute_reply":"2021-05-20T10:18:26.373792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\ninsincere_qes = train_df[train_df['target'] == 1]\nprint(insincere_qes[-5:].question.to_string())","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:26.375489Z","iopub.execute_input":"2021-05-20T10:18:26.375891Z","iopub.status.idle":"2021-05-20T10:18:26.399647Z","shell.execute_reply.started":"2021-05-20T10:18:26.375849Z","shell.execute_reply":"2021-05-20T10:18:26.398736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\nsincere_qes = train_df[train_df['target'] == 0]\nprint(sincere_qes[-5:].question)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:26.400905Z","iopub.execute_input":"2021-05-20T10:18:26.401173Z","iopub.status.idle":"2021-05-20T10:18:26.474661Z","shell.execute_reply.started":"2021-05-20T10:18:26.401146Z","shell.execute_reply":"2021-05-20T10:18:26.473699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Words cloud**","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:26.476116Z","iopub.execute_input":"2021-05-20T10:18:26.476549Z","iopub.status.idle":"2021-05-20T10:18:26.481015Z","shell.execute_reply.started":"2021-05-20T10:18:26.476507Z","shell.execute_reply":"2021-05-20T10:18:26.479966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Word cloud image generated from insincere questions')\ninsincere_wordcloud = WordCloud(width=800, height=400, background_color ='black', min_font_size = 10).generate(str(train_df[train_df[\"target\"] == 1][\"question\"]))\n#Positive Word cloud\nplt.figure(figsize=(15,6), facecolor=None)\nplt.imshow(insincere_wordcloud)\nplt.axis(\"off\")\n# plt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:26.482689Z","iopub.execute_input":"2021-05-20T10:18:26.483318Z","iopub.status.idle":"2021-05-20T10:18:27.015540Z","shell.execute_reply.started":"2021-05-20T10:18:26.483271Z","shell.execute_reply":"2021-05-20T10:18:27.014607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Word cloud image generated from sincere questions')\nsincere_wordcloud = WordCloud(width=600, height=400, background_color ='black', min_font_size = 10).generate(str(train_df[train_df[\"target\"] == 0][\"question\"]))\n#Positive Word cloud\nplt.figure(figsize=(15,6), facecolor=None)\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:27.016883Z","iopub.execute_input":"2021-05-20T10:18:27.017200Z","iopub.status.idle":"2021-05-20T10:18:27.657006Z","shell.execute_reply.started":"2021-05-20T10:18:27.017170Z","shell.execute_reply":"2021-05-20T10:18:27.655952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Text statistics** ","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:27.658285Z","iopub.execute_input":"2021-05-20T10:18:27.658595Z","iopub.status.idle":"2021-05-20T10:18:27.668610Z","shell.execute_reply.started":"2021-05-20T10:18:27.658553Z","shell.execute_reply":"2021-05-20T10:18:27.667860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import string\nfrom nltk.corpus import stopwords\n\n# Set up contraction dictionary\ncontraction_dict = {\"dont\": \"do not\", \"aint\": \"is not\", \"isnt\": \"is not\", \"doesnt\": \"does not\", \"cant\": \"cannot\", \"mustnt\": \"must not\", \"hasnt\": \"has not\", \"havent\": \"have not\", \"arent\": \"are not\", \"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"‘cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"Iam\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}\n\n# Set up stop words list\nstop_words = stopwords.words('english')\nstop_words.remove('not')\n\n# Set up puntuation list\npunctuation = string.punctuation","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:27.669876Z","iopub.execute_input":"2021-05-20T10:18:27.670320Z","iopub.status.idle":"2021-05-20T10:18:27.687303Z","shell.execute_reply.started":"2021-05-20T10:18:27.670284Z","shell.execute_reply":"2021-05-20T10:18:27.686585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Feature engineering****","metadata":{}},{"cell_type":"code","source":"#Feature Engineering on train_df data\n\ndef create_features(df):\n    \"\"\"Retrieve from the text column the number of: characters, words, unique words, stopwords,\n    punctuations, upper/lower case chars\"\"\"\n    df[\"lenght\"] = df[\"question\"].apply(lambda x: len(str(x)))\n    df[\"no_words\"] = df[\"question\"].apply(lambda x: len(x.split()))\n    df[\"no_unique_words\"] = df[\"question\"].apply(lambda x: len(set(str(x).split())))\n    df[\"no_stopwords\"] = df[\"question\"].apply(lambda x : len([nw for nw in str(x).split() if nw.lower() in stop_words]))\n    df[\"no_punctuation\"] = df[\"question\"].apply(lambda x : len([np for np in str(x) if np in punctuation]))\n    df[\"no_uppercase\"] = df[\"question\"].apply(lambda x : len([nu for nu in str(x).split() if nu.isupper()]))\n    df[\"no_lowercase\"] = df[\"question\"].apply(lambda x : len([nl for nl in str(x).split() if nl.islower()]))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:27.688490Z","iopub.execute_input":"2021-05-20T10:18:27.689012Z","iopub.status.idle":"2021-05-20T10:18:27.709696Z","shell.execute_reply.started":"2021-05-20T10:18:27.688971Z","shell.execute_reply":"2021-05-20T10:18:27.708545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = create_features(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:18:27.710977Z","iopub.execute_input":"2021-05-20T10:18:27.711488Z","iopub.status.idle":"2021-05-20T10:19:14.899487Z","shell.execute_reply.started":"2021-05-20T10:18:27.711456Z","shell.execute_reply":"2021-05-20T10:19:14.898674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The statistics of insincere questions","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.3f' % x)\ntrain_df[train_df['target'] == 0].describe()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:14.900715Z","iopub.execute_input":"2021-05-20T10:19:14.901196Z","iopub.status.idle":"2021-05-20T10:19:15.439618Z","shell.execute_reply.started":"2021-05-20T10:19:14.901165Z","shell.execute_reply":"2021-05-20T10:19:15.438564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The statistics of sincere questions","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.3f' % x)\ntrain_df[train_df['target'] == 1].describe()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:15.440764Z","iopub.execute_input":"2021-05-20T10:19:15.441031Z","iopub.status.idle":"2021-05-20T10:19:15.522416Z","shell.execute_reply.started":"2021-05-20T10:19:15.441005Z","shell.execute_reply":"2021-05-20T10:19:15.521333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_feat = ['lenght', 'no_unique_words', 'no_stopwords', \n#             'no_punctuation', 'no_uppercase', 'no_lowercase', 'target'] \n# # side note : remove target if needed later\n\n# dfsample = train_df[num_feat].sample(n=round(train_df.shape[0]/6), random_state=42)\n\n# plt.figure(figsize=(15,15))\n# sns.set_context(\"paper\", rc={\"axes.labelsize\":16})\n# sns.pairplot(data=dfsample, hue='target')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:15.523626Z","iopub.execute_input":"2021-05-20T10:19:15.523922Z","iopub.status.idle":"2021-05-20T10:19:15.526906Z","shell.execute_reply.started":"2021-05-20T10:19:15.523893Z","shell.execute_reply":"2021-05-20T10:19:15.526105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****The distribution of the questions's number of lowercases****","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nsns.displot(train_df, x='no_lowercase', hue='target', kind='hist', bins=50)\nplt.title(\"Distribution of the question's number of lowercases\")","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:15.530837Z","iopub.execute_input":"2021-05-20T10:19:15.531375Z","iopub.status.idle":"2021-05-20T10:19:16.518190Z","shell.execute_reply.started":"2021-05-20T10:19:15.531332Z","shell.execute_reply":"2021-05-20T10:19:16.517495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_correlation = train_df.corr()['target'][1:]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:16.519905Z","iopub.execute_input":"2021-05-20T10:19:16.520299Z","iopub.status.idle":"2021-05-20T10:19:16.868925Z","shell.execute_reply.started":"2021-05-20T10:19:16.520261Z","shell.execute_reply":"2021-05-20T10:19:16.868072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mask = np.zeros_like(train_df[num_feat].corr(), dtype=np.bool) \n# mask[np.triu_indices_from(mask)] = True \n\n# f, ax = plt.subplots(figsize=(10, 10))\n# plt.title('Question features Correlation Matrix',fontsize=20)\n\n# sns.heatmap(train_df[num_feat].corr(),square=True, linewidths=0.25,vmax=0.7,cmap=\"YlGnBu\",\n#             linecolor='w',annot=True,annot_kws={\"size\":10},mask=mask,cbar_kws={\"shrink\": .9});","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:16.870113Z","iopub.execute_input":"2021-05-20T10:19:16.870734Z","iopub.status.idle":"2021-05-20T10:19:16.874681Z","shell.execute_reply.started":"2021-05-20T10:19:16.870690Z","shell.execute_reply":"2021-05-20T10:19:16.873667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\n\ntarget_correlation = train_df.corr()['target'][1:]\nplt.plot(target_correlation)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:16.876153Z","iopub.execute_input":"2021-05-20T10:19:16.876747Z","iopub.status.idle":"2021-05-20T10:19:17.613371Z","shell.execute_reply.started":"2021-05-20T10:19:16.876700Z","shell.execute_reply":"2021-05-20T10:19:17.612458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"import re\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.stem import SnowballStemmer\n\nlemmatizer = WordNetLemmatizer()\nstemmer = SnowballStemmer('english')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:17.614403Z","iopub.execute_input":"2021-05-20T10:19:17.614685Z","iopub.status.idle":"2021-05-20T10:19:17.619826Z","shell.execute_reply.started":"2021-05-20T10:19:17.614658Z","shell.execute_reply":"2021-05-20T10:19:17.618686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question']","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:17.621359Z","iopub.execute_input":"2021-05-20T10:19:17.621795Z","iopub.status.idle":"2021-05-20T10:19:17.638650Z","shell.execute_reply.started":"2021-05-20T10:19:17.621749Z","shell.execute_reply":"2021-05-20T10:19:17.637491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def qes_preprocessing(qes):\n    # Data cleaning:\n    qes = re.sub(re.compile('<.*?>'), '', qes)\n    qes = re.sub('[^A-Za-z0-9]+', ' ', qes)\n\n    # Lowercase:\n    qes = qes.lower()\n\n    # Tokenization:\n    tokens = word_tokenize(qes)\n\n    # Contractions replacement:\n    tokens = [contraction_dict.get(token) if (contraction_dict.get(token) != None) else token for token in tokens]\n\n    # Stop words removal:\n    tokens = [w for w in tokens if w not in stop_words]\n\n    # Stemming:\n    tokens = [stemmer.stem(token) for token in tokens]\n    \n    # Lemmatization:\n    tokens = [lemmatizer.lemmatize(w) for w in tokens]\n\n    # Join words after preprocessed:\n    qes = ' '.join(tokens) \n    \n#     aug = naw.ContextualWordEmbsAug(\n#     model_path='bert-base-uncased', action=\"insert\")\n#     augmented_text = aug.augment(text)\n\n    return qes","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:17.640425Z","iopub.execute_input":"2021-05-20T10:19:17.640871Z","iopub.status.idle":"2021-05-20T10:19:17.651700Z","shell.execute_reply.started":"2021-05-20T10:19:17.640828Z","shell.execute_reply":"2021-05-20T10:19:17.650678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['preprocessed_questions'] = train_df['question'].apply(qes_preprocessing)\ntrain_df['preprocessed_questions']","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:19:17.652668Z","iopub.execute_input":"2021-05-20T10:19:17.653040Z","iopub.status.idle":"2021-05-20T10:27:19.238582Z","shell.execute_reply.started":"2021-05-20T10:19:17.653012Z","shell.execute_reply":"2021-05-20T10:27:19.237662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train model**","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import MultinomialNB\n\nfrom sklearn.metrics import f1_score, confusion_matrix, classification_report\n\nX = train_df['preprocessed_questions']\ny = train_df.target","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:19.239727Z","iopub.execute_input":"2021-05-20T10:27:19.240077Z","iopub.status.idle":"2021-05-20T10:27:19.246269Z","shell.execute_reply.started":"2021-05-20T10:27:19.240049Z","shell.execute_reply":"2021-05-20T10:27:19.245012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Vectorizing**","metadata":{}},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(lowercase=False, analyzer=lambda x: x, min_df=0.01, max_df=0.999)\n# min_df & max_df param added for less memory usage\n\ntf_idf = vectorizer.fit_transform(X).toarray()\npd.DataFrame(tf_idf, columns=vectorizer.get_feature_names()).head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:19.247682Z","iopub.execute_input":"2021-05-20T10:27:19.247962Z","iopub.status.idle":"2021-05-20T10:27:43.303343Z","shell.execute_reply.started":"2021-05-20T10:27:19.247936Z","shell.execute_reply":"2021-05-20T10:27:43.302402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(tf_idf, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:43.304628Z","iopub.execute_input":"2021-05-20T10:27:43.304913Z","iopub.status.idle":"2021-05-20T10:27:43.693598Z","shell.execute_reply.started":"2021-05-20T10:27:43.304885Z","shell.execute_reply":"2021-05-20T10:27:43.692634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model 1: MultinomialNB**","metadata":{}},{"cell_type":"code","source":"# # BOW\n# nb_bow_pipeline = Pipeline([(\"cv\", CountVectorizer(analyzer=\"word\", ngram_range=(2,4), max_df=0.85)),\n#                      (\"model\", MultinomialNB())])\n\n# # TF-IDF\n# nb_tdf_pipelione = Pipeline([(\"tfid\", TfidfVectorizer(lowercase=False, min_df=0.01, max_df=0.95)),\n#                      (\"model\", MultinomialNB())])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:43.695134Z","iopub.execute_input":"2021-05-20T10:27:43.695691Z","iopub.status.idle":"2021-05-20T10:27:43.700327Z","shell.execute_reply.started":"2021-05-20T10:27:43.695649Z","shell.execute_reply":"2021-05-20T10:27:43.699173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model 2: Logistic Regression**","metadata":{}},{"cell_type":"code","source":"# # BOW\n# lr_bow_pipeline = Pipeline([(\"cv\", CountVectorizer(analyzer=\"word\", ngram_range=(1,4), max_df=0.9)),\n#                      (\"model\", LogisticRegression(solver=\"saga\", class_weight=\"balanced\", C=0.45, max_iter=250, verbose=1, n_jobs=-1))\n#                            ])\n\n# # TF-IDF\n# lr_tdf_pipeline = Pipeline([(\"tfid\", TfidfVectorizer(lowercase=False, min_df=0.01, max_df=0.95)),\n#                      (\"model\", LogisticRegression(solver=\"saga\", class_weight=\"balanced\", C=0.45, max_iter=250, verbose=1, n_jobs=-1))\n#                            ])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:43.701865Z","iopub.execute_input":"2021-05-20T10:27:43.702316Z","iopub.status.idle":"2021-05-20T10:27:43.715662Z","shell.execute_reply.started":"2021-05-20T10:27:43.702273Z","shell.execute_reply":"2021-05-20T10:27:43.714522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model 3: XGBoost Classifier without weigths**","metadata":{}},{"cell_type":"code","source":"def get_fscore_matrix(fitted_clf, model_name):\n    print(model_name, ' :')\n    \n    # get classes predictions for the classification report \n    y_train_pred, y_pred = fitted_clf.predict(X_train), fitted_clf.predict(X_test)\n    print(classification_report(y_test, y_pred), '\\n') # target_names=y\n    \n    # computes probabilities keep the ones for the positive outcome only      \n    print(f'F1-score = {f1_score(y_test, y_pred):.2f}')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:43.717298Z","iopub.execute_input":"2021-05-20T10:27:43.717916Z","iopub.status.idle":"2021-05-20T10:27:43.731848Z","shell.execute_reply.started":"2021-05-20T10:27:43.717847Z","shell.execute_reply":"2021-05-20T10:27:43.730709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\n\nXGBmodel = XGBClassifier(objective=\"binary:logistic\")\nXGBmodel.fit(X_train, y_train)\nget_fscore_matrix(XGBmodel, 'XGB Clf withOUT weights')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:27:43.733295Z","iopub.execute_input":"2021-05-20T10:27:43.733729Z","iopub.status.idle":"2021-05-20T10:32:17.978125Z","shell.execute_reply.started":"2021-05-20T10:27:43.733686Z","shell.execute_reply":"2021-05-20T10:32:17.976838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model 4: XGBoost Classifier with weigths**","metadata":{}},{"cell_type":"code","source":"ratio = ((len(y_train) - y_train.sum()) - y_train.sum()) / y_train.sum()\nratio","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:32:17.979849Z","iopub.execute_input":"2021-05-20T10:32:17.980278Z","iopub.status.idle":"2021-05-20T10:32:17.991116Z","shell.execute_reply.started":"2021-05-20T10:32:17.980234Z","shell.execute_reply":"2021-05-20T10:32:17.989968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WXGBmodel = XGBClassifier(objective=\"binary:logistic\", scale_pos_weight=ratio)\nWXGBmodel.fit(X_train, y_train)\nget_fscore_matrix(WXGBmodel, 'XGB Clf WITH weights')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:32:17.992655Z","iopub.execute_input":"2021-05-20T10:32:17.993070Z","iopub.status.idle":"2021-05-20T10:36:56.259995Z","shell.execute_reply.started":"2021-05-20T10:32:17.993026Z","shell.execute_reply":"2021-05-20T10:36:56.258959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model 5: LGBM with weights**","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\nLGBMmodel = lgb.LGBMClassifier(n_jobs = -1, class_weight={0:y_train.sum(), 1:len(y_train) - y_train.sum()})\nLGBMmodel.fit(X_train, y_train)\nget_fscore_matrix(LGBMmodel, 'LGBM weighted')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:36:56.261299Z","iopub.execute_input":"2021-05-20T10:36:56.261626Z","iopub.status.idle":"2021-05-20T10:37:10.700530Z","shell.execute_reply.started":"2021-05-20T10:36:56.261584Z","shell.execute_reply":"2021-05-20T10:37:10.699421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Add model to model list**","metadata":{}},{"cell_type":"code","source":"# models = [nb_bow_pipeline, nb_tdf_pipelione, lr_bow_pipeline, lr_tdf_pipeline]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:10.701797Z","iopub.execute_input":"2021-05-20T10:37:10.702080Z","iopub.status.idle":"2021-05-20T10:37:10.705635Z","shell.execute_reply.started":"2021-05-20T10:37:10.702052Z","shell.execute_reply":"2021-05-20T10:37:10.704656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plot Vectorize matrix**","metadata":{}},{"cell_type":"code","source":"# import matplotlib.pyplot as plt  \n# from sklearn.datasets import make_classification\n# from sklearn.metrics import plot_confusion_matrix\n# from sklearn.model_selection import train_test_split\n# from sklearn.svm import SVC\n\n# # X, y = make_classification(random_state=0)\n# # X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n# # clf = SVC(random_state=0)\n# # clf.fit(X_train, y_train)\n# # SVC(random_state=0)\n# # plot_confusion_matrix(clf, X_test, y_test, normalize=None, cmap=plt.cm.Blues)\n# # plt.show()\n\n# vectorizer = CountVectorizer(analyzer='word', ngram_range=(1, 1))\n# vectorized = vectorizer.fit_transform(X)\n# pd.DataFrame(vectorized.toarray(), \n#             index=['sentence '+str(i) \n#                    for i in range(1, 1+len(X))],\n#             columns=vectorizer.get_feature_names())\n\n# # vectorizer = TfidfVectorizer()\n# # X = vectorizer.fit_transform(X)\n# # print(vectorizer.get_feature_names())\n# # print(X.shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:10.707150Z","iopub.execute_input":"2021-05-20T10:37:10.707784Z","iopub.status.idle":"2021-05-20T10:37:10.725914Z","shell.execute_reply.started":"2021-05-20T10:37:10.707749Z","shell.execute_reply":"2021-05-20T10:37:10.724964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# kfold = KFold(n_splits=2, shuffle=False)\n\n# fold = 1\n\n# for train_index, test_index in kfold.split(X, y):\n#     X_train, X_test, y_train, y_test = X[train_index], X[test_index], y[train_index], y[test_index] \n#     print(X_train)\n#     print(f'Fold {fold}:')\n#     for model_pipeline in models:\n#         print(model_pipeline)\n#         model_pipeline.fit(X_train, y_train)\n# #         predictions = model_pipeline.predict_log_proba(X_test)\n\n#         print(classification_report(y_test, predictions), '\\n') \n        \n#         print(f'F1-score = {f1_score(y_test, predictions):.2f}')\n\n#     fold += 1","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:10.727060Z","iopub.execute_input":"2021-05-20T10:37:10.727537Z","iopub.status.idle":"2021-05-20T10:37:10.743679Z","shell.execute_reply.started":"2021-05-20T10:37:10.727505Z","shell.execute_reply":"2021-05-20T10:37:10.742860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2. Load test.csv, make predictions and output submission**","metadata":{}},{"cell_type":"markdown","source":"**2.1 Load dataset**","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:10.744941Z","iopub.execute_input":"2021-05-20T10:37:10.745342Z","iopub.status.idle":"2021-05-20T10:37:11.580278Z","shell.execute_reply.started":"2021-05-20T10:37:10.745312Z","shell.execute_reply":"2021-05-20T10:37:11.579521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:11.581295Z","iopub.execute_input":"2021-05-20T10:37:11.581707Z","iopub.status.idle":"2021-05-20T10:37:11.591327Z","shell.execute_reply.started":"2021-05-20T10:37:11.581677Z","shell.execute_reply":"2021-05-20T10:37:11.590261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:11.592696Z","iopub.execute_input":"2021-05-20T10:37:11.593034Z","iopub.status.idle":"2021-05-20T10:37:11.685360Z","shell.execute_reply.started":"2021-05-20T10:37:11.593001Z","shell.execute_reply":"2021-05-20T10:37:11.684400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:11.686752Z","iopub.execute_input":"2021-05-20T10:37:11.687021Z","iopub.status.idle":"2021-05-20T10:37:11.692697Z","shell.execute_reply.started":"2021-05-20T10:37:11.686995Z","shell.execute_reply":"2021-05-20T10:37:11.691615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.2 Preprocessing data**","metadata":{}},{"cell_type":"code","source":"test_df['preprocessed'] = test_df['question_text'].apply(qes_preprocessing)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:37:11.694316Z","iopub.execute_input":"2021-05-20T10:37:11.694727Z","iopub.status.idle":"2021-05-20T10:39:30.636192Z","shell.execute_reply.started":"2021-05-20T10:37:11.694674Z","shell.execute_reply":"2021-05-20T10:39:30.635057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.3 Make predictions**","metadata":{}},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(lowercase=False, analyzer=lambda x: x, min_df=0.01, max_df=0.999)\n# min_df & max_df param added for less memory usage\n\ntf_idf = vectorizer.fit_transform(test_df['preprocessed']).toarray()\npd.DataFrame(tf_idf, columns=vectorizer.get_feature_names()).head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:30.637749Z","iopub.execute_input":"2021-05-20T10:39:30.638083Z","iopub.status.idle":"2021-05-20T10:39:37.535265Z","shell.execute_reply.started":"2021-05-20T10:39:30.638043Z","shell.execute_reply":"2021-05-20T10:39:37.534192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test, y_test = train_test_split(tf_idf, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.536886Z","iopub.execute_input":"2021-05-20T10:39:37.537222Z","iopub.status.idle":"2021-05-20T10:39:37.568187Z","shell.execute_reply.started":"2021-05-20T10:39:37.537190Z","shell.execute_reply":"2021-05-20T10:39:37.566189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = models[2].predict(test_df['preprocessed'])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.569238Z","iopub.status.idle":"2021-05-20T10:39:37.569676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction1 = XGBmodel.predict(tf_idf)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.571022Z","iopub.status.idle":"2021-05-20T10:39:37.571661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction2 = WXGBmodel.predict(tf_idf)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.572939Z","iopub.status.idle":"2021-05-20T10:39:37.573549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction3 = LGBMmodel.predict(tf_idf)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.575199Z","iopub.status.idle":"2021-05-20T10:39:37.575853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(prediction1))\nprint(prediction1[-10:])\nprint(test_df['question_text'][-10:])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.577160Z","iopub.status.idle":"2021-05-20T10:39:37.577809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.4 Make submission**","metadata":{}},{"cell_type":"code","source":"test_df['prediction'] = prediction1\nresults = test_df[['qid', 'prediction']]\nresults.to_csv('submission.csv', index=False)\nresults.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-20T10:39:37.579235Z","iopub.status.idle":"2021-05-20T10:39:37.579890Z"},"trusted":true},"execution_count":null,"outputs":[]}]}