{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-04T17:58:36.481472Z","iopub.execute_input":"2021-06-04T17:58:36.482089Z","iopub.status.idle":"2021-06-04T17:58:39.527152Z","shell.execute_reply.started":"2021-06-04T17:58:36.482038Z","shell.execute_reply":"2021-06-04T17:58:39.526216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Files:**\n\nFollowing data files are given.","metadata":{"_uuid":"c0d647a81bb5b1a13332a17fffad6c6b0e5cf787"}},{"cell_type":"code","source":"!ls ../input/","metadata":{"_uuid":"6fc5aa15387e3585ac51fbc5db66565a058f1a2d","execution":{"iopub.status.busy":"2021-06-04T17:58:59.874147Z","iopub.execute_input":"2021-06-04T17:58:59.874683Z","iopub.status.idle":"2021-06-04T17:59:00.607076Z","shell.execute_reply.started":"2021-06-04T17:58:59.874635Z","shell.execute_reply":"2021-06-04T17:59:00.605982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* train.csv - the training set\n* test.csv - the test set\n* sample_submission.csv - A sample submission in the correct format\n* enbeddings/ - Folder containing word embeddings.\n\nWe are not allowed to use any external data sources. The following embeddings are given to us which can be used for building our models.","metadata":{"_uuid":"ca4069cd714e42e1c9b6b3bb4f95c95aa0fc7838"}},{"cell_type":"code","source":"!ls ../input/embeddings/","metadata":{"_uuid":"5adcd5cd216cf95623326abf3235b7670a2af729","execution":{"iopub.status.busy":"2021-06-04T17:59:04.201141Z","iopub.execute_input":"2021-06-04T17:59:04.201522Z","iopub.status.idle":"2021-06-04T17:59:04.915174Z","shell.execute_reply.started":"2021-06-04T17:59:04.201466Z","shell.execute_reply":"2021-06-04T17:59:04.913507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* GoogleNews-vectors-negative300 - https://code.google.com/archive/p/word2vec/\n* glove.840B.300d - https://nlp.stanford.edu/projects/glove/\n* paragram_300_sl999 - https://cogcomp.org/page/resource_view/106\n* wiki-news-300d-1M - https://fasttext.cc/docs/en/english-vectors.html","metadata":{"_uuid":"e8e3bc86f52a01c4125201a736ce77343a48c11e"}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train_df.shape)\nprint(\"Test shape : \", test_df.shape)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-06-04T17:59:19.786187Z","iopub.execute_input":"2021-06-04T17:59:19.786728Z","iopub.status.idle":"2021-06-04T17:59:25.873119Z","shell.execute_reply.started":"2021-06-04T17:59:19.786681Z","shell.execute_reply":"2021-06-04T17:59:25.871984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"scrolled":true,"_uuid":"6473283634cdd157b12575c0b828cdff9721eadc","execution":{"iopub.status.busy":"2021-06-04T17:59:26.298432Z","iopub.execute_input":"2021-06-04T17:59:26.298713Z","iopub.status.idle":"2021-06-04T17:59:26.324056Z","shell.execute_reply.started":"2021-06-04T17:59:26.298672Z","shell.execute_reply":"2021-06-04T17:59:26.323144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Target Distribution:**\n\nFirst let us look at the distribution of the target variable to understand more about the imbalance and so on.","metadata":{"_uuid":"a5fb3e1dd7990f063b5cbb199c51a6f6fc3a9d91"}},{"cell_type":"code","source":"## target count ##\ncnt_srs = train_df['target'].value_counts()\ntrace = go.Bar(\n    x=cnt_srs.index,\n    y=cnt_srs.values,\n    marker=dict(\n        color=cnt_srs.values,\n        colorscale = 'Picnic',\n        reversescale = True\n    ),\n)\n\nlayout = go.Layout(\n    title='Target Count',\n    font=dict(size=18)\n)\n\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"TargetCount\")\n\n## target distribution ##\nlabels = (np.array(cnt_srs.index))\nsizes = (np.array((cnt_srs / cnt_srs.sum())*100))\n\ntrace = go.Pie(labels=labels, values=sizes)\nlayout = go.Layout(\n    title='Target distribution',\n    font=dict(size=18),\n    width=600,\n    height=600,\n)\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"usertype\")","metadata":{"_kg_hide-input":true,"_uuid":"dea34a7f45dc6656c34472f0b7a941070bc9bee6","execution":{"iopub.status.busy":"2021-06-04T18:00:09.317448Z","iopub.execute_input":"2021-06-04T18:00:09.317766Z","iopub.status.idle":"2021-06-04T18:00:11.013023Z","shell.execute_reply.started":"2021-06-04T18:00:09.317715Z","shell.execute_reply":"2021-06-04T18:00:11.011959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So about 6% of the training data are insincere questions (target=1) and rest of them are sincere. \n\n**Word Cloud:**\n\nNow let us look at the frequently occuring words in the data by creating a word cloud on the 'question_text' column.","metadata":{"_uuid":"595a4e32ed75442c2500b9b12b178e40b0a7e155"}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\n# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n                   title = None, title_size=40, image_color=False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n    stopwords = stopwords.union(more_stopwords)\n\n    wordcloud = WordCloud(background_color='black',\n                    stopwords = stopwords,\n                    max_words = max_words,\n                    max_font_size = max_font_size, \n                    random_state = 42,\n                    width=800, \n                    height=400,\n                    mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize=figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask);\n        plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n        plt.title(title, fontdict={'size': title_size,  \n                                  'verticalalignment': 'bottom'})\n    else:\n        plt.imshow(wordcloud);\n        plt.title(title, fontdict={'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.tight_layout()  \n    \nplot_wordcloud(train_df[\"question_text\"], title=\"Word Cloud of Questions\")","metadata":{"_kg_hide-input":true,"_uuid":"0b0fcfc1ab5b12bb9efcff2882a799bdee8d3959","execution":{"iopub.status.busy":"2021-06-04T18:00:21.964235Z","iopub.execute_input":"2021-06-04T18:00:21.964537Z","iopub.status.idle":"2021-06-04T18:00:23.472083Z","shell.execute_reply.started":"2021-06-04T18:00:21.964486Z","shell.execute_reply":"2021-06-04T18:00:23.471399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seem to be a variety of words in there. May be it is a good idea to look at the most frequent words in each of the classes separately.\n\n**Word Frequency plot of sincere & insincere questions:****","metadata":{"_uuid":"1fe54117c937a36f48c76fcb53f424d60f1db1a4"}},{"cell_type":"code","source":"from collections import defaultdict\ntrain1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]\n\n## custom function for ngram generation ##\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n## custom function for horizontal bar chart ##\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n## Get the bar chart from sincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n## Get the bar chart from insincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                          subplot_titles=[\"Frequent words of sincere questions\", \n                                          \"Frequent words of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Word Count Plots\")\npy.iplot(fig, filename='word-plots')\n\n#plt.figure(figsize=(10,16))\n#sns.barplot(x=\"ngram_count\", y=\"ngram\", data=fd_sorted.loc[:50,:], color=\"b\")\n#plt.title(\"Frequent words for Insincere Questions\", fontsize=16)\n#plt.show()\n","metadata":{"_uuid":"04c98e7f39d052efd187330436a0797c6745a8d5","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-04T18:00:40.719456Z","iopub.execute_input":"2021-06-04T18:00:40.720208Z","iopub.status.idle":"2021-06-04T18:00:53.789386Z","shell.execute_reply.started":"2021-06-04T18:00:40.72015Z","shell.execute_reply":"2021-06-04T18:00:53.788471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n* Some of the top words are common across both the classes like 'people', 'will', 'think' etc\n* The other top words in sincere questions after excluding the common ones at the very top are 'best', 'good' etc\n* The other top words in insincere questions after excluding the common ones are 'trump', 'women', 'white' etc\n\nNow let us also create bigram frequency plots for both the classes separately to get more idea.","metadata":{"_uuid":"ae522b4ba487398d13b482718d4578b7176f5f65"}},{"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'orange')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'orange')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,horizontal_spacing=0.15,\n                          subplot_titles=[\"Frequent bigrams of sincere questions\", \n                                          \"Frequent bigrams of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Bigram Count Plots\")\npy.iplot(fig, filename='word-plots')\n","metadata":{"_uuid":"0e9e5b7478800de62faffaeb4415136142a6cdf5","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-04T18:01:00.144696Z","iopub.execute_input":"2021-06-04T18:01:00.145057Z","iopub.status.idle":"2021-06-04T18:01:19.834679Z","shell.execute_reply.started":"2021-06-04T18:01:00.144991Z","shell.execute_reply":"2021-06-04T18:01:19.833735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n* The plot says it all. Please look at the plots and do the inference by yourselves ;)\n\nNow let usl look at the trigram plots as well.","metadata":{"_uuid":"3d5c9c88e687cfde98a39f03691c4d2654d8e05c"}},{"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'green')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'green')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04, horizontal_spacing=0.2,\n                          subplot_titles=[\"Frequent trigrams of sincere questions\", \n                                          \"Frequent trigrams of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=1200, paper_bgcolor='rgb(233,233,233)', title=\"Trigram Count Plots\")\npy.iplot(fig, filename='word-plots')","metadata":{"_kg_hide-input":true,"_uuid":"f076a0aebb6b0e4e9966bc1e65fe40d7a3437c9b","execution":{"iopub.status.busy":"2021-06-04T18:01:19.837081Z","iopub.execute_input":"2021-06-04T18:01:19.837445Z","iopub.status.idle":"2021-06-04T18:01:38.733179Z","shell.execute_reply.started":"2021-06-04T18:01:19.837367Z","shell.execute_reply":"2021-06-04T18:01:38.732081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Meta Features:**\n\nNow let us create some meta features and then look at how they are distributed between the classes. The ones that we will create are\n1. Number of words in the text\n2. Number of unique words in the text\n3. Number of characters in the text\n4. Number of stopwords\n5. Number of punctuations\n6. Number of upper case words\n7. Number of title case words\n8. Average length of the words","metadata":{"_uuid":"510eb370265db86a06cb0dd78c3a7a3a57f1d151"}},{"cell_type":"code","source":"## Number of words in the text ##\ntrain_df[\"num_words\"] = train_df[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest_df[\"num_words\"] = test_df[\"question_text\"].apply(lambda x: len(str(x).split()))\n\n## Number of unique words in the text ##\ntrain_df[\"num_unique_words\"] = train_df[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest_df[\"num_unique_words\"] = test_df[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\n## Number of characters in the text ##\ntrain_df[\"num_chars\"] = train_df[\"question_text\"].apply(lambda x: len(str(x)))\ntest_df[\"num_chars\"] = test_df[\"question_text\"].apply(lambda x: len(str(x)))\n\n## Number of stopwords in the text ##\ntrain_df[\"num_stopwords\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\ntest_df[\"num_stopwords\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\n\n## Number of punctuations in the text ##\ntrain_df[\"num_punctuations\"] =train_df['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\ntest_df[\"num_punctuations\"] =test_df['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n\n## Number of title case words in the text ##\ntrain_df[\"num_words_upper\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\ntest_df[\"num_words_upper\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n## Number of title case words in the text ##\ntrain_df[\"num_words_title\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\ntest_df[\"num_words_title\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n## Average length of the words in the text ##\ntrain_df[\"mean_word_len\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\ntest_df[\"mean_word_len\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))","metadata":{"_uuid":"add38afa4885fecfa4eb12057fb1da9897db20c2","execution":{"iopub.status.busy":"2021-06-04T18:01:38.734647Z","iopub.execute_input":"2021-06-04T18:01:38.735033Z","iopub.status.idle":"2021-06-04T18:02:52.570615Z","shell.execute_reply.started":"2021-06-04T18:01:38.734964Z","shell.execute_reply":"2021-06-04T18:02:52.569583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let us see how these meta features are distributed between both sincere and insincere questions.","metadata":{"_uuid":"bb78505d4b82074447f41ff345959584595de623"}},{"cell_type":"code","source":"## Truncate some extreme values for better visuals ##\ntrain_df['num_words'].loc[train_df['num_words']>60] = 60 #truncation for better visuals\ntrain_df['num_punctuations'].loc[train_df['num_punctuations']>10] = 10 #truncation for better visuals\ntrain_df['num_chars'].loc[train_df['num_chars']>350] = 350 #truncation for better visuals\n\nf, axes = plt.subplots(3, 1, figsize=(10,20))\nsns.boxplot(x='target', y='num_words', data=train_df, ax=axes[0])\naxes[0].set_xlabel('Target', fontsize=12)\naxes[0].set_title(\"Number of words in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='num_chars', data=train_df, ax=axes[1])\naxes[1].set_xlabel('Target', fontsize=12)\naxes[1].set_title(\"Number of characters in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='num_punctuations', data=train_df, ax=axes[2])\naxes[2].set_xlabel('Target', fontsize=12)\n#plt.ylabel('Number of punctuations in text', fontsize=12)\naxes[2].set_title(\"Number of punctuations in each class\", fontsize=15)\nplt.show()\n","metadata":{"_kg_hide-input":true,"_uuid":"539ca7bb74261742b87ce5fa1bfe6071a1f861f8","execution":{"iopub.status.busy":"2021-06-04T18:02:52.573069Z","iopub.execute_input":"2021-06-04T18:02:52.573427Z","iopub.status.idle":"2021-06-04T18:02:54.749032Z","shell.execute_reply.started":"2021-06-04T18:02:52.573359Z","shell.execute_reply":"2021-06-04T18:02:54.748218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference:**\n* We can see that the insincere questions have more number of words as well as characters compared to sincere questions. So this might be a useful feature in our model.\n\n**Baseline Model:**\n\nTo start with, let us just build a baseline model (Logistic Regression) with TFIDF vectors.","metadata":{"_uuid":"2efe9f3d1deff6bfa12ef60a731c2b707be1d377"}},{"cell_type":"code","source":"# Get the tfidf vectors #\ntfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))\ntfidf_vec.fit_transform(train_df['question_text'].values.tolist() + test_df['question_text'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['question_text'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['question_text'].values.tolist())","metadata":{"_uuid":"ae694b35b307f1dd57761384742891da7096caf5","execution":{"iopub.status.busy":"2021-06-04T18:02:54.750472Z","iopub.execute_input":"2021-06-04T18:02:54.750783Z","iopub.status.idle":"2021-06-04T18:07:01.620765Z","shell.execute_reply.started":"2021-06-04T18:02:54.75073Z","shell.execute_reply":"2021-06-04T18:07:01.6196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us build the model now.","metadata":{"_uuid":"691b434b68a36e6689f874b43c8d072e4d4b49ec"}},{"cell_type":"code","source":"train_y = train_df[\"target\"].values\n\ndef runModel(train_X, train_y, test_X, test_y, test_X2):\n    model = linear_model.LogisticRegression(C=5., solver='sag')\n    model.fit(train_X, train_y)\n    pred_test_y = model.predict_proba(test_X)[:,1]\n    pred_test_y2 = model.predict_proba(test_X2)[:,1]\n    return pred_test_y, pred_test_y2, model\n\nprint(\"Building model.\")\ncv_scores = []\npred_full_test = 0\npred_train = np.zeros([train_df.shape[0]])\nkf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)\nfor dev_index, val_index in kf.split(train_df):\n    dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    pred_val_y, pred_test_y, model = runModel(dev_X, dev_y, val_X, val_y, test_tfidf)\n    pred_full_test = pred_full_test + pred_test_y\n    pred_train[val_index] = pred_val_y\n    cv_scores.append(metrics.log_loss(val_y, pred_val_y))\n    break","metadata":{"_uuid":"865965945fbf113a3dae9702b3674e3fd3501a17","execution":{"iopub.status.busy":"2021-06-04T18:07:16.119757Z","iopub.execute_input":"2021-06-04T18:07:16.120092Z","iopub.status.idle":"2021-06-04T18:08:24.194241Z","shell.execute_reply.started":"2021-06-04T18:07:16.12001Z","shell.execute_reply":"2021-06-04T18:08:24.193083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Getting the best threshold based on validation sample.","metadata":{"_uuid":"0ce56f0d6e4d18705dcb9e90706459305b2edd18"}},{"cell_type":"code","source":"for thresh in np.arange(0.1, 0.201, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))\n","metadata":{"_uuid":"1ae3f1341ed8e0652836945a6838a1385cb9a037","execution":{"iopub.status.busy":"2021-06-04T18:08:24.195657Z","iopub.execute_input":"2021-06-04T18:08:24.196051Z","iopub.status.idle":"2021-06-04T18:08:24.680474Z","shell.execute_reply.started":"2021-06-04T18:08:24.195981Z","shell.execute_reply":"2021-06-04T18:08:24.679455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we are getting a better F1 score for this model at 0.17.! \n\nNow let us look at the important words used for classifying the insincere questions. We will use eli5 library for the same. Thanks to [this excellent kernel](https://www.kaggle.com/lopuhin/eli5-for-mercari) by @lopuhin","metadata":{"_uuid":"c4fe6d873b7ea9a2aeb773f93e044c0bc36bf352"}},{"cell_type":"code","source":"import eli5\neli5.show_weights(model, vec=tfidf_vec, top=100, feature_filter=lambda x: x != '<BIAS>')","metadata":{"_uuid":"029dd8d87acd64c03827f28594cedd3b45309078","execution":{"iopub.status.busy":"2021-06-04T18:08:24.681909Z","iopub.execute_input":"2021-06-04T18:08:24.682244Z","iopub.status.idle":"2021-06-04T18:09:23.055253Z","shell.execute_reply.started":"2021-06-04T18:08:24.682192Z","shell.execute_reply":"2021-06-04T18:09:23.054257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:01:35.481028Z","iopub.execute_input":"2021-06-05T05:01:35.481511Z","iopub.status.idle":"2021-06-05T05:01:35.498468Z","shell.execute_reply.started":"2021-06-05T05:01:35.481467Z","shell.execute_reply":"2021-06-05T05:01:35.497747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nsub = pd.read_csv('../input/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:01:54.482184Z","iopub.execute_input":"2021-06-05T05:01:54.482648Z","iopub.status.idle":"2021-06-05T05:02:00.418042Z","shell.execute_reply.started":"2021-06-05T05:01:54.482606Z","shell.execute_reply":"2021-06-05T05:02:00.417358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:02:00.419460Z","iopub.execute_input":"2021-06-05T05:02:00.419769Z","iopub.status.idle":"2021-06-05T05:02:00.453645Z","shell.execute_reply.started":"2021-06-05T05:02:00.419708Z","shell.execute_reply":"2021-06-05T05:02:00.452838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n# Clean the text\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x.lower()))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x.lower()))\n\n# Clean numbers\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))\n\n# Clean speelings\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:02:00.454979Z","iopub.execute_input":"2021-06-05T05:02:00.455260Z","iopub.status.idle":"2021-06-05T05:03:17.747791Z","shell.execute_reply.started":"2021-06-05T05:02:00.455198Z","shell.execute_reply":"2021-06-05T05:03:17.746792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:03:17.749010Z","iopub.execute_input":"2021-06-05T05:03:17.749296Z","iopub.status.idle":"2021-06-05T05:03:19.782949Z","shell.execute_reply.started":"2021-06-05T05:03:17.749242Z","shell.execute_reply":"2021-06-05T05:03:19.782259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:03:19.784366Z","iopub.execute_input":"2021-06-05T05:03:19.784613Z","iopub.status.idle":"2021-06-05T05:03:21.928962Z","shell.execute_reply.started":"2021-06-05T05:03:19.784564Z","shell.execute_reply":"2021-06-05T05:03:21.927999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:03:33.843124Z","iopub.execute_input":"2021-06-05T05:03:33.843432Z","iopub.status.idle":"2021-06-05T05:03:34.613721Z","shell.execute_reply.started":"2021-06-05T05:03:33.843359Z","shell.execute_reply":"2021-06-05T05:03:34.612669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.multiclass import OneVsRestClassifier\nimport time\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nimport torch.utils.data\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", message=\"F-score is ill-defined and being set to 0.0 due to no predicted samples.\")\nimport re\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:03:41.335593Z","iopub.execute_input":"2021-06-05T05:03:41.335867Z","iopub.status.idle":"2021-06-05T05:03:41.347575Z","shell.execute_reply.started":"2021-06-05T05:03:41.335819Z","shell.execute_reply":"2021-06-05T05:03:41.346545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_features = 120000\ntk = Tokenizer(lower = True, filters='', num_words=max_features)\nfull_text = list(train['question_text'].values) + list(test['question_text'].values)\ntk.fit_on_texts(full_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:03:44.661268Z","iopub.execute_input":"2021-06-05T05:03:44.661592Z","iopub.status.idle":"2021-06-05T05:04:11.029600Z","shell.execute_reply.started":"2021-06-05T05:03:44.661533Z","shell.execute_reply":"2021-06-05T05:04:11.028938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('missing'))\ntest_tokenized = tk.texts_to_sequences(test['question_text'].fillna('missing'))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:04:11.030444Z","iopub.execute_input":"2021-06-05T05:04:11.030765Z","iopub.status.idle":"2021-06-05T05:04:33.366019Z","shell.execute_reply.started":"2021-06-05T05:04:11.030729Z","shell.execute_reply":"2021-06-05T05:04:33.365172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['question_text'].apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log');\nplt.title('Distribution of question text length in characters');","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:04:33.367067Z","iopub.execute_input":"2021-06-05T05:04:33.367274Z","iopub.status.idle":"2021-06-05T05:04:36.390711Z","shell.execute_reply.started":"2021-06-05T05:04:33.367241Z","shell.execute_reply":"2021-06-05T05:04:36.389701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 74\nmaxlen = 74\nX_train = pad_sequences(train_tokenized, maxlen = max_len)\nX_test = pad_sequences(test_tokenized, maxlen = max_len)","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:04:36.392060Z","iopub.execute_input":"2021-06-05T05:04:36.392379Z","iopub.status.idle":"2021-06-05T05:04:44.325489Z","shell.execute_reply.started":"2021-06-05T05:04:36.392295Z","shell.execute_reply":"2021-06-05T05:04:44.324463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train['target'].values","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:05:27.087850Z","iopub.execute_input":"2021-06-05T05:05:27.088340Z","iopub.status.idle":"2021-06-05T05:05:27.092109Z","shell.execute_reply.started":"2021-06-05T05:05:27.088295Z","shell.execute_reply":"2021-06-05T05:05:27.091503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","metadata":{"execution":{"iopub.status.busy":"2021-06-04T18:22:16.457456Z","iopub.execute_input":"2021-06-04T18:22:16.457739Z","iopub.status.idle":"2021-06-04T18:22:16.468636Z","shell.execute_reply.started":"2021-06-04T18:22:16.457691Z","shell.execute_reply":"2021-06-04T18:22:16.467831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10).split(X_train, y_train))","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:05:29.883290Z","iopub.execute_input":"2021-06-05T05:05:29.883744Z","iopub.status.idle":"2021-06-05T05:05:30.222895Z","shell.execute_reply.started":"2021-06-05T05:05:29.883696Z","shell.execute_reply":"2021-06-05T05:05:30.221951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import zipfile\n\n# # Will unzip the files so that you can see them..\n# with zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\",\"r\") as z:\n#     z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2021-06-04T18:25:48.771516Z","iopub.execute_input":"2021-06-04T18:25:48.772387Z","iopub.status.idle":"2021-06-04T18:25:48.876423Z","shell.execute_reply.started":"2021-06-04T18:25:48.771899Z","shell.execute_reply":"2021-06-04T18:25:48.874398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# !unzip ../input/quora-insincere-questions-classification/embeddings.zip","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:06:34.493938Z","iopub.execute_input":"2021-06-05T05:06:34.494230Z","iopub.status.idle":"2021-06-05T05:06:35.258716Z","shell.execute_reply.started":"2021-06-05T05:06:34.494187Z","shell.execute_reply":"2021-06-05T05:06:35.257627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# embed_size = 300\n# embedding_path = \"../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt\"\n# def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n# embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))\n# # all_embs = np.stack(embedding_index.values())\n# # emb_mean,emb_std = all_embs.mean(), all_embs.std()\n# emb_mean,emb_std = -0.005838499, 0.48782197\n# word_index = tk.word_index\n# nb_words = min(max_features, len(word_index))\n# embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\n# for word, i in word_index.items():\n#     if i >= max_features: continue\n#     embedding_vector = embedding_index.get(word)\n#     if embedding_vector is not None: embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-06-05T05:06:28.358761Z","iopub.execute_input":"2021-06-05T05:06:28.359074Z","iopub.status.idle":"2021-06-05T05:06:28.381338Z","shell.execute_reply.started":"2021-06-05T05:06:28.359015Z","shell.execute_reply":"2021-06-05T05:06:28.380260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# embedding_path = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"\n# def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n# embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n# # all_embs = np.stack(embedding_index.values())\n# # emb_mean,emb_std = all_embs.mean(), all_embs.std()\n# emb_mean,emb_std = -0.0053247833, 0.49346462\n# embedding_matrix1 = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\n# for word, i in word_index.items():\n#     if i >= max_features: continue\n#     embedding_vector = embedding_index.get(word)\n#     if embedding_vector is not None: embedding_matrix1[i] = embedding_vecto","metadata":{"execution":{"iopub.status.busy":"2021-06-04T18:12:40.641253Z","iopub.execute_input":"2021-06-04T18:12:40.641549Z","iopub.status.idle":"2021-06-04T18:12:40.675742Z","shell.execute_reply.started":"2021-06-04T18:12:40.641503Z","shell.execute_reply":"2021-06-04T18:12:40.674381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)\n# del embedding_matrix1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**References:**\n\nThanks to all the below kernels which I used for reference.\n\n1. https://www.kaggle.com/aashita/word-clouds-of-various-shapes\n2. https://www.kaggle.com/tunguz/just-some-simple-eda\n3. https://www.kaggle.com/lopuhin/eli5-for-mercari","metadata":{"_uuid":"59e87ce9629062f8375a6a3e251767e7f3b40172"}},{"cell_type":"markdown","source":"**More to come. Stay tuned.!**","metadata":{"_uuid":"4d29bf884b3fc298e901f2d3fdeae0d76c8455b7"}},{"cell_type":"code","source":"","metadata":{"_uuid":"0e59af0195cf991bf30af4dfbc1d4ec1a6c760ff","trusted":true},"execution_count":null,"outputs":[]}]}