{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!set KERAS_BACKEND=tensorflow","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import string\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected = True)\nimport plotly.graph_objs as go\n\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn import linear_model, model_selection, metrics\nimport eli5\n\n\n\nimport os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reference:\nhttps://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-qiqc"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!ls ../input/quora-insincere-questions-classification/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/quora-insincere-questions-classification/embeddings/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nprint(train_df.shape)\nprint(test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt_srs = train_df.target.value_counts()\ntrace = go.Bar(\n        x = cnt_srs.index,\n        y = cnt_srs.values,\n        marker = dict(\n                    color = cnt_srs.values,\n                    colorscale = 'Picnic',\n                    reversescale = True\n                    ),\n            )\n\nlayout = go.Layout(\n            title ='Target count',\n            font = dict(size = 18)\n            )\n\ndata = [trace]\nfig = go.Figure(data = data, layout = layout)\npy.iplot(fig, filename = \"TargetCount\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Target distribution\nlabels = (np.array(cnt_srs.index))\nsizes = np.array((cnt_srs/cnt_srs.sum())*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trace = go.Pie(labels = labels, values = sizes)\nlayout = go.Layout(\n            title ='Target distribution',\n            font = dict(size=18),\n            height = 600,\n            width = 600\n            )\ndata = [trace]\nfig = go.Figure(data = data, layout = layout)\npy.iplot(fig, filename = 'usertype')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wordcloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_wordcloud(text, mask = None, max_words = 200, max_font_size = 100, figure_size= (24.0, 16.0),\n                  title = None, title_size = 40, image_color = False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n    stopwords = stopwords.union(more_stopwords)\n    \n    wordcloud = WordCloud(background_color = 'black',\n                        stopwords = stopwords,\n                         max_words = max_words,\n                         max_font_size = max_font_size,\n                         random_state = 42,\n                         width = 800,\n                         height = 400,\n                         mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize = figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask)\n        plt.imshow(wordcloud.recolor(color_func = image_colors), interpolation='bilinear');\n        plt.title(title, fontdict = {'size': title_size, \n                                    'verticalalignment':'bottom'})\n    else:\n        plt.imshow(wordcloud)\n        plt.title(title, fontdict= {'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.tight_layout()  \n    \nplot_wordcloud(train_df[\"question_text\"], title=\"Word Cloud of Questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## word cloud for sincere questions\ntrain0_df = train_df[train_df['target']==0]\nplot_wordcloud(train0_df[\"question_text\"], title=\"Word Cloud of Questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## word cloud for nonsincere questions\ntrain1_df = train_df[train_df['target']==1]\nplot_wordcloud(train1_df[\"question_text\"], title=\"Word Cloud of Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Word Frequency plot for sincere and insincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ntrain1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]\n# ngram generation\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n#custom function for horizontal bar chart\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n# Get barchart from sincere questions\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Get barchart from insincere questions\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n## Creating two subplots\n\nfig = tools.make_subplots(rows = 1, cols = 2, vertical_spacing = 0.04,\n                         subplot_titles = ['frequent words in sincere questions',\n                                              'frequent words in insincere questions']\n                         )\nfig.append_trace(trace0,1,1)\nfig.append_trace(trace1, 1,2)\nfig['layout'].update(height =1200, width = 900, paper_bgcolor = 'rgb(233,233,233)', \n                    title ='Word count plots')\npy.iplot(fig, filename = 'word_count_plots')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# frequent bigram plots in both classes\n\nfrom collections import defaultdict\n\n# ngram generation\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n#custom function for horizontal bar chart\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n# Get barchart from sincere questions\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,n_gram = 2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'red')\n\n# Get barchart from insincere questions\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent,n_gram =2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'red')\n\n## Creating two subplots\n\nfig = tools.make_subplots(rows = 1, cols = 2, vertical_spacing = 0.04,\n                         subplot_titles = ['frequent bigrams in sincere questions',\n                                              'frequent bigrams in insincere questions']\n                         )\nfig.append_trace(trace0,1,1)\nfig.append_trace(trace1, 1,2)\nfig['layout'].update(height =1200, width = 900, paper_bgcolor = 'rgb(233,233,233)', \n                    title ='Bigram count plots')\npy.iplot(fig, filename = 'Bigram_count_plots')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\n# ngram generation\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n#custom function for horizontal bar chart\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n# Get barchart from sincere questions\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,n_gram = 3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Get barchart from insincere questions\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent, n_gram = 3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n## Creating two subplots\n\nfig = tools.make_subplots(rows = 1, cols = 2, vertical_spacing = 0.04,\n                         subplot_titles = ['frequent trigrams in sincere questions',\n                                              'frequent trigrams in insincere questions']\n                         )\nfig.append_trace(trace0,1,1)\nfig.append_trace(trace1, 1,2)\nfig['layout'].update(height =1200, width = 900, paper_bgcolor = 'rgb(233,233,233)', \n                    title ='Trigram count plots')\npy.iplot(fig, filename = 'Trigram_count_plots')\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Meta Features:\n\nNow let us create some meta features and then look at how they are distributed between the classes. The ones that we will create are\n\n1. Number of words in the text\n2. Number of unique words in the text\n3. Number of characters in the text\n4. Number of stopwords\n5. Number of punctuations\n6. Number of upper case words\n7. Number of title case words\n8. Average length of the words"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Number of words in text\n\ntrain_df['num_words'] = train_df[\"question_text\"].apply(lambda x : len(str(x).split(\" \")))\ntest_df['num_words'] = test_df['question_text'].apply(lambda x: len(str(x).split(\" \")))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Number of unique words\ntrain_df['num_unique_words'] = train_df[\"question_text\"].apply(lambda x : len(set(str(x).split(\" \"))))\ntest_df['num_unique_words'] = test_df['question_text'].apply(lambda x: len(set(str(x).split(\" \"))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of characters in text\ntrain_df['num_chars'] = train_df['question_text'].apply(lambda x : len(str(x)))\ntest_df['num_chars'] = test_df['question_text'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Number of stopwords in text\ntrain_df['num_stopwords'] = train_df['question_text'].apply(lambda x : len([w for w in str(x).lower().split(\" \") if w in STOPWORDS ]))\ntest_df['num_stopwords'] = test_df['question_text'].apply(lambda x : len([w for w in str(x).lower().split(\" \") if w in STOPWORDS ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Number of punctuations\n\ntrain_df['num_punct'] = train_df['question_text'].apply(lambda x : len([w for w in str(x) if w in string.punctuation ]))\ntest_df['num_punct'] = test_df['question_text'].apply(lambda x : len([w for w in str(x) if w in string.punctuation ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## number of upper case words\n\ntrain_df['num_words_upper'] = train_df['question_text'].apply(lambda x : len([w for w in str(x).split(\" \") if w.isupper() ]))\ntest_df['num_words_upper'] = test_df['question_text'].apply(lambda x : len([w for w in str(x).split(\" \") if w.isupper() ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Number of title case\ntrain_df['num_words_title'] = train_df['question_text'].apply(lambda x : len([w for w in str(x).split(\" \") if w.istitle() ]))\ntest_df['num_words_upper'] = test_df['question_text'].apply(lambda x : len([w for w in str(x).split(\" \") if w.isupper() ]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Average length of words in text\ntrain_df['mean_word_len'] = train_df['question_text'].apply(lambda x : np.mean([len(w) for w in str(x).split(\" \")]))\ntest_df['mean_word_len'] = test_df['question_text'].apply(lambda x : np.mean([len(w) for w in str(x).split(\" \")]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Distribution of meta features in sincere and insincere questions\n\ntrain_df[\"num_words\"].loc[train_df[\"num_words\"]>60] = 60 # truncating for better visuals\ntrain_df['num_punct'].loc[train_df['num_punct']>10] = 10\ntrain_df['num_chars'].loc[train_df['num_chars']>350] = 350","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, axes = plt.subplots(3,1,figsize =(10,20))\n\nsns.boxplot(x = 'target', y ='num_words', data = train_df, ax = axes[0])\naxes[0].set_xlabel('Target', fontsize = 12)\naxes[0].set_title('Words count in each class', fontsize = 15)\n\nsns.boxplot(x ='target', y='num_chars',data = train_df, ax = axes[1])\naxes[1].set_xlabel('Target', fontsize=12)\naxes[1].set_title('Characters count in each class', fontsize=15)\n\nsns.boxplot(x = 'target', y='num_punct', data = train_df, ax = axes[2])\naxes[2].set_xlabel('Target', fontsize = 12)\naxes[2].set_title('Punctuations in each class', fontsize = 15)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Baseline model\n\ntfidf_vec = TfidfVectorizer(stop_words='english', ngram_range = (1,3))\ntfidf_vec.fit_transform(train_df[\"question_text\"].values.tolist() + \n                       test_df[\"question_text\"].values.tolist())\n\ntrain_tfidf = tfidf_vec.transform(train_df[\"question_text\"].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df[\"question_text\"].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y = train_df['target'].values\n\ndef runModel(train_X, train_y, test_X, test_y, test_X2):\n    model = linear_model.LogisticRegression(C=5, solver = 'sag')\n    model.fit(train_X, train_y)\n    pred_test_y = model.predict_proba(test_X)[:,1]\n    pred_test_y2 = model.predict_proba(test_X2)[:,1]\n    return pred_test_y, pred_test_y2, model\n\nprint('Building model')\ncv_scores =[]\npred_full_test = 0\npred_train = np.zeros(train_df.shape[0])\nkf = model_selection.KFold(n_splits = 5, shuffle = True, random_state = 2017)\nfor dev_index, val_index in kf.split(train_df):\n    dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    pred_val_y, pred_test_y, model = runModel(dev_X, dev_y, val_X, val_y, test_tfidf)\n    pred_full_test = pred_full_test + pred_test_y\n    pred_train[val_index] = pred_val_y\n    cv_scores.append(metrics.log_loss(val_y, pred_val_y))\n    break\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Getting best threshold on validation sample\n\nfor thresh in np.arange(0.1, 0.201, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at thresh {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"F1 score is better at 0.17 threshold"},{"metadata":{"trusted":true},"cell_type":"code","source":"##Lets look at important words used in the classification using eli5\n\neli5.show_weights(model, vec = tfidf_vec, feature_filter = lambda x: x!='<BIAS>')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## splitting training and test set\ntrain_df, val_df = model_selection.train_test_split(train_df, test_size = 0.1, random_state = 2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## filling na values\ntrain_X = train_df['question_text'].fillna(\"_na_\").values\nval_X = val_df['question_text'].fillna(\"_na_\").values\ntest_X = test_df['question_text'].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words = max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## pad sequences\ntrain_X = pad_sequences(train_X, maxlen= maxlen)\nval_X = pad_sequences(val_X, maxlen = maxlen)\ntest_X = pad_sequences(test_X, maxlen = maxlen)\n\n## get the target vales\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Without pretrained embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nmodel.fit(train_X,train_y, batch_size = 512, epochs = 2, validation_data =(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size = 1024, verbose = 1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at thresh {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# testset prediction\npred_noemb_test_y = model.predict([test_X], batch_size = 1024, verbose =1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cleaning up memory\n\ndel model, inp, x\nimport gc\ngc.collect()\ntime.sleep(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using pretrained embeddings\n!ls ../input/quora-insincere-questions-classification/embeddings/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Glove Embeddings\nEMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size = 512, epochs = 2, validation_data = (val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size = 1024, verbose = 1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at thresh {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_glove_test_y = model.predict(test_X, batch_size = 1024, verbose = 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del word_index, embedding_matrix, all_embs, embeddings_index, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wiki News FastText Embeddings:\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype ='float32')\n\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean, emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape = (maxlen,))\nx = Embedding(max_features, embed_size, weights = [embedding_matrix])(inp)\nx = Bidirectional(LSTM(64, return_sequences = True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation = 'relu')(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation = 'sigmoid')(x)\nmodel = Model(inputs= inp, outputs = x)\nmodel.compile(loss= 'binary_crossentropy', optimizer = 'adam', metrics =['accuracy'])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size = 512, epochs = 2, validation_data =(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_fasttext_val_y = model.predict([val_X], batch_size = 1024, verbose = 2)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.arange(thresh, 2)\n    print(\"F1 score at thresh {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_fasttext_val_y>thresh).astype(int))))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_fasttext_test_y = model.predict(test_X, batch_size = 1024, verbose =1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Paragram Embeddings:"},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_paragram_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y \nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y\npred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}