{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\nimport operator \nfrom sklearn import metrics\n\nimport re\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nfrom sklearn.preprocessing import LabelEncoder\nfrom gensim.models import KeyedVectors\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/glove.840B.300d\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('target').size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> - About 94% of the questions are labeled as sincere and only about 6% of the questions are labeled as insincere. \n- Imbalanced class"},{"metadata":{},"cell_type":"markdown","source":"Let's make word cloud for each class separately to see what words are frequently used:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\n# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=300, max_font_size=100, figure_size=(24.0,16.0), \n                   title = None, title_size=40, image_color=False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n    stopwords = stopwords.union(more_stopwords)\n\n    wordcloud = WordCloud(background_color='brown',\n                    stopwords = stopwords,\n                    max_words = max_words,\n                    max_font_size = max_font_size, \n                    random_state = 0,\n                    width=800, \n                    height=400,\n                    mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize=figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask);\n        plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n        plt.title(title, fontdict={'size': title_size,  \n                                  'verticalalignment': 'bottom'})\n    else:\n        plt.imshow(wordcloud);\n        plt.title(title, fontdict={'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.tight_layout()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(train1_df[\"question_text\"], title=\"Word Cloud of Insincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(train0_df[\"question_text\"], title=\"Word Cloud of Sincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Words like American, Trump, people, woman, Europeans, choice, race, Iran, Muslim, school are frequently used in insincere questions.\n- Words like know, good, career, Amazon, velocity, quebec, nationalist, think, time, adopted are frequently used in sincere questions. \n- These words show the difference in topics in two groups.\n- It might be a better idea to look at bigram and trigram.\n"},{"metadata":{},"cell_type":"markdown","source":"- Looking at the sentiment of each question might give some clue as well.. It is not necessary that all insincere questions have negative sentiment to it.. So let's see how that looks like:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def word_freq(sentences, verbose =  True):\n    \"\"\"\n    :param sentences: list of list of words\n    :return: dictionary of words and their count\n    \"\"\"\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train_df[\"question_text\"].apply(lambda x: x.split())\nvocab = word_freq(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedd_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(embedd_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_embeddings_coverage(vocab,embeddings_index):\n    vocab_embed = {}\n    out_of_vocab= {}\n    k = 0\n    i = 0\n    for word in vocab:\n        try:\n            vocab_embed[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n            out_of_vocab[word] = vocab[word]\n            i += vocab[word]\n            pass\n    print('found word embeddings for {:.2%} of vocab'.format(len(vocab_embed) / len(vocab)))\n    print('found word embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_oov = sorted(out_of_vocab.items(), key=operator.itemgetter(1))[::-1]\n\n    return vocab_embed, sorted_oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_embed, oov = get_embeddings_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_punctuation(sentence):\n    \"\"\"\n    Utility function to remove punctuations from sentence text using simple regex statements..\n    \"\"\"\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        sentence = sentence.replace(punct, '')\n        \n    return sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: remove_punctuation(x))\nsentences = train_df[\"question_text\"].apply(lambda x: x.split())\nvocab = word_freq(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_embed, oov = get_embeddings_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_numbers(x):\n    \"\"\"\n    Utility function to format the numbers in the sentences\n    \"\"\"\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_numbers(x))\nsentences = train_df[\"question_text\"].apply(lambda x: x.split())\nvocab = word_freq(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_embed, oov = get_embeddings_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#thanks to https://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\nsentences = train_df[\"question_text\"].apply(lambda x: x.split())\nto_remove = ['a','to','of','and']\nsentences = [[word for word in sentence if not word in to_remove] for sentence in sentences]\nvocab = word_freq(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_embed, oov = get_embeddings_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Great, we have word embeddigs for 98.5% of the text. let's do sentiment analysis now."},{"metadata":{"trusted":true},"cell_type":"code","source":"# some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## split to train and val\nfrom sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=32)\ntrain_x = train_df['question_text']\nval_x = val_df['question_text']\ntest_x = test_df['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Tokenize the sentences\nfrom keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_x))\ntrain_x = tokenizer.texts_to_sequences(train_x)\nval_x = tokenizer.texts_to_sequences(val_x)\ntest_x = tokenizer.texts_to_sequences(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Pad the sentences \nfrom keras.preprocessing.sequence import pad_sequences\n\ntrain_x = pad_sequences(train_x, maxlen=maxlen)\nval_x = pad_sequences(val_x, maxlen=maxlen)\ntest_x = pad_sequences(test_x, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    if word in embeddings_index:\n        embedding_vector = embeddings_index.get_vector(word)\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv1D, GlobalAveragePooling1D, MaxPooling1D\nfrom keras.models import Model\nfrom keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(max_features, embed_size, input_length=maxlen)) # weights=[embedding_matrix]))\nmodel.add(Conv1D(64, 5, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(MaxPooling1D(pool_size=4))\nmodel.add(LSTM(50))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', \n                    optimizer='adam', \n                    metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_x, train_y, batch_size=512, epochs=2, validation_data=(val_x, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(val_x, val_y, batch_size=128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = model.predict([val_x], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = model.predict([val_x], batch_size=512, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(max_features, embed_size, input_length=maxlen, weights=[embedding_matrix]))\nmodel.add(Conv1D(64, 5, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(MaxPooling1D(pool_size=4))\nmodel.add(LSTM(50))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', \n                    optimizer='adam', \n                    metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_x, train_y, batch_size=512, epochs=1, validation_data=(val_x, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = model.predict([val_x], batch_size=512, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_y>thresh).astype(int))))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}