{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Installing TF-gpu 2.0 Alpha"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install tensorflow-gpu==2.0.0-alpha0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nimport operator\nfrom collections import Counter\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\nfrom tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.compat.v1.keras.layers import CuDNNLSTM\nfrom tensorflow.keras import layers\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_colwidth', 1000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Environment checks"},{"metadata":{},"cell_type":"markdown","source":"#### Check Tensorflow Version"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Check GPU detection for tensorflow"},{"metadata":{"trusted":true},"cell_type":"code","source":"x = tf.random.uniform([3, 3])\n\nprint(\"Is there a GPU available: \"),\nprint(tf.test.is_gpu_available())\n\nprint(\"Is the Tensor on GPU #0:  \"),\nprint(x.device.endswith('GPU:0'))\n\nprint(\"Device name: {}\".format((x.device)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.executing_eagerly())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df  = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv('../input/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df.target==1].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of questions: \", df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA on the Question Text corpus"},{"metadata":{},"cell_type":"markdown","source":"## Target variable response rate\n- __6.18 %__ questions are insincere in the provided dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Percentage of insincere questions: {}\".format(sum(df.target == 1)*100/len(df.target)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Wordclouds for the question text corpus\n- This will show us the most frequently used words for sincere and insincere labeled questions"},{"metadata":{},"cell_type":"markdown","source":"### Defining function for generating wordcloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n                   title = None, title_size=40, image_color=False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n    stopwords = stopwords.union(more_stopwords)\n\n    wordcloud = WordCloud(background_color='black',\n                    stopwords = stopwords,\n                    max_words = max_words,\n                    max_font_size = max_font_size, \n                    random_state = 42,\n                    width=800, \n                    height=400,\n                    mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize=figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask);\n        plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n        plt.title(title, fontdict={'size': title_size,  \n                                  'verticalalignment': 'bottom'})\n    else:\n        plt.imshow(wordcloud);\n        plt.title(title, fontdict={'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.tight_layout()  \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Wordcloud for sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(df[df.target == 0][\"question_text\"], title=\"Word Cloud of Sincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Wordcloud for insincere questions\n- We observe that insincere questions contain highly debated and emotion based topics of Trump, Muslims, America, Russia, Obama, Liberal etc."},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(df[df.target == 1][\"question_text\"], title=\"Word Cloud of Insincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Word frequency and n-gram frequency\n- Frequency of words can show the most used words for sincere and insincere questions\n- N-grams can be useful in distinguishing sincere and insincere questions by identifying patterns in topics"},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_words = df[df.target==0].question_text.apply(lambda x: x.lower().split()).tolist()\ninsincere_words = df[df.target==1].question_text.apply(lambda x: x.lower().split()).tolist()\n\nsincere_words = [item for sublist in sincere_words for item in sublist if item not in stopwords]\ninsincere_words = [item for sublist in insincere_words  for item in sublist if item not in stopwords ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of sincere words',len(sincere_words))\nprint('Number of insincere words',len(insincere_words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_words_counter = Counter(sincere_words)\ninsincere_words_counter = Counter(insincere_words)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common sincere words"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"most_common_sincere_words = sincere_words_counter.most_common()[:10]\nmost_common_sincere_words = pd.DataFrame(most_common_sincere_words)\nmost_common_sincere_words.columns = ['word', 'freq']\nmost_common_sincere_words['percentage'] = most_common_sincere_words.freq *100 / sum(most_common_sincere_words.freq)\nmost_common_sincere_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common insincere words"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_insincere_words = insincere_words_counter.most_common()[:10]\nmost_common_insincere_words = pd.DataFrame(most_common_insincere_words)\nmost_common_insincere_words.columns = ['word', 'freq']\nmost_common_insincere_words['percentage'] = most_common_insincere_words.freq *100 / sum(most_common_insincere_words.freq)\nmost_common_insincere_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Function for generating n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_ngrams(words, n):\n    \n    # Use the zip function to help us generate n-grams\n    # Concatentate the tokens into ngrams and return\n    ngrams = zip(*[words[i:] for i in range(n)])\n    return [\" \".join(ngram) for ngram in ngrams]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_ngram_counter = Counter(generate_ngrams(sincere_words, n))\ninsincere_ngram_counter = Counter(generate_ngrams(insincere_words, n))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common sincere n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_sincere_ngram = sincere_ngram_counter.most_common()[:10]\nmost_common_sincere_ngram = pd.DataFrame(most_common_sincere_ngram)\nmost_common_sincere_ngram.columns = ['word', 'freq']\nmost_common_sincere_ngram['percentage'] = most_common_sincere_ngram.freq *100 / sum(most_common_sincere_ngram.freq)\nmost_common_sincere_ngram","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common insincere n-grams\n- We observe a pattern of suggestive/controversial n-grams amongst the insincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_insincere_ngram = insincere_ngram_counter.most_common()[:10]\nmost_common_insincere_ngram = pd.DataFrame(most_common_insincere_ngram)\nmost_common_insincere_ngram.columns = ['word', 'freq']\nmost_common_insincere_ngram['percentage'] = most_common_insincere_ngram.freq *100 / sum(most_common_insincere_ngram.freq)\nmost_common_insincere_ngram","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters for preprocessing and algorithms"},{"metadata":{"trusted":true},"cell_type":"code","source":"# config values\nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation"},{"metadata":{},"cell_type":"markdown","source":"## Training and Test data split"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test  = train_test_split(df, test_size=0.1, random_state=2019)\ny_train, y_test = X_train['target'].values, X_test['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Missing value treatment\n- Fill missing questions with a string placeholder \"\\_NA\\_\""},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train['question_text'].fillna('_NA_').values\nX_test = X_test['question_text'].fillna('_NA_').values\nX_submission = df_test['question_text'].fillna('_NA_').values","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizing the sentences to words"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nX_submission = tokenizer.texts_to_sequences(X_submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Padding sequences for uniform dimensions"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pad_sequences(X_train, maxlen=maxlen)\nX_test = pad_sequences(X_test, maxlen=maxlen)\nX_submission = pad_sequences(X_submission, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data prep function for future use\n- Creating a function including all of the above data prep steps, to use for quick future data processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_prep(df):\n    print(\"Splitting dataframe with shape {} into training and test datasets\".format(df.shape))\n    X_train, X_test  = train_test_split(df, test_size=0.1, random_state=2019)\n    y_train, y_test = X_train['target'].values, X_test['target'].values\n    \n    print(\"Filling missing values\")\n    X_train = X_train['question_text'].fillna('_NA_').values\n    X_test = X_test['question_text'].fillna('_NA_').values\n    X_submission = df_test['question_text'].fillna('_NA_').values\n    \n    print(\"Tokenizing {} questions into words\".format(df.shape[0]))\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(X_train))\n    X_train = tokenizer.texts_to_sequences(X_train)\n    X_test = tokenizer.texts_to_sequences(X_test)\n    X_submission = tokenizer.texts_to_sequences(X_submission)\n    \n    print(\"Padding sequences for uniform dimensions\")\n    X_train = pad_sequences(X_train, maxlen=maxlen)\n    X_test = pad_sequences(X_test, maxlen=maxlen)\n    X_submission = pad_sequences(X_submission, maxlen=maxlen)\n    \n    print(\"Completed data preparation, returning training, test and submission datasets, split as dependent(X) and independent(Y) variables\")\n    \n    return X_train, X_test, y_train, y_test, X_submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define LSTM layers and parameters\n- Bidirectional LSTM using CUDA GPU processing\n- No embeddings used currently\n- Two fully connected layers with dropout layers for reducing chances of overfit"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Embedding(max_features, embed_size, input_length=maxlen))\nmodel1.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))\nmodel1.add(GlobalMaxPool1D())\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(64, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(32, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(1, activation='sigmoid'))\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit LSTM model "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%time model1.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_test, y_test), verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict using trained model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model1.predict([X_test], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate optimal probability threshold for classification\n- Calculating best probability cut-off giving the highest F1 - Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt_prob = None\nf1_max = 0\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(y_test, (pred_test_y > thresh).astype(int))\n    print('F1 score at threshold {} is {}'.format(thresh, f1))\n    \n    if f1 > f1_max:\n        f1_max = f1\n        opt_prob = thresh\n        \nprint('Optimal probabilty threshold is {} for maximum F1 score {}'.format(opt_prob, f1_max))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_submission_y = model1.predict([X_submission], batch_size=1024, verbose=1)\npred_submission_y = (pred_submission_y > opt_prob).astype(int)\n\ndf_submission = pd.DataFrame({'qid': df_test['qid'].values})\ndf_submission['prediction'] = pred_submission_y\n#df_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading embeddings"},{"metadata":{},"cell_type":"markdown","source":"## Function to load embeddings from file"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding=\"utf8\") if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir('../input/'))\n# glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n# paragram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n# wiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Extracting GloVe embedding\")\nembed_glove = load_embed(glove)\n#print(\"Extracting Paragram embedding\")\n#embed_paragram = load_embed(paragram)\n#print(\"Extracting FastText embedding\")\n#embed_fasttext = load_embed(wiki_news)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Vocabulary and calculating coverage"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bulding dataset vocabulary"},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculating coverage for each embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dict(list(embed_glove.items())[20:22])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Text pre-processing to improve coverage of embeddings"},{"metadata":{},"cell_type":"markdown","source":"## Lower casing questions for uniform matching"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['processed_question'] = df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_low = build_vocab(df['processed_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[1:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adding lower case words to embeddings if missing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\nadd_lower(embed_glove, vocab)\n#print(\"Paragram : \")\n#add_lower(embed_paragram, vocab)\n#print(\"FastText : \")\n#add_lower(embed_fasttext, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[1:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing special characters appropriately\n- This ensures better a match to embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"punctuations = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in punctuations:\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"processed_question\"] = df[\"processed_question\"].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_low = build_vocab(df['processed_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['processed_question']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test, X_submission = data_prep(df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilizing embeddings in LSTM classifier\n- Following a similar model network structure as previous for comparable results"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Embedding(max_features, embed_size, input_length=maxlen, weights = [embed_glove]))\nmodel1.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))\nmodel1.add(GlobalMaxPool1D())\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(64, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(32, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(1, activation='sigmoid'))\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit LSTM model "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%time model1.fit(X_train, y_train, batch_size=512, epochs=5, validation_data=(X_test, y_test), verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict using trained model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model1.predict([X_test], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate optimal probability threshold for classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt_prob = None\nf1_max = 0\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(y_test, (pred_test_y > thresh).astype(int))\n    print('F1 score at threshold {} is {}'.format(thresh, f1))\n    \n    if f1 > f1_max:\n        f1_max = f1\n        opt_prob = thresh\n        \nprint('Optimal probabilty threshold is {} for maximum F1 score {}'.format(opt_prob, f1_max))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"pred_submission_y = model1.predict([X_submission], batch_size=1024, verbose=1)\npred_submission_y = (pred_submission_y > opt_prob).astype(int)\n\ndf_submission = pd.DataFrame({'qid': df_test['qid'].values})\ndf_submission['prediction'] = pred_submission_y\ndf_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"}},"nbformat":4,"nbformat_minor":4}