Text: \"\n for token, attention in token_attention_dic.items():\n html_text += \"{} \".format(attention2color(attention),\n token)\n #html_text += \"
Classified as: \"\n #html_text += label2id[label] \n #html_text += \"
\"\n \n # Display text enriched with attention scores \n display(HTML(html_text))\n \n # PLOT EMOTION SCORES\n \n _labels = ['sincere', 'insincere']\n plt.figure(figsize=(5,2))\n plt.bar(np.arange(len(_labels)), label_probs.squeeze(), align='center', alpha=0.5, color=['black', 'red', 'green', 'blue', 'cyan', \"purple\"])\n plt.xticks(np.arange(len(_labels)), _labels)\n plt.ylabel('Scores')\n plt.show()\n \ndef under_sample(train_df):\n # UNDER SAMPLE\n insincere = len(train_df[train_df.target == 1])\n insincere_indices = train_df[train_df.target == 1].index\n\n sincere_indices = train_df[train_df.target == 0].index\n random_indices = np.random.choice(sincere_indices, insincere, replace=False)\n\n under_sample_indices = np.concatenate([insincere_indices,random_indices])\n under_sample = train_df.loc[under_sample_indices]\n train_df = under_sample.sample(frac=1)\n train_df.info()\n return train_df\n\ndef get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n\ndef get_embeddings_matrix():\n embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))\n\n word_index = tokenizer.word_index\n nb_words = min(MAX_FEATURES, len(word_index))\n print('nb_words: %d' % nb_words)\n embedding_matrix = np.zeros((nb_words, EMB_SIZE))\n print('Embedding matrix shape: %d/%d' % (nb_words, EMB_SIZE))\n for word, i in word_index.items():\n if i >= nb_words: continue\n embedding_vector = embeddings_index.get(word)\n if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n return embedding_matrix\n\ndef visualize_attention():\n # Make new model for output predictions and attentions\n '''\n model.get_layer('attention_vec').output:\n attention_vec (Attention) [(None, 128), (None, 54)] <- We want (None,54) that is the word att\n '''\n model_att = Model(inputs=model.input, \\\n outputs=[model.output, model.get_layer('attention_vec').output[-1]])\n idx = np.random.randint(low = 0, high=X_te.shape[0]) # Get a random test\n tokenized_sample = np.trim_zeros(X_te[idx]) # Get the tokenized text\n label_probs, attentions = model_att.predict(X_te[idx:idx+1]) # Perform the prediction\n\n # Get decoded text and labels\n id2word = dict(map(reversed, tokenizer.word_index.items()))\n decoded_text = [id2word[word] for word in tokenized_sample] \n \n # Get classification\n label = (label_probs>0.5).astype(int).squeeze() # Only one\n label2id = ['Sincere', 'Insincere']\n\n # Get word attentions using attenion vector\n token_attention_dic = {}\n max_score = 0.0\n min_score = 0.0\n \n attentions_text = attentions[0,-len(tokenized_sample):]\n #plt.bar(np.arange(0,len(attentions.squeeze())), attentions.squeeze())\n #plt.show();\n #print(attentions_text)\n attentions_text = (attentions_text - np.min(attentions_text)) / (np.max(attentions_text) - np.min(attentions_text))\n for token, attention_score in zip(decoded_text, attentions_text):\n #print(token, attention_score)\n token_attention_dic[token] = attention_score\n \n\n # Build HTML String to viualize attentions\n html_text = \"Text: \"\n for token, attention in token_attention_dic.items():\n html_text += \"{} \".format(attention2color(attention),\n token)\n #html_text += \"
Classified as: \"\n #html_text += label2id[label] \n #html_text += \"
\"\n \n # Display text enriched with attention scores \n display(HTML(html_text))\n \n # PLOT EMOTION SCORES\n _labels = ['sincere', 'insincere']\n probs = np.zeros(2)\n probs[1] = label_probs\n probs[0] = 1- label_probs\n plt.figure(figsize=(5,2))\n plt.bar(np.arange(len(_labels)), probs.squeeze(), align='center', alpha=0.5, color=['black', 'red', 'green', 'blue', 'cyan', \"purple\"])\n plt.xticks(np.arange(len(_labels)), _labels)\n plt.ylabel('Scores')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"9d02d361f78069d72a788612e0a24ef3187b3150"},"cell_type":"code","source":"# Util classes\nclass Attention(Layer):\n def __init__(self,\n W_regularizer=None, b_regularizer=None,\n W_constraint=None, b_constraint=None,\n bias=True, return_attention=False,\n **kwargs):\n self.supports_masking = True\n self.return_attention = return_attention\n self.init = initializers.get('glorot_uniform')\n\n self.W_regularizer = regularizers.get(W_regularizer)\n self.b_regularizer = regularizers.get(b_regularizer)\n\n self.W_constraint = constraints.get(W_constraint)\n self.b_constraint = constraints.get(b_constraint)\n\n self.bias = bias\n super(Attention, self).__init__(**kwargs)\n\n\n def build(self, input_shape):\n assert len(input_shape) == 3\n\n self.W = self.add_weight((input_shape[-1],),\n initializer=self.init,\n name='{}_W'.format(self.name),\n regularizer=self.W_regularizer,\n constraint=self.W_constraint)\n if self.bias:\n self.b = self.add_weight((input_shape[1],),\n initializer='zero',\n name='{}_b'.format(self.name),\n regularizer=self.b_regularizer,\n constraint=self.b_constraint)\n else:\n self.b = None\n\n self.built = True\n\n def compute_mask(self, input, input_mask=None):\n # do not pass the mask to the next layers\n return None\n\n def call(self, x, mask=None):\n eij = K.squeeze(K.dot(x, K.expand_dims(self.W)), axis=-1)\n\n if self.bias:\n eij += self.b\n\n eij = K.tanh(eij)\n\n a = K.exp(eij)\n\n # apply mask after the exp. will be re-normalized next\n if mask is not None:\n # Cast the mask to floatX to avoid float64 upcasting in theano\n a *= K.cast(mask, K.floatx())\n\n # in some cases especially in the early stages of training the sum may be almost zero\n # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n weighted_input = x * K.expand_dims(a)\n\n result = K.sum(weighted_input, axis=1)\n\n if self.return_attention:\n return [result, a]\n return result\n\n def compute_output_shape(self, input_shape):\n if self.return_attention:\n return [(input_shape[0], input_shape[-1]),\n (input_shape[0], input_shape[1])]\n else:\n return input_shape[0], input_shape[-1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5d68f3e67f9fedd9ce938c2d3c845090719f3b6"},"cell_type":"markdown","source":"## Hyperparameters"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Hyperparameters\n\nEMB_SIZE = 300\nMAX_FEATURES = 50000 # how many unique words to use (i.e num rows in embedding vector)\nMAX_LEN = 100 # Maximum length for texts\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7a6778b925b0ac2ac106d025f8b48ef4d1fea10"},"cell_type":"markdown","source":"## Load & prepare data"},{"metadata":{"trusted":true,"_uuid":"319b73c03ac0e115db68e062054fab98d556c9c8"},"cell_type":"code","source":"# Load Train/Test\ntrain_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb90aec7fc21edb07a3b75ff9717f1a073177c3a"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.05, random_state=2018)\n\n## fill up the missing values\nX_tra = train_df[\"question_text\"].fillna(\"_na_\").values\nX_val = val_df[\"question_text\"].fillna(\"_na_\").values\nX_te = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=MAX_FEATURES)\ntokenizer.fit_on_texts(list(X_tra))\nX_tra = tokenizer.texts_to_sequences(X_tra)\nX_val = tokenizer.texts_to_sequences(X_val)\nX_te = tokenizer.texts_to_sequences(X_te)\n\n## Pad the sentences \nMAX_LEN = min(MAX_LEN, len(max(X_tra, key=len)))\nX_tra = pad_sequences(X_tra, maxlen=MAX_LEN)\nX_val = pad_sequences(X_val, maxlen=MAX_LEN)\nX_te = pad_sequences(X_te, maxlen=MAX_LEN)\n\n## Get the target values\nY_tra = train_df['target'].values\nY_val = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7791302db9d95cd5414eba2281aa48e91191dcc0"},"cell_type":"code","source":"# Load embeddings\nembedding_matrix = get_embeddings_matrix()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be28e207634b96a17b7767cb25dd6965e60bc475"},"cell_type":"markdown","source":"## Model design "},{"metadata":{"trusted":true,"_uuid":"5dcd170ddb0edd3826de2fdf3633dddabf03b90f"},"cell_type":"code","source":"# Define input tensor\ninp = Input(shape=(X_tra.shape[1],), dtype='int32')\n\n# Word embedding layer\nembedded_inputs = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], \n weights=[embedding_matrix], trainable = False)(inp)\n\n# Apply dropout to prevent overfitting\nembedded_inputs = SpatialDropout1D(0.2)(embedded_inputs)\n\n# Apply Bidirectional GRU over embedded inputs\nrnn_outs = Bidirectional(\\\n CuDNNGRU(64, return_sequences=True))(embedded_inputs)\nrnn_outs = Dropout(0.2)(rnn_outs) # Apply dropout to GRU outputs to prevent overfitting\n\n# Attention Mechanism - Generate attention vectors\nsentence, word_scores = Attention(return_attention=True, name = \"attention_vec\")(rnn_outs)\n\n# Dense layers\nfc = Dense(64, activation='relu')(sentence)\nfc = Dropout(0.5)(fc)\noutput = Dense(1, activation='sigmoid')(fc)\n\n# Finally building model\nmodel = Model(inputs=inp, outputs=output)\nmodel.compile(loss='binary_crossentropy', metrics=[\"accuracy\"], optimizer='adam')\n\n# Print model summary\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5103d3bc9a867caef9c438b6feddc740a76f761"},"cell_type":"code","source":"# Train model\nF1_Score = F1Evaluation(validation_data=(X_val, Y_val), interval=1)\nhist = model.fit(X_tra, Y_tra, validation_data=(X_val, Y_val), \n epochs=3, batch_size=512, callbacks=[F1_Score])\nval_loss = hist.history['val_loss'];val_acc = hist.history['val_acc']\nloss = hist.history['loss'];acc = hist.history['acc']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8bd1acee0b33cdab5ebef9021ca89e29c26d874"},"cell_type":"markdown","source":"## Performance"},{"metadata":{"trusted":true,"_uuid":"24d86649c94740d4483dab1196092634a051ebf2"},"cell_type":"code","source":"make_plot(loss, val_loss, acc, val_acc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"732e05ddba4ab7922a5b0f22f7dd9c514dd90704"},"cell_type":"markdown","source":"## Post processing (find best threshold)"},{"metadata":{"trusted":true,"_uuid":"ac23d9a57a9213da9324421fefd231b55194c253"},"cell_type":"code","source":"val_pred = model.predict([X_val], batch_size=1024, verbose=1)\nf1s = []\nmax_thresh, max_f1 = 0, 0\nfor thresh in np.arange(0.1, 0.9, 0.01):\n f1s.append(metrics.f1_score(Y_val, (val_pred>thresh)))\nmax_f1 = np.max(f1s)\nmax_thresh = np.arange(0.1, 0.9, 0.01)[np.argmax(f1s)]\nprint('Validation set: Max F1-Score: %.2f - reached with threshold: %.2f' % (max_f1, max_thresh))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f9ef64483c7d4ee4a09771a22567ab3046dea90"},"cell_type":"markdown","source":"## Visualize attention\n\nCredit assignment is allocating importance to input features through the weights of the neural network’s model. This is exactly what an attention layer does. \n\nThe attention layer allocates more or less importance to each part of the input, and it learns to do this while training.\n\nIn this visualization, the input sentences are displayed. The background colour varies between white and red. More intense red means more attention given to the word."},{"metadata":{"trusted":true,"_uuid":"66a098dcb944567270ba8548b26266a2d72d61c5"},"cell_type":"code","source":"for _ in range(3):\n visualize_attention()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38019eb6821c50ca1db6e6b04038eda8083528f4"},"cell_type":"markdown","source":"## Predictions over test set"},{"metadata":{"trusted":true,"_uuid":"5687184a4da2d3b313664d10a6dafe3612a9fada"},"cell_type":"code","source":"test_pred = model.predict([X_te], batch_size=1024, verbose=1)\ntest_pred_thresh = (test_pred>max_thresh).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08c0aa245f8f125df7d4f57cc1f9ab7006d0c770"},"cell_type":"markdown","source":"## Generate submission"},{"metadata":{"trusted":true,"_uuid":"fcaa1ebf797e3fe975124f68de3e751e83b329e0"},"cell_type":"code","source":"out_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = test_pred_thresh\nout_df.to_csv(\"submission.csv\", index=False)\nout_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05a48694ee328826ea70f780f1f0380ad9c8bf0f"},"cell_type":"markdown","source":"## Thanks\n\nThis is my first kernel ever on Kaggle. If you are still here, thanks :)\n \nI hope you find it useful in some way and special thanks to all my sources and references.\n\n- Alber"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}