{"cells":[{"metadata":{"_uuid":"724ee57e793dee8fea96cf97c0639c8971b5c970"},"cell_type":"markdown","source":"# Quara RNN Baseline"},{"metadata":{"_uuid":"7bf28db87a5eccff81cebbeacddfab5ebebbed20"},"cell_type":"markdown","source":"This kernel is based on:\n1. Kernel [\"LSTM is all you need! well, maybe embeddings also\"](https://www.kaggle.com/mihaskalic/lstm-is-all-you-need-well-maybe-embeddings-also)\n2. [Practical Text Classification With Python and Keras](https://mlwhiz.com/blog/2018/12/17/text_classification/)\n\nThe kernel is organised as follows:\n1. Utils\n2. Setup and Data Preprocessing\n3. Neural Network\n4. Prediction and Submission"},{"metadata":{"trusted":true,"_uuid":"9cf1303d5a3ca28ce61013ccd216cce901b21c81"},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c46484c59ae029dafa312255a03234c1d0ae0c8"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\n\nseed = 123","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f3c444e6196f2aab92abf592827aee613a72723"},"cell_type":"markdown","source":"## 1. Utils"},{"metadata":{"_uuid":"2679808a0ffe1c9c3674ac2b2f2f196e7753a656"},"cell_type":"markdown","source":"#### 1.1 Dataset Loader"},{"metadata":{"trusted":true,"_uuid":"657fafe3634fcea8b1141e36f3647271eed8dcb0"},"cell_type":"code","source":"class DataLoader:\n    \n    def load(self, file_names):\n        self.train_df = pd.read_csv(file_names['train'])\n        self.test_df = pd.read_csv(file_names['test'])\n        ## fill up the missing values\n        self.train_df['question_text'].fillna('_na_', inplace=True)\n        self.test_df['question_text'].fillna('_na_', inplace=True)\n        print('Train shape : ', self.train_df.shape)\n        print('Test shape : ', self.test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2dd66786ffed3fcaa5486e330827284f4e1c0a6"},"cell_type":"markdown","source":"#### 1.2 Load embeddings & transform text to vectors"},{"metadata":{"trusted":true,"_uuid":"4e160709276adfa9d34e93f3d55363847c8c6494"},"cell_type":"code","source":"class Embeddings:\n    \n    def __init__(self):\n        self.embed_len = 300\n        self.punctuation = set('!#$%&()*+,.:;<>?@[\\\\]')\n    \n    ## load embeddings\n    def load(self, embedding_file_name):\n        self.embeddings_index = {}\n        f = open(embedding_file_name)\n        for line in tqdm(f):\n            values = line.split(' ');\n            word = values[0]\n            coef = np.asarray(values[1:], dtype='float32')\n            self.embeddings_index[word] = coef\n        self.embed_len = len(coef) # length of embeddings\n        f.close()\n        \n    def text_to_vec(self, text, max_text_len=30):\n        if text[-1] in self.punctuation:\n            text = text[:-1].split() + [text[-1]]\n        else:\n            text = text.split()\n        text = text[:max_text_len]\n        empyt_emb = np.zeros(self.embed_len)\n        embeds = [self.embeddings_index.get(word, empyt_emb) for word in text]\n        embeds+= [empyt_emb] * (max_text_len - len(embeds))\n        \n        return np.array(embeds)\n    \n    def sequences_to_vec(self, text_sequences, max_text_len=30):\n        vectors = [self.text_to_vec(text, max_text_len) for text in text_sequences]\n        \n        return np.array(vectors)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf336672b6236962642028683da574e96b406df3"},"cell_type":"markdown","source":"#### 1.3 Batch Generators"},{"metadata":{"trusted":true,"_uuid":"1b5431201787b6f4a20472b8c24f9aab5081fe43"},"cell_type":"code","source":"# generator for training NNet (used via fit_generator method in keras)\ndef batch_gen(train_df, batch_size, emb, max_text_len=30):\n    import math\n    n_batches = math.ceil(len(train_df) / batch_size)\n    while True: \n        train_df = train_df.sample(frac=1.)  # Shuffle the data.\n        for i in range(n_batches):\n            batch_df = train_df.iloc[i*batch_size:(i+1)*batch_size]\n            X_ = emb.sequences_to_vec( batch_df[\"question_text\"], max_text_len)\n            y_ = batch_df[\"target\"].values\n            yield X_, y_\n            \n# generater to do predication on test data\ndef batch_gen_test(test_df, emb, batch_size_test=256, max_text_len=30):\n    import math\n    n_batches = math.ceil(len(test_df) / batch_size)\n    for i in range(n_batches):\n        batch_df = test_df.iloc[i*batch_size:(i+1)*batch_size]\n        X_ = emb.sequences_to_vec( batch_df[\"question_text\"], max_text_len)\n        yield X_","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6368ba8fd9f99e075b6b7357e415276dcfcbad32"},"cell_type":"markdown","source":"#### 1.4 Visualisation script (Learning curves)"},{"metadata":{"trusted":true,"_uuid":"0bcf26cea9fb5aafed66a252090e378bf58f38ea"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.style.use('ggplot')\n\ndef plot_history(history):\n    acc = history.history['acc']\n    val_acc = history.history['val_acc']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    x = range(1, len(acc) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(x, acc, 'b', label='Training acc')\n    plt.plot(x, val_acc, 'r', label='Validation acc')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.title('Training and validation accuracy')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.plot(x, loss, 'b', label='Training loss')\n    plt.plot(x, val_loss, 'r', label='Validation loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title('Training and validation loss')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5419fc4114ef0c0483aa78d030d9ffcbab9e533"},"cell_type":"markdown","source":"## 2. Setup and Data Preprocessing"},{"metadata":{"_uuid":"88126d667637e9959ce664b72ed5a7c5321c9a2b"},"cell_type":"markdown","source":"#### 2.1 Load data"},{"metadata":{"trusted":true,"_uuid":"3d4be8a15b1d2bb2af0c61e2e15e9ad91a68fbc6"},"cell_type":"code","source":"file_names = {'train': '../input/train.csv', 'test': '../input/test.csv'}\n\ndl = DataLoader()\ndl.load(file_names)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58e7808af4ea7d6adb9c482e4f0e2fe6e1c6826a"},"cell_type":"markdown","source":"#### 2.2 Distribution of the number of words in questions"},{"metadata":{"trusted":true,"_uuid":"c50fd16d2ea13dc41c118af00c65f9247f81e112"},"cell_type":"code","source":"num_words_in_questions = [len(question.split()) for question in dl.train_df['question_text']]\nplt.hist(num_words_in_questions,bins = np.arange(0,103,3))\nplt.show()\n\nmax_text_len = 40 # according to the histogram below","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d921633c252f51901e17b9db49945d8ff7c78af"},"cell_type":"markdown","source":"#### 2.3  Load embeddings"},{"metadata":{"trusted":true,"_uuid":"bec131109db240a61d197d8dc570726d787103ca"},"cell_type":"code","source":"embedding_file_name = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nemb = Embeddings()\nemb.load(embedding_file_name)\nprint('Embedding loaded')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e061f026d5cdad97bdff3b2e233de8a36f0c9966"},"cell_type":"markdown","source":"#### 2.4 Prepare train/validation sets"},{"metadata":{"trusted":true,"_uuid":"6a59200b17ba9e495adb4958be0e5276d289305e"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nval_size = 0.03\ntrain_df, val_df, _, _ = train_test_split(dl.train_df, dl.train_df['target'], \n                                          test_size=val_size, \n                                          stratify=dl.train_df['target'],\n                                          shuffle=True, random_state=seed)\nval_X = emb.sequences_to_vec(val_df[\"question_text\"], max_text_len)\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcabf9993a110fb6396ed8562bc26a4220b05fd2"},"cell_type":"markdown","source":"## 3. Neural Network"},{"metadata":{"trusted":true,"_uuid":"e3092d8117bc873c0062ed39a28057c2d456ccb1"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd4228ed4bd787c988de99d296e12631ecd92434"},"cell_type":"markdown","source":"#### 3.1 Define NNet"},{"metadata":{"trusted":true,"_uuid":"af4e52967389faf3c01fd3da788bce1419a7feca"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Bidirectional(CuDNNLSTM(64, return_sequences=True),\n                        input_shape=(max_text_len, 300)))\nmodel.add(Bidirectional(CuDNNLSTM(64)))\nmodel.add(Dense(1, activation=\"sigmoid\"))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74cec3307a492a32cbb083bf5ef4000550522ebb"},"cell_type":"markdown","source":"#### 3.2 Train NNet"},{"metadata":{"trusted":true,"_uuid":"18081e23c74d8ca8ecae27d9e24fd146b28a5f70"},"cell_type":"code","source":"batch_size = 128\nmg = batch_gen(train_df, batch_size, emb, max_text_len)\nmodel.fit_generator(mg, epochs=20,\n                    steps_per_epoch=1000,\n                    validation_data=(val_X, val_y),\n                    verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc736b91a3c402d02c11c2bf23029e4f36a26805"},"cell_type":"markdown","source":"#### 3.3 Plot Learning curves"},{"metadata":{"trusted":true,"_uuid":"c70236435b8ae98da660e52124a5206270012648"},"cell_type":"code","source":"plot_history(model.history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dfc5613ec771d961937f2fd6c95f73470c6b39b5"},"cell_type":"markdown","source":"## 4. Prediction and Submission"},{"metadata":{"_uuid":"db8beb27e55b5a168a734a385e1087fe65eca925"},"cell_type":"markdown","source":"#### 4.1 Search for an optimal classification threshold on validation set"},{"metadata":{"trusted":true,"_uuid":"20767d884a3d642c8ec0347fc87bcfc14f3fcee4"},"cell_type":"code","source":"from sklearn import metrics\npredicted_val_prob = model.predict(val_X).flatten()\n \nbest_thr, best_f1 = 0., 0.    \nfor clf_thr in np.arange(0.1, 0.501, 0.01):\n    clf_thr = np.round(clf_thr, 2)\n    predicted_val_bin = (np.array(predicted_val_prob) > clf_thr).astype(np.int)\n    f1_val = metrics.f1_score(val_y, predicted_val_bin)\n    if best_f1 <= f1_val:\n        best_f1, best_thr = f1_val, clf_thr\n    print('F1 score at threshold {} is {:1.3f}'.format(clf_thr, f1_val))\nprint('Best classification threshold on validation set is {}, F1 is {:1.3f}'.format(best_thr, best_f1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abf753aa614392ed10a9d2cd774f91d390f4e7c2"},"cell_type":"markdown","source":"#### 4. 2 Prediction"},{"metadata":{"trusted":true,"_uuid":"78baca992a796a7292ad4506970bd5b4a5c81bdf"},"cell_type":"code","source":"# predict probabilities\nall_preds_prob = []\nbatch_size_test = 256\nfor x in tqdm(batch_gen_test(dl.test_df, emb, batch_size_test, max_text_len)):\n    all_preds_prob.extend(model.predict(x).flatten())\n    \nall_preds_prob_bin = (np.array(all_preds_prob) > best_thr).astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9dd45414ad1e4d9a35740dbd7eacb84d374470e3"},"cell_type":"markdown","source":"#### 4. 3 Submission"},{"metadata":{"trusted":true,"_uuid":"190741dae85bff398e1f099d81266cf6c05820a7"},"cell_type":"code","source":"submit_df = pd.DataFrame({\"qid\": dl.test_df[\"qid\"], \"prediction\": all_preds_prob_bin})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb819c55df35f94f86376e151494d3b97dda79e0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}