{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"SEED = 1337\n\nimport os\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"\nos.environ[\"PYTHONHASHSEED\"] =  \"0\"\n\nimport time\nimport numpy as np\nimport tensorflow as tf\nimport pandas as pd\nimport itertools as it\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"## Text Cleaning\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }\n\ndef clean_text(x):\n    for dic in [contraction_mapping, mispell_dict, punct_mapping]:\n        for word in dic.keys():\n            x = x.replace(word, dic[word])\n    return x\n\n## Loading and preprocessing text\ndef load_and_preprocess_data(max_features=50000, maxlen=70):\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    #print(\"Train shape : \",train_df.shape)\n    #print(\"Test shape : \",test_df.shape)\n\n    ## split to train and val\n    train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=SEED)\n\n    train_df['question_text'] = train_df['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\n    test_df['question_text'] = test_df['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\n\n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    val_X = val_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X) + list(test_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences\n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    val_y = val_df['target'].values\n\n    #shuffling the data\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n    val_idx = np.random.permutation(len(val_X))\n\n    train_X = train_X[trn_idx]\n    val_X = val_X[val_idx]\n    train_y = train_y[trn_idx]\n    val_y = val_y[val_idx]\n\n    return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fefc7a5a4efb08de9b9ccfad4f3edd8a5438748"},"cell_type":"code","source":"# Generator for dataset\ndef data_gen(x, y, batch_size=32, shuffle=True):\n        x = np.array(x)\n        if len(x) == len(y) : y = np.array(y)\n\n        data_size = len(x)\n        num_batches_per_epoch = int((len(x)-1)/batch_size) + 1\n\n        if shuffle:\n            shuffle_indices = np.random.permutation(np.arange(data_size))\n            x = x[shuffle_indices]\n            if len(x) == len(y): y = y[shuffle_indices]\n\n        for batch_num in range(num_batches_per_epoch):\n            start_index = batch_num * batch_size\n            end_index = min((batch_num + 1) * batch_size, data_size)\n            bat_x = x[start_index:end_index]\n            bat_y = y[start_index:end_index]\n            bat_y = np.reshape(bat_y, (-1, 1))\n            yield np.array(bat_x), np.array(bat_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2723468480f08e2e7540c7c19903249e2a09940c"},"cell_type":"code","source":"## TensorFlow BiRNN Model\nclass RNN:\n    def __init__(self,\n                num_classes=1,\n                learning_rate=0.001,\n                batch_size=None,\n                seq_length=70,\n                vocab_size=10000,\n                embed_size=300,\n                hidden_size=64,\n                training=True):\n\n        ## Hyperparamters\n        self.num_classes = num_classes\n        self.learning_rate = learning_rate\n        self.batch_size = batch_size\n        self.seq_length = seq_length\n        self.vocab_size = vocab_size\n        self.embed_size = embed_size\n        self.hidden_size = hidden_size\n        self.training = True\n        self.decay_steps = 1000\n        self.decay_rate = 0.95\n\n        ## Placeholders\n        self.input_x = tf.placeholder(tf.int64, shape=[self.batch_size, self.seq_length], name=\"question\")\n        self.input_y = tf.placeholder(tf.int64, shape=[self.batch_size, self.num_classes], name=\"target\")\n        self.dropout_keep_prob = tf.placeholder(tf.float32, name=\"dropout_keep_prob\")\n\n        ## Model Checkpoint\n        self.model_dir = \"weights\"\n        self.model_name = \"rnn.ckpt\"\n\n        ## Embedding Weight\n        with tf.name_scope(\"embedding\"):\n            self.embedding_matrix = tf.get_variable(\"embedding_matrix\", shape=[self.vocab_size, self.embed_size], initializer=tf.random_normal_initializer(stddev=0.1))\n\n        ##\n        self.global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n        self.logits = self.inference()\n        self.predictions = tf.nn.sigmoid(self.logits)\n        self.loss_val = self.loss()\n        self.optimizer_op = self.optimizer().minimize(self.loss_val, global_step=self.global_step)\n        correct_predictions = tf.equal(tf.cast(tf.round(self.predictions), tf.int64), self.input_y)\n        self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name=\"accuracy\")\n        _, self.f1 = tf.contrib.metrics.f1_score(self.input_y, self.predictions)\n\n    def inference(self):\n        \"\"\" Embedding \"\"\"\n        self.embedding_words = tf.nn.embedding_lookup(self.embedding_matrix, self.input_x)\n\n        \"\"\" BiGRU \"\"\"\n        fw_cell = tf.nn.rnn_cell.GRUCell(self.hidden_size)\n        bw_cell = tf.nn.rnn_cell.GRUCell(self.hidden_size)\n        \n        #fw_cell = tf.contrib.cudnn_rnn.CudnnGRU(1, self.hidden_size, self.hidden_size)\n        #bw_cell = tf.contrib.cudnn_rnn.CudnnGRU(1, self.hidden_size, self.hidden_size)\n\n        if self.dropout_keep_prob is not None:\n            fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=self.dropout_keep_prob)\n            bw_cell = tf.nn.rnn_cell.DropoutWrapper(bw_cell, output_keep_prob=self.dropout_keep_prob)\n\n        outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, self.embedding_words, dtype=tf.float32)\n        output_rnn = tf.concat(outputs, axis=2)\n        final_output = output_rnn[:, -1, :]\n        \n        x = tf.layers.dense(final_output, self.hidden_size)\n        logits = tf.layers.dense(x, self.num_classes)\n        return logits\n\n    def loss(self, l2_lambda=0.0001):\n        with tf.name_scope(\"loss\"):\n            loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.cast(self.input_y, tf.float32), logits=self.logits))\n            l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * l2_lambda\n            loss = loss + l2_loss\n        return loss\n\n    def optimizer(self):\n        learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step, self.decay_steps, self.decay_rate, staircase=True)\n        train_op = tf.train.AdamOptimizer(learning_rate)\n        return train_op\n\n    def save_model(self, sess):\n        saver = tf.train.Saver()\n        save_path = os.path.join(self.model_dir, self.model_name)\n        saver.save(sess, save_path, global_step=self.global_step)\n        #print(\"Session saved.\")\n\n    def restore_model(self, sess):\n        saver = tf.train.Saver()\n        ckpt = tf.train.get_checkpoint_state(self.model_dir)\n        if ckpt and ckpt.model_checkpoint_path:\n            ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n            saver.restore(sess, os.path.join(self.model_dir, ckpt_name))\n            print(\"Session restored.\")\n\n    def train_once(self, sess, train_data):\n        loss = 0\n        accuracy = 0\n        f1 = 0\n        n_batch = 0\n\n        while True:\n            try:\n                x, y = next(train_data)\n                feed_data = {self.input_x: x, self.input_y: y, self.dropout_keep_prob: 1.0}\n                _, l, a, f = sess.run([self.optimizer_op, self.loss_val, self.accuracy, self.f1], feed_dict=feed_data)\n                loss += l\n                accuracy += a\n                f1 += f\n                n_batch += 1\n            except StopIteration as e:\n                break\n\n        if n_batch != 0:\n            loss = loss/n_batch\n            accuracy = accuracy/n_batch\n            f1 = f1/n_batch\n\n        return [loss, accuracy, f1]\n\n    def eval_once(self, sess, eval_data):\n        loss = 0\n        accuracy = 0\n        f1 = 0\n        n_batch = 0\n\n        while True:\n            try:\n                x, y = next(eval_data)\n                feed_data = {self.input_x: x, self.input_y: y, self.dropout_keep_prob: 1.0}\n                l, a, f = sess.run([self.loss_val, self.accuracy, self.f1], feed_dict=feed_data)\n                loss += l\n                accuracy += a\n                f1 += f\n                n_batch += 1\n            except StopIteration as e:\n                break\n\n        if n_batch != 0:\n            loss = loss/n_batch\n            accuracy = accuracy/n_batch\n            f1 = f1/n_batch\n\n        return [loss, accuracy, f1]\n\n    def train(self, train_data, valid_data, epochs=5):\n        with tf.Session() as sess:\n            sess.run(tf.global_variables_initializer())\n            sess.run(tf.local_variables_initializer())\n\n            ## Restore Session\n            self.restore_model(sess)\n\n            train_data_copy = list(it.tee(train_data, epochs))\n            valid_data_copy = list(it.tee(valid_data, epochs))\n\n            print(\"Epoch\\tTime\\tTrain Loss\\tTrain Acc\\tTrain F1\\tVal Loss\\tVal Acc\\t\\tVal F1\")\n            print(\"=\"*110)\n\n            for epoch in range(epochs):\n                start_time = time.time()\n                train_loss, train_acc, train_f1 = self.train_once(sess, train_data_copy[epoch])\n                valid_loss, valid_acc, valid_f1 = self.eval_once(sess, valid_data_copy[epoch])\n                time_taken = time.time() - start_time\n\n                # print(\"Epoch: {:2d} - {:3.4f} - Loss: {:1.5f} - Acc: {:0.5f} - Val loss: {:1.5f} - Val acc: {:0.5f}\".\n                #     format(epoch, time_taken, train_loss, train_acc, valid_loss, valid_acc))\n                print(\"{:2d}\\t{:3.2f}\\t{:1.8f}\\t{:0.8f}\\t{:0.8f}\\t{:1.8f}\\t{:0.8f}\\t{:0.8f}\".\n                    format(epoch+1, time_taken, train_loss, train_acc, train_f1, valid_loss, valid_acc, valid_f1))\n\n                ## Save Session\n                self.save_model(sess)\n                \n    def predict(self, test_data):\n        with tf.Session() as sess:\n            sess.run(tf.global_variables_initializer())\n            sess.run(tf.local_variables_initializer())\n\n            start_time = time.time()\n            test_predict = np.array([])\n\n            while True:\n                try:\n                    x, y = next(test_data)\n                    feed_data = {self.input_x: x, self.dropout_keep_prob: 1.0}\n                    r = sess.run([self.predictions], feed_dict=feed_data)\n                    test_predict = np.append(test_predict, r)\n                except StopIteration as e:\n                    break\n\n            time_taken = time.time() - start_time\n            print(\"Time Taken: {:2.5f}\".format(time_taken))\n\n        return test_predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"420826a61d1d460cf80cd82a89518b4837d962a9"},"cell_type":"code","source":"start_time = time.time()\n# Hyperparamters\nnum_classes=1\nlearning_rate=0.05\nbatch_size=512\nseq_length=40\nvocab_size=95000\nembed_size=300\nhidden_size=64\ntraining=True\nepochs=10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3a87100f36885f7d9fa68f7f9891ad4c0751f82"},"cell_type":"code","source":"print(\"[+]Preprocessing text...\")\ntrain_X, valid_X, test_X, train_y, valid_y, word_index = load_and_preprocess_data(max_features=vocab_size, maxlen=seq_length)\n\nprint(\"[+]Preparing generators...\")\ntrain_gen = data_gen(train_X, train_y, batch_size=batch_size)\nvalid_gen = data_gen(valid_X, valid_y, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"6b11b8bba0509951e4e9899823a6618e8932f46d"},"cell_type":"code","source":"print(\"[+]TensorFlow model...\")\nrnn = RNN(\n        num_classes=num_classes,\n        learning_rate=learning_rate,\n        batch_size=None,\n        seq_length=seq_length,\n        vocab_size=vocab_size,\n        embed_size=embed_size,\n        hidden_size=hidden_size,\n        training=training)\n\nprint(\"[+]Training...\")\ntime_taken = time.time() - start_time\nprint(\"Time Taken: {:2.5f}\".format(time_taken))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"738103f67cc9169eb8cb05d7ce11f53e75b4e4fc"},"cell_type":"code","source":"rnn.train(train_gen, valid_gen, epochs=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4c00c2a86625f01f9ccda99e703117f43ddd5b1"},"cell_type":"code","source":"test_gen = data_gen(test_X, test_X, batch_size=1024, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccfb7c6ca98bd0463f2c44571ac68c3da1122f8b"},"cell_type":"code","source":"test_predict = rnn.predict(test_gen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"989d035bc8f2e3a45aa1cf89bc7d10765056d14c"},"cell_type":"code","source":"pred_test_y = (test_predict > 0.5).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fac54a7e87a7e1cd7c870c122dda83e2fb022b2"},"cell_type":"code","source":"from IPython.display import HTML\nimport base64  \nimport pandas as pd  \n\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index =False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(out_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e40ae5243b3f89e91796a8b95ee7c8174ed3d6fd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}