{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom gensim.models import KeyedVectors\nimport tensorflow as tf\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nsubmission_df = pd.read_csv(\"../input/sample_submission.csv\")\nprint(train_df.shape)\nprint(test_df.shape)\nprint(submission_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a37f71c657ed1fc36dac9e0c3be273c9bb642d16"},"cell_type":"code","source":"news_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05c0c3438a70e2fe6152ed939079a60ecc891326"},"cell_type":"code","source":"def build_vocab(sentences):\n    vocab = {}\n    for sentence in tqdm(sentences, desc=\"Building Vocabulary\"):\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef check_coverage(vocab, embeddings):\n    found = {}\n    oov = {}\n    f, nf = 0, 0\n    for word in vocab:\n        if word in embeddings:\n            found[word] = vocab[word]\n            f += vocab[word]\n        else:\n            oov[word] = vocab[word]\n            nf += vocab[word]\n    print(\"Found embeddings for %.4f of vocab\"%(len(found)/len(vocab)))\n    print(\"Found embeddings for %.4f of total text\"%(f/(f+nf)))\n    return oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef4b34632dbca46035541a7314cb8795a7e34745"},"cell_type":"code","source":"sentences = train_df[\"question_text\"].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\nprint({k: vocab[k] for k in list(vocab)[:10]})\noov = check_coverage(vocab, embeddings)\nprint(list(oov)[:20])\ndel sentences\ndel vocab\ndel oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b641c57f771b1b0706bd2c1f7ceeb1949e0cd0cf"},"cell_type":"code","source":"contraction_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\ncontraction_re = re.compile(\"(%s)\"%\"|\".join(contraction_dict.keys()))\nmispell_re = re.compile(\"(%s)\"%\"|\".join(mispell_dict.keys()))\n\ndef remove_contraction(text):\n    text = str(text)\n    return contraction_re.sub(lambda match: contraction_dict[match.group(0)], text)\n\ndef remove_mispell(text):\n    text = str(text)\n    return mispell_re.sub(lambda match: mispell_dict[match.group(0)], text)\n\ndef clean_numbers(x):\n    x = str(x)\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef clean_text(x):\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, ' ')\n    return x\n\ndef remove_oovs(x):\n    x = str(x)\n    to_remove = ['a','to','of','and']\n    x = re.sub(r'\\ba\\b', '', x)\n    x = re.sub(r'\\bto\\b', '', x)\n    x = re.sub(r'\\bof\\b', '', x)\n    x = re.sub(r'\\band\\b', '', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"307bcae8cad56a4e12b32e3ad39bad9d5f06423d"},"cell_type":"code","source":"def process_data(df):\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(remove_contraction)\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(clean_text)\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(clean_numbers)\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(remove_mispell)\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(remove_oovs)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0879742817f36f37d60ef7872b735eab726cb0d3"},"cell_type":"code","source":"train_df = process_data(train_df)\ntest_df = process_data(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cefd9e9ad59e923d613eefa40bce8d46d6c6a79"},"cell_type":"code","source":"sentences = train_df[\"question_text\"].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\nprint({k: vocab[k] for k in list(vocab)[:10]})\noov = check_coverage(vocab, embeddings)\nprint(list(oov)[:20])\ndel sentences\ndel vocab\ndel oov","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"437e672f9c86b9a7a982732cefb26f837d0a4852"},"cell_type":"code","source":"max_sent_len = 72\nembedding_size = 300\n\ndef tokenize_sentences(df, field, new_field=\"tokens\"):\n    df[new_field] = [[] for _ in range(len(df))]\n    for i, row in enumerate(df[field]):\n        tokens = row.split()\n        if len(tokens) > max_sent_len:\n            tokens = tokens[:max_sent_len]\n        df[new_field][i].extend(tokens)\n    return df\n\ntrain_df = tokenize_sentences(train_df, \"question_text\")\ntest_df = tokenize_sentences(test_df, \"question_text\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14b58772c4a194fd223bbc337e1f174ff7961baf"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0981f680b021d1a843076a81b71fe4f5d45f3543"},"cell_type":"code","source":"def generate_embeddings(words, word2vec, max_sent_len, embedding_size=300):\n    embeddings = np.zeros((max_sent_len, embedding_size), dtype=np.float32)\n    for i, word in enumerate(words):\n        try:\n            embeddings[i] = word2vec.word_vec(word)\n        except KeyError:\n            pass\n    return embeddings\n\ndef batch_generator(df, batch_size, word2vec, max_sent_len, token_field=\"tokens\", lbl_field=\"target\", embedding_size=300):\n    iterations = len(df)//batch_size\n    for i in range(iterations):\n        inp = np.ndarray(shape=(batch_size, max_sent_len, embedding_size), dtype=np.float32)\n        lbl = np.ndarray(shape=(batch_size, 1), dtype=np.float32)\n        for j in range(batch_size):\n            tokens = df[token_field][i*batch_size+j]\n            inp[j] = generate_embeddings(tokens, word2vec, max_sent_len, embedding_size)\n            lbl[j] = df[lbl_field][i*batch_size+j]\n        yield np.expand_dims(inp, axis=3), lbl\n    return\n\ndef test_batch_generator(df, batch_size, word2vec, max_sent_len, token_field=\"tokens\", embedding_size=300):\n    iterations = len(df)//batch_size\n    for i in range(iterations):\n        inp = np.ndarray(shape=(batch_size, max_sent_len, embedding_size), dtype=np.float32)\n        for j in range(batch_size):\n            tokens = df[token_field][i*batch_size+j]\n            inp[j] = generate_embeddings(tokens, word2vec, max_sent_len, embedding_size)\n        yield np.expand_dims(inp, axis=3)\n    return\n\ngen = batch_generator(train_df, 128, embeddings, max_sent_len)\nt_gen = test_batch_generator(test_df, 100, embeddings, max_sent_len)\ninp, lbl = next(gen)\nt_inp = next(t_gen)\nprint(inp.shape)\nprint(lbl.shape)\nprint(t_inp.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e754d36c38b8bbd6aa82bd21e148d8cd59fad3c"},"cell_type":"code","source":"def conv2d(x, output_channels, filter_size):\n    w_init = tf.contrib.layers.variance_scaling_initializer()\n    b_init = tf.random_uniform_initializer(0.0, 0.01)\n    w = tf.get_variable(\"w\", shape=[filter_size, 300, 1, output_channels], dtype=tf.float32, initializer=w_init)\n    b = tf.get_variable(\"b\", shape=[output_channels], dtype=tf.float32, initializer=b_init)\n    conv = tf.nn.conv2d(x, w, strides=[1, 1, 1, 1], padding=\"VALID\") + b\n    return tf.nn.tanh(conv)\n\ndef score(y_pred, y_true):\n    assert len(y_pred) == len(y_true)\n    return np.sum(np.equal(np.round(y_pred).astype(np.int32), y_true.astype(np.int32)))/len(y_true)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"476438a5391ac80d457d1d6e9ebbdc824123084b"},"cell_type":"code","source":"batch_size = 128\ntf.reset_default_graph()\ninp_plh = tf.placeholder(tf.float32, [None, max_sent_len, embedding_size, 1], \"input_placeholder\")\nlbl_plh = tf.placeholder(tf.float32, [None, 1], \"label_placeholder\")\ntrain_plh = tf.placeholder(tf.bool, None, \"training_placeholder\")\n\nwith tf.variable_scope(\"conv1\"):\n    conv1 = conv2d(inp_plh, 36, 1)\n    conv1 = tf.nn.max_pool(conv1, ksize=[1, max_sent_len-1+1, 1, 1], strides=[1, 1, 1, 1], padding=\"VALID\")\nwith tf.variable_scope(\"conv2\"):\n    conv2 = conv2d(inp_plh, 36, 2)\n    conv2 = tf.nn.max_pool(conv2, ksize=[1, max_sent_len-2+1, 1, 1], strides=[1, 1, 1, 1], padding=\"VALID\")\nwith tf.variable_scope(\"conv3\"):\n    conv3 = conv2d(inp_plh, 36, 3)\n    conv3 = tf.nn.max_pool(conv3, ksize=[1, max_sent_len-3+1, 1, 1], strides=[1, 1, 1, 1], padding=\"VALID\")\nwith tf.variable_scope(\"conv4\"):\n    conv4 = conv2d(inp_plh, 36, 5)\n    conv4 = tf.nn.max_pool(conv4, ksize=[1, max_sent_len-5+1, 1, 1], strides=[1, 1, 1, 1], padding=\"VALID\")\nconc = tf.concat([conv1, conv2, conv3, conv4], axis=1)\nshape = conc.get_shape().as_list()\ndim = shape[1]*shape[2]*shape[3]\nreshaped = tf.reshape(conc, shape=[-1, dim])\n# reshaped = tf.cond(train_plh, lambda: reshaped, lambda: tf.nn.dropout(reshaped, keep_prob=0.9))\nw_init = tf.contrib.layers.variance_scaling_initializer()\nb_init = tf.constant_initializer(0.01)\nw = tf.get_variable(\"w\", shape=[dim, 1], dtype=tf.float32, initializer=w_init)\nb = tf.get_variable(\"b\", shape=[1], dtype=tf.float32, initializer=b_init)\nlogits = tf.matmul(reshaped, w) + b\npred = tf.nn.sigmoid(logits)\nloss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=lbl_plh))\noptimizer = tf.train.AdamOptimizer().minimize(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a750c800ae19fabf264220fc4573acbfa34a182d"},"cell_type":"code","source":"num_epochs = 9\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n    print(\"Training Started...\")\n    for epoch in range(num_epochs):\n        loss_list = []\n        acc_list = []\n        gen = batch_generator(train_df, batch_size, embeddings, max_sent_len)\n        for inp, lbl in gen:\n            feed_dict = {inp_plh: inp, lbl_plh: lbl, train_plh: True}\n            _, loss_val, pred_val = sess.run([optimizer, loss, pred], feed_dict=feed_dict)\n            loss_list.append(float(loss_val))\n            acc_list.append(float(score(pred_val, lbl)))\n        print(\"##########Epoch %d completed##############\"%(epoch+1))\n        print(\"Average Loss: \", float(np.mean(loss_list)))\n        print(\"Average Accuracy: \", float(np.mean(acc_list)))\n        print(\"##########################################\")\n    print(\"Generating submission data...\")\n    sub_df = pd.read_csv(\"../input/sample_submission.csv\")\n    gen = test_batch_generator(test_df, 100, embeddings, max_sent_len)\n    index = 0\n    for inp in gen:\n        feed_dict = {inp_plh: inp, train_plh: False}\n        pred_val = sess.run(pred, feed_dict=feed_dict)\n        sub_df.prediction.iloc[index: index+100] = np.round(pred_val).astype(np.int32).reshape(100)\n        index += 100\n    sub_df.to_csv(\"submission.csv\", index=False)\n    print(\"Submission file generated as submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}