{"cells":[{"metadata":{"_uuid":"fbad4fb9a94e192dc4ef6f1ff3156214ca02b7f1"},"cell_type":"markdown","source":"This kernel is based off two ideas from two papers: <br />\n1) Hierarchical Attention Networks for Document Classification by Yang et al. (2016)  <br />\n2) Convolutional Neural Networks for Sentence Classification by Yoon KIm (2014)\n\nThe second paper was included as a way to add more features to the model. Adding these features improved the score from 0.635 to 0.656 (ongoing effort!)\n\nTo find out more about the technical implementation of the Hierarchical Attention Network, please visit my Linkedin post:https://www.linkedin.com/pulse/implementation-hierarchical-attention-network-nitin-venkateswaran/\n\nDo note that this score was achieved by using only ~720,000 training records in mini-batches of 256, out of the 1.3 million provided, with upsampling of positive records to match the negative class count in the  mini-batch.\nThe constraint is in the GPU time allocated by the competition (only 2 hours).\n\n**If Kaggle can allow longer GPU time for training, that would be much appreciated :)**\n\nTo read this kernel, navigate to the bottom all the way to the main function, and then follow the function implementations as you read. \n\nSuggestions and feedback are very welcome."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n#import zipfile\n#import nltk\n#from num2words import num2words\n\n#Dataset = \"airline-safety\"\n\n# Will unzip the files so that you can see them..\n#with zipfile.ZipFile(\"../input/embeddings.zip/glove.840B.300d/glove.840B.300d.txt\",\"r\") as z:\n#    z.extractall(\".\")\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings/glove.840B.300d/\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec85bae6d1da17bef98b561a793c65019b2e130f"},"cell_type":"code","source":"#import nltk\n#nltk.download('stopwords')\n#from nltk.corpus import stopwords\n#stop_words = set(stopwords.words('english'))\n\nimport pandas as pd\npd.set_option('display.max_columns',10)\n\n\nimport re, os, shutil\nimport numpy as np\nnp.set_printoptions(threshold=np.nan)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, f1_score\nfrom sklearn.utils import shuffle\n\nimport tensorflow as tf\n\ncutoff_shape = 2196017\nglove_dim = 300\nmax_seq_len = 122\ncutoff_seq = 40\nmax_sent_seq_len = 12 # 12 sentences in a doc\nsent_cutoff_seq = 3\npos_wt = 0.25\n\n#reload_mmap = True\n\n\n'''\ndef load_npy_file(npy_file):\n\n    mmap = np.memmap(npy_file, dtype='float32', mode='r', shape=(cutoff_shape + 2, glove_dim))\n\n    print (mmap[[0,2196015,2196017,2196018]].shape)\n\n\ndef get_max_len(train_file):\n\n    max = -1\n    df = pd.read_csv(train_file,low_memory=False)\n    for index, item in df.iterrows():\n        l = len(str(item[1]).split(' '))\n        #print (l)\n        if (max < l):\n            max = l\n\n    print ('Max is:')\n    print (max)\n'''\n\n\ndef load_glove_vectors(memmap_loc, glove_file, reload_mmap = False):\n\n    \"\"\"\n    Create a memmap\n    \"\"\"\n\n    #wts = []\n    glove_dict = {}\n\n    i = 0\n\n    mmap = None\n\n    if (reload_mmap):\n        mmap = np.memmap(memmap_loc, dtype='float32', mode='w+', shape=(cutoff_shape + 2, glove_dim))\n\n    with open (glove_file,'r') as f:\n\n        for index, line in enumerate(f):\n            l = line.split(' ')\n\n            #add to the dictionary\n            glove_dict[str(l[0])] = index\n\n            # Add to memmap\n            if (reload_mmap):\n                del l[0]\n                mmap[index,:] = l\n\n            #wts.append(l)\n            i += 1\n            #if (i > cutoff_shape): # for dev purposes only\n            #    break\n\n\n    # contains the word embeddings. assumes indexes start from 0-based in the txt file\n\n    if (reload_mmap):\n        # Add the UNK\n        #unk_wt = np.random.randn(glove_dim)\n        unk_wt = np.zeros(glove_dim)\n        #wts.append(unk_wt.tolist())\n        mmap[i:] = unk_wt\n        i += 1\n\n        # Add the _NULL\n        null_wt = np.zeros(glove_dim)\n        #wts.append(null_wt.tolist())\n        mmap[i:] = null_wt\n        mmap.flush()\n\n    #weights = np.asarray(wts)\n\n    #assert weights.shape[1] == glove_dim\n    #assert weights.shape[0] == cutoff_shape + 3\n\n    return glove_dict\n\n\ndef get_train_df_glove_dict(train_file, glove_file,mmap_loc, reload_mmap, is_training = True, ):\n\n\n    df = pd.read_csv(train_file,low_memory=False)\n\n    # creates the mmap\n    glove_dict = load_glove_vectors(mmap_loc,glove_file,reload_mmap)\n\n    if (is_training):\n\n        y = df.loc[:, 'target']\n        X_train, X_dev, y_train, y_dev = train_test_split(df, y, test_size=0.01, stratify=y)\n\n        X_train_0 = X_train.loc[X_train['target'] == 0]\n        X_train_1 = X_train.loc[X_train['target'] == 1]\n\n        X_train_0_sample = X_train_0.sample(n=X_train_0.shape[0])\n        X_train_1_sample = X_train_1.sample(n=X_train_1.shape[0])\n\n        X_dev_0 = X_dev.loc[X_dev['target'] == 0]\n        X_dev_1 = X_dev.loc[X_dev['target'] == 1]\n\n        X_dev_0_sample = X_dev_0.sample(n=X_dev_0.shape[0])\n        X_dev_1_sample = X_dev_1.sample(n=X_dev_1.shape[0])\n\n        X_train_f = pd.concat([X_train_0_sample,X_train_1_sample],axis=0)\n        X_train_f = shuffle(X_train_f)\n        X_dev_f = pd.concat([X_dev_0_sample, X_dev_1_sample], axis=0)\n\n        return X_train_f, X_dev_f, glove_dict\n\n    else:\n\n        return None, None , glove_dict\n\n\n\ndef process_questions(qn_df, glove_dict, mmap, is_training = True):\n\n    #UNK = 2196017\n    UNK = cutoff_shape\n    _NULL = cutoff_shape + 1\n    sentence_batch_len = []\n    sentence_batch_len_2 = []\n    y_len_2 = []\n\n    qn_ls_word_idx = []\n    qn_batch_len = []\n\n    l = 0\n    stop_words = ['how', 'were', 'through', 'up', 'ma', 'because', 'his', \"hasn't\", 'myself', 'that', 'then', 'm', 'haven', 'during',\n     'being', 'needn', 'whom', 'did', 'the', 'very', 'd', 'as', 'their', 'ours', 'was', 'be', 'themselves', \"mustn't\",\n     'she', 'or', 'but', 'yourself', 'if', 'had', 'not', 'itself', 'than', 'such', 'having', 'we', 'again', 'about',\n     'this', 'with', 'any', 'on', 'has', 'a', 'there', 'so', 'all', 'both', 'him', 'its', 'yourselves', \"couldn't\",\n     'by', 'they', \"mightn't\", 'mightn', \"wouldn't\", 're', 'you', 'above', 'more', 'while', \"you've\", 'after', 'mustn',\n     'shan', 's', 'some', 'in', 'will', 'and', 'same', 'hadn', \"haven't\", 'weren', \"don't\", 'herself', \"won't\", 'of',\n     'down', 'aren', 'ourselves', 'wasn', 'your', 'just', 'yours', 'out', 'have', 'isn', 'most', 'until', 'can', 'it',\n     \"didn't\", 'don', \"you're\", 'over', 't', 'couldn', 'who', 'these', 'below', 'each', 'should', 'hers', 'other',\n     \"she's\", \"shan't\", 'shouldn', 've', \"it's\", 'y', 'wouldn', 'into', 'what', 'does', 'where', 'to', \"you'll\",\n     \"weren't\", 'at', \"that'll\", 'do', 'further', 'been', 'are', 'i', 'which', 'own', 'from', 'why', 'between',\n     \"shouldn't\", 'nor', \"hadn't\", \"you'd\", 'is', 'an', 'am', 'o', 'didn', 'for', 'them', 'only', 'hasn', 'me',\n     \"wasn't\", 'he', \"aren't\", 'too', \"should've\", 'now', 'ain', 'few', 'll', 'her', 'against', 'theirs', 'off',\n     'doing', \"doesn't\", 'won', \"isn't\", 'once', 'here', 'my', 'before', 'those', 'under', 'himself', 'doesn', 'when',\n     \"needn't\", 'no', 'our','why','how','when','if','who','what']\n    for index, item in qn_df.iterrows():\n\n\n        qn1 = item[1].split('. ') # Extract the sentences\n        #print (qn1)\n        qn2 = [x.split('? ') for x in qn1]\n        #print (qn2)\n        qn3 = [x for y in qn2 for x in y if x != '']\n        #print (qn3)\n        \n        qn_ls_0 = [q.replace('/',' ') for q in qn3]\n        qn_ls_0 = [re.sub(' +', ' ',q) for q in qn_ls_0]\n        qn_ls = [re.sub('[^A-Za-z0-9\\' ]+', '', q) for q in qn_ls_0]\n\n        # Cutoff\n        if (len(qn_ls) > sent_cutoff_seq):\n            qn_ls = qn_ls[:sent_cutoff_seq]\n\n        sentence_batch_len.append(len(qn_ls))\n\n        # word level tokens\n        qn_ls_word = [x.split(' ') for x in qn_ls]\n        #print (qn_ls_word)\n\n        # Bit misnomer. qn_ls_word is still array of sentences in each document / answer\n        for y in qn_ls_word:\n\n            if (is_training):\n                y_len_2.append(item[2]) # 'elongate' the target variable. This will also need to be re-stitched later on.\n            else:\n                y_len_2.append(item[0]) # this is called during inference. The same trick is played on the key ID during the graph.\n\n\n            tmp0 = [x for x in y if x.strip().lower() not in stop_words]\n            tmp = [glove_dict[x] if (x in glove_dict.keys()) else UNK for x in tmp0]\n            \n            qn_ls_word_idx.append(tmp)\n\n            if (len(tmp) > cutoff_seq):\n                qn_batch_len.append(cutoff_seq)\n            else:\n                qn_batch_len.append(len(tmp)) \n\n            sentence_batch_len_2.append(len(qn_ls))\n\n    # Now we have max_len, a flattened sentence matrix, and batch sequence length for dynamic rnn.\n    # Apply the _null padding.\n\n    batch_shape = len(qn_ls_word_idx)\n    #qn_ls_word_embd = np.empty((batch_shape, max_seq_len, glove_dim))\n    qn_ls_word_embd = np.empty((batch_shape, cutoff_seq, glove_dim))\n\n    for item in qn_ls_word_idx:\n        item += [_NULL] * (max_seq_len - len(item))\n\n        # Apply a cutoff\n        item = item[:cutoff_seq]\n\n        # Word embedding lookup from memmapped file\n        tmp_npy = mmap[item]\n        \n        tmp_npy = np.expand_dims(tmp_npy,axis=0)\n        #tmp_npy_bi = tmp_npy_bi[1:,:]\n        #print (tmp_npy_bi.shape)\n\n        #tmp_npy_bi = np.expand_dims(tmp_npy_bi,axis=0)\n        qn_ls_word_embd[l] = tmp_npy\n        \n        l += 1\n\n    return qn_ls_word_embd, qn_batch_len, sentence_batch_len, sentence_batch_len_2, y_len_2 # Flattened qn_lengths, and sentence_len at document level\n\n\n\ndef build_graph(max_sentence_len):\n\n    keep_fc = tf.placeholder(dtype=tf.float32, name=\"keep_1\")\n    keep_conv = tf.placeholder(dtype=tf.float32, name=\"keep_conv\")\n\n    def sparse_softmax(T):\n\n        # Creating partition based on condition:\n        condition_mask = tf.cast(tf.equal(T, 0.), tf.int32)\n        partitioned_T = tf.dynamic_partition(T, condition_mask, 2)\n        # Applying the operation to the target partition:\n        partitioned_T[0] = tf.nn.softmax(partitioned_T[0],axis=0)\n\n        # Stitching back together, flattening T and its indices to make things easier::\n        condition_indices = tf.dynamic_partition(tf.range(tf.size(T)), tf.reshape(condition_mask, [-1]), 2)\n        res_T = tf.dynamic_stitch(condition_indices, partitioned_T)\n        res_T = tf.reshape(res_T, tf.shape(T))\n\n        return res_T\n\n    gru_units = 256\n    output_size = 256\n\n    gru_units_sent = 256\n    output_size_sent = 256\n\n    cell_fw = tf.nn.rnn_cell.GRUCell(gru_units)\n    cell_bw = tf.nn.rnn_cell.GRUCell(gru_units)\n\n    cell_sent_fw = tf.nn.rnn_cell.GRUCell(gru_units_sent)\n    cell_sent_bw = tf.nn.rnn_cell.GRUCell(gru_units_sent)\n\n    # Now load the inputs and convert them to word vectors\n    with tf.variable_scope(\"layer_inputs\"):\n        inputs = tf.placeholder(dtype=tf.float32, shape=[None,max_sentence_len,glove_dim],name=\"input\")\n        batch_sequence_lengths = tf.placeholder(dtype=tf.int32,name=\"sequence_length\")\n\n\n    with tf.variable_scope('layer_conv_1D'):\n\n        # Does feature selection using n-grams + CNN for each sentence in a document.\n        feature_map_size = 256\n        kernel_1 = 3\n        kernel_2 = 4\n        kernel_3 = 5\n\n        l1w_init = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n        l2w_init = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n        l3w_init = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n\n        first_conv_layer = tf.layers.conv1d(inputs=inputs, filters=feature_map_size,\n                                            kernel_size=kernel_1, strides=1, padding='valid',\n                                            activation=tf.nn.relu, use_bias=True, kernel_initializer=l1w_init,\n                                            bias_initializer=tf.zeros_initializer(), name=\"first_conv_layer\")\n\n        first_maxpool_layer = tf.reduce_max(first_conv_layer, axis=1, keepdims=True, name=\"conv_layer_1_max\")\n        first_maxpool_layer = tf.squeeze(first_maxpool_layer)\n\n        second_conv_layer = tf.layers.conv1d(inputs=inputs, filters=feature_map_size,\n                                             kernel_size=kernel_2, strides=1, padding='valid',\n                                             activation=tf.nn.relu, use_bias=True, kernel_initializer=l2w_init,\n                                             bias_initializer=tf.zeros_initializer(), name=\"second_conv_layer\")\n\n\n        second_maxpool_layer = tf.reduce_max(second_conv_layer, axis=1, keepdims=True, name=\"conv_layer_2_max\")\n        second_maxpool_layer = tf.squeeze(second_maxpool_layer)\n\n        third_conv_layer = tf.layers.conv1d(inputs=inputs, filters=feature_map_size,\n                                            kernel_size=kernel_3, strides=1, padding='valid',\n                                            activation=tf.nn.relu, use_bias=True, kernel_initializer=l3w_init,\n                                            bias_initializer=tf.zeros_initializer(), name=\"third_conv_layer\")\n\n        third_maxpool_layer = tf.reduce_max(third_conv_layer, axis=1, keepdims=True, name=\"conv_layer_3_max\")\n        third_maxpool_layer = tf.squeeze(third_maxpool_layer)\n\n        output_features_conv = tf.concat(values=[first_maxpool_layer, second_maxpool_layer, third_maxpool_layer], axis = 1,\n                                         name=\"concat_conv\")\n\n        output_features_conv.set_shape(shape=[None,feature_map_size * 3])\n        output_features_conv = tf.nn.dropout(output_features_conv,keep_prob=keep_conv)\n\n    with tf.variable_scope(\"layer_word_hidden_states\"):\n        ((fw_outputs,bw_outputs),\n         _) = (\n            tf.nn.bidirectional_dynamic_rnn(cell_fw=cell_fw,\n                                            cell_bw=cell_bw,\n                                            inputs=inputs,\n                                            sequence_length=batch_sequence_lengths,\n                                            dtype=tf.float32,\n                                            swap_memory=True,\n                                            ))\n    outputs_hidden = tf.concat((fw_outputs, bw_outputs), 2)\n\n    with tf.variable_scope(\"layer_word_attention\"):\n\n        initializer = tf.contrib.layers.xavier_initializer()\n\n        # Big brain #1\n        attention_context_vector = tf.get_variable(name='attention_context_vector',\n                                                   shape=[output_size ],\n                                                   initializer=initializer,\n                                                   dtype=tf.float32)\n\n        input_projection = tf.contrib.layers.fully_connected(outputs_hidden, output_size ,\n                                                  activation_fn=tf.nn.tanh)\n        vector_attn = tf.tensordot(input_projection,attention_context_vector,axes=[[2],[0]],name=\"vector_attn\")\n        attn_softmax = tf.map_fn(lambda batch:\n                               sparse_softmax(batch)\n                               , vector_attn, dtype=tf.float32)\n\n        attn_softmax = tf.expand_dims(input=attn_softmax,axis=2,name='attn_softmax')\n\n        weighted_projection = tf.multiply(outputs_hidden, attn_softmax)\n        outputs = tf.reduce_sum(weighted_projection, axis=1)\n        #outputs = tf.nn.dropout(outputs_1,keep_prob=keep_att_1)\n\n\n    with tf.variable_scope('layer_gather_conv'):\n\n        tf_padded_final_conv = tf.zeros(shape=[1,sent_cutoff_seq,feature_map_size * 3]) # as 3 windows\n\n        sentence_batch_length_2 = tf.placeholder(shape=[None],dtype=tf.int32,name=\"sentence_batch_len_2\")\n\n        i_conv = tf.constant(1)\n\n        #This rolls up sentences dyamically, each sentence-batch-shape at a time.\n        def while_cond_conv (i_conv, tf_padded_final_conv):\n\n            mb_conv = tf.constant(sent_cutoff_seq)\n            return tf.less_equal(i_conv,mb_conv)\n\n        def body_conv(i_conv,tf_padded_final_conv):\n\n            tf_mask_conv = tf.equal(sentence_batch_length_2,i_conv)\n            tf_slice_conv = tf.boolean_mask(output_features_conv,tf_mask_conv,axis=0)\n\n            tf_slice_reshape_conv = tf.reshape(tf_slice_conv,shape=[-1,i_conv,tf_slice_conv.get_shape().as_list()[1]])\n            pad_len_conv = sent_cutoff_seq - i_conv\n\n            tf_slice_padding_conv = [[0,0], [0, pad_len_conv], [0, 0]]\n            tf_slice_padded_conv = tf.pad(tf_slice_reshape_conv, tf_slice_padding_conv, 'CONSTANT')\n\n            tf_padded_final_conv = tf.concat([tf_padded_final_conv,tf_slice_padded_conv],axis=0)\n\n            i_conv = tf.add(i_conv,1)\n\n            return i_conv, tf_padded_final_conv\n\n        _, tf_padded_final_2_conv = \\\n            tf.while_loop(while_cond_conv, body_conv, [i_conv, tf_padded_final_conv],shape_invariants=[i_conv.get_shape(),tf.TensorShape([None,sent_cutoff_seq,feature_map_size * 3])])\n\n    # Give it a haircut\n    tf_padded_final_2_conv = tf_padded_final_2_conv[1:,:]\n\n\n    with tf.variable_scope('layer_gather'):\n\n        tf_padded_final = tf.zeros(shape=[1,sent_cutoff_seq,output_size * 2])\n        tf_y_final = tf.zeros(shape=[1,1],dtype=tf.int32)\n\n        sentence_batch_len = tf.placeholder(shape=[None],dtype=tf.int32,name=\"sentence_batch_len\")\n        sentence_index_offsets = tf.placeholder(shape=[None,2],dtype=tf.int32,name=\"sentence_index_offsets\")\n\n        #sentence_batch_length_2 = tf.placeholder(shape=[None],dtype=tf.int32,name=\"sentence_batch_len_2\")\n        ylen_2 = tf.placeholder(shape=[None],dtype=tf.int32,name=\"ylen_2\")\n\n        i = tf.constant(1)\n\n        # A proud moment =)\n        # Used tensorflow conditionals for the first time!!\n\n        def while_cond (i, tf_padded_final, tf_y_final):\n\n            mb = tf.constant(sent_cutoff_seq)\n            return tf.less_equal(i,mb)\n\n        def body(i,tf_padded_final,tf_y_final):\n\n            tf_mask = tf.equal(sentence_batch_length_2,i)\n            tf_slice = tf.boolean_mask(outputs,tf_mask,axis=0)\n            tf_y_slice = tf.boolean_mask(ylen_2,tf_mask,axis=0) # reshaping the y to fit the data\n\n            tf_slice_reshape = tf.reshape(tf_slice,shape=[-1,i,tf_slice.get_shape().as_list()[1]])\n            tf_y_slice_reshape = tf.reshape(tf_y_slice,shape=[-1,i])\n            tf_y_slice_max = tf.reduce_max(tf_y_slice_reshape,axis=1,keep_dims=True) # the elements should be the same across the col\n\n            pad_len = sent_cutoff_seq - i\n\n            tf_slice_padding = [[0,0], [0, pad_len], [0, 0]]\n            tf_slice_padded = tf.pad(tf_slice_reshape, tf_slice_padding, 'CONSTANT')\n\n            tf_padded_final = tf.concat([tf_padded_final,tf_slice_padded],axis=0)\n            tf_y_final = tf.concat([tf_y_final,tf_y_slice_max],axis=0)\n\n            i = tf.add(i,1)\n\n            return i, tf_padded_final, tf_y_final\n\n        '''\n        # This is the old way\n        def while_cond (i, tf_padded_final):\n            mb = tf.constant(max_sent_seq_len)\n            return tf.less(i,mb)\n\n        def body(i,tf_padded_final):\n\n            #tf.print(i,[i])\n            end_idx = sentence_index_offsets[i,1]\n            st_idx = sentence_index_offsets[i,0]\n            tf_range = tf.range(start=st_idx,limit=end_idx)\n            pad_len = max_sent_seq_len - sentence_batch_len[i]\n\n            tf_slice = tf.gather(outputs,tf_range)\n            tf_slice_padding = [[0, pad_len], [0, 0]]\n            tf_slice_padded = tf.pad(tf_slice, tf_slice_padding, 'CONSTANT')\n            tf_slice_padded_3D = tf.expand_dims(tf_slice_padded, axis=0)\n\n            tf_padded_final = tf.concat([tf_padded_final,tf_slice_padded_3D],axis=0)\n\n            i = tf.add(i,1)\n\n            return i, tf_padded_final\n        '''\n\n        _, tf_padded_final_2, tf_y_final_2 = tf.while_loop(while_cond, body, [i, tf_padded_final, tf_y_final],shape_invariants=[i.get_shape(),tf.TensorShape([None,sent_cutoff_seq,output_size_sent * 2]),tf.TensorShape([None,1])])\n\n    # Give it a haircut\n    tf_padded_final_2 = tf_padded_final_2[1:,:]\n    tf_y_final_2 = tf_y_final_2[1:,:]\n\n\n    with tf.variable_scope('layer_sentence_conv'):\n\n        '''\n        feature_map_size_sent = 100 # preserve the feature map\n        kernel_1_sent = 2\n        kernel_2_sent = 3\n        #kernel_3_sent = 4\n\n        l1w_init_sent = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n        l2w_init_sent = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n        l3w_init_sent = tf.contrib.layers.xavier_initializer(uniform=True, dtype=tf.float32)\n\n        first_conv_layer_sent = tf.layers.conv1d(inputs=tf_padded_final_2_conv, filters=feature_map_size_sent,\n                                            kernel_size=kernel_1_sent, strides=1, padding='valid',\n                                            activation=tf.nn.relu, use_bias=True, kernel_initializer=l1w_init_sent,\n                                            bias_initializer=tf.zeros_initializer(), name=\"first_conv_layer_sent\")\n\n        first_maxpool_layer_sent = tf.reduce_max(first_conv_layer_sent, axis=1, keepdims=True, name=\"conv_layer_1_max_sent\")\n        first_maxpool_layer_sent = tf.squeeze(first_maxpool_layer_sent)\n\n        second_conv_layer_sent = tf.layers.conv1d(inputs=tf_padded_final_2_conv, filters=feature_map_size_sent,\n                                             kernel_size=kernel_2_sent, strides=1, padding='valid',\n                                             activation=tf.nn.relu, use_bias=True, kernel_initializer=l2w_init_sent,\n                                             bias_initializer=tf.zeros_initializer(), name=\"second_conv_layer_sent\")\n\n        second_maxpool_layer_sent = tf.reduce_max(second_conv_layer_sent, axis=1, keepdims=True, name=\"conv_layer_2_max_sent\")\n        second_maxpool_layer_sent = tf.squeeze(second_maxpool_layer_sent)\n\n        third_conv_layer_sent = tf.layers.conv1d(inputs=tf_padded_final_2_conv, filters=feature_map_size_sent,\n                                                  kernel_size=kernel_3_sent, strides=1, padding='valid',\n                                                  activation=tf.nn.relu, use_bias=True,\n                                                  kernel_initializer=l3w_init_sent,\n                                                  bias_initializer=tf.zeros_initializer(),\n                                                  name=\"third_conv_layer_sent\")\n\n        third_maxpool_layer_sent = tf.reduce_max(third_conv_layer_sent, axis=1, keepdims=True,\n                                                  name=\"conv_layer_3_max_sent\")\n        third_maxpool_layer_sent = tf.squeeze(third_maxpool_layer_sent)\n\n        #output_conv_sent = tf.concat([first_maxpool_layer_sent,second_maxpool_layer_sent, third_maxpool_layer_sent],axis=1,name=\"output_conv_sent\")\n        output_conv_sent_1 = tf.concat([first_maxpool_layer_sent,second_maxpool_layer_sent],axis=1,name=\"output_conv_sent\")\n        output_conv_sent = tf.nn.dropout(output_conv_sent_1,keep_prob=keep_2)\n        '''\n\n        output_conv_sent = tf.reduce_sum(tf_padded_final_2_conv,axis=1,keepdims=True)\n        output_conv_sent = tf.squeeze(output_conv_sent)\n\n\n    with tf.variable_scope('layer_sentence_hidden_states'):\n\n        ((fw_outputs_sent, bw_outputs_sent),\n         _) = (\n            tf.nn.bidirectional_dynamic_rnn(cell_fw=cell_sent_fw,\n                                            cell_bw=cell_sent_bw,\n                                            inputs=tf_padded_final_2,\n                                            sequence_length=sentence_batch_len,\n                                            dtype=tf.float32,\n                                            swap_memory=True,\n                                            ))\n        outputs_hidden_sent = tf.concat((fw_outputs_sent, bw_outputs_sent), 2)\n\n    with tf.variable_scope('layer_sentence_attention'):\n\n        initializer_sent = tf.contrib.layers.xavier_initializer()\n\n        # Big brain #2 (or is this Pinky..?)\n        attention_context_vector_sent = tf.get_variable(name='attention_context_vector_sent',\n                                                   shape=[output_size_sent ],\n                                                   initializer=initializer_sent,\n                                                   dtype=tf.float32)\n\n        input_projection_sent = tf.contrib.layers.fully_connected(outputs_hidden_sent, output_size_sent,\n                                                             activation_fn=tf.nn.tanh)\n        vector_attn_sent = tf.tensordot(input_projection_sent, attention_context_vector_sent, axes=[[2], [0]], name=\"vector_attn_sent\")\n        attn_softmax_sent = tf.map_fn(lambda batch:\n                                 sparse_softmax(batch)\n                                 , vector_attn_sent, dtype=tf.float32)\n\n        attn_softmax_sent = tf.expand_dims(input=attn_softmax_sent, axis=2, name='attn_softmax_sent')\n\n        weighted_projection_sent = tf.multiply(outputs_hidden_sent, attn_softmax_sent)\n        outputs_sent = tf.reduce_sum(weighted_projection_sent, axis=1)\n        \n\n     # Add a fully connected layer for impact\n    with tf.variable_scope('fc_layers'):\n\n        \n\n        # more features !\n        final_output = tf.concat([output_conv_sent, outputs_sent], axis=1, name=\"final_concat\")\n        final_output.set_shape(shape = [None,feature_map_size * 3 + output_size_sent * 2])\n\n        fc1_size = 1024\n        fc1 = tf.contrib.layers.fully_connected(final_output, fc1_size,\n                                                             activation_fn=tf.nn.relu)\n        fc1_drop = tf.nn.dropout(fc1,keep_fc)\n        #fc2_size = 64\n        #fc2 = tf.contrib.layers.fully_connected(fc1, fc2_size,\n                                                #activation_fn=tf.nn.relu)\n\n\n    with tf.variable_scope('layer_classification'):\n\n\n\n        wt_init = tf.contrib.layers.xavier_initializer()\n        #wt = tf.get_variable(name=\"wt\",shape=[output_size * 2 +  feature_map_size * 3 ,1],initializer=wt_init)\n        wt = tf.get_variable(name=\"wt\",shape=[fc1_size ,1],initializer=wt_init)\n        #wt = tf.get_variable(name=\"wt\",shape=[output_size * 2,1],initializer=wt_init)\n        bias = tf.get_variable(name=\"bias\",shape=[1],initializer=tf.zeros_initializer())\n\n\n        #logits = tf.layers.dense(final_output,units = 1 , activation=tf.nn.sigmoid,use_bias=True,kernel_initializer = wt_init)\n        #logits = tf.add(tf.matmul(outputs_sent,wt),bias)\n\n        logits = tf.add(tf.matmul(fc1_drop,wt),bias)\n        logits = tf.squeeze(logits)\n        #logits = tf.squeeze(outputs_sent_2)\n        probs = tf.sigmoid(logits)\n        tf_y_final_2 = tf.squeeze(tf_y_final_2)\n        tf_y_final_2 = tf.to_float(tf_y_final_2)\n\n\n    with tf.variable_scope('cross_entropy'):\n\n        global_step = tf.Variable(0,trainable=False,dtype=tf.int32,name='global_step')\n\n        cross_entropy_mean = tf.nn.weighted_cross_entropy_with_logits(\n            targets=tf_y_final_2, logits=logits, pos_weight=pos_wt)\n        learning_rate_input = tf.placeholder(\n            tf.float32, [], name='learning_rate_input')\n\n        loss = tf.reduce_mean(cross_entropy_mean, name=\"cross_entropy_loss\")\n\n        train_step = tf.train.AdamOptimizer(\n            learning_rate_input).minimize(loss,global_step=global_step)\n\n        #train_step = tf.train.GradientDescentOptimizer(\n        #    learning_rate_input).minimize(loss)\n\n        #train_step = tf.train.AdagradOptimizer(\n        #    learning_rate_input).minimize(loss)\n\n        #train_step = tf.train.MomentumOptimizer(\n        #    learning_rate_input,momentum=0.9).minimize(loss,global_step=global_step)\n\n        predicted_indices = tf.to_int32(tf.greater_equal(logits,0.5))\n        confusion_matrix = tf.confusion_matrix(\n            tf_y_final_2, predicted_indices, num_classes=2, name=\"confusion_matrix\")\n\n\n    return probs, logits, inputs,batch_sequence_lengths, sentence_batch_len, \\\n            sentence_index_offsets,  sentence_batch_length_2, tf_y_final_2, ylen_2, \\\n            learning_rate_input, train_step, confusion_matrix, cross_entropy_mean, \\\n            loss, global_step, predicted_indices, keep_fc, keep_conv\n\n\ndef build_loss_optimizer(logits):\n    # Create the back propagation and training evaluation machinery in the graph.\n    pass\n\n\n\ndef build_session(train_file, glove_file,mmap_loc,chkpoint_dir,train_tensorboard_dir,valid_tensorboard_dir):\n\n    num_epochs = 7000\n    mini_batch_size = 64\n    learning_rate = 0.001\n\n    if (os.path.exists(train_tensorboard_dir)):\n        shutil.rmtree(train_tensorboard_dir)\n    os.mkdir(train_tensorboard_dir)\n\n    if (os.path.exists(valid_tensorboard_dir)):\n        shutil.rmtree(valid_tensorboard_dir)\n    os.mkdir(valid_tensorboard_dir)\n\n\n    # Build the graph and the optimizer and loss\n    with tf.Graph().as_default() as gr:\n        final_probs, logits,  inputs, batch_sequence_lengths, sentence_batch_len,\\\n        sentence_index_offsets, sentence_batch_length_2, tf_y_final_2, ylen_2, \\\n        learning_rate_input, train_step, confusion_matrix, cross_entropy_mean, \\\n        loss, global_step, predicted_indices, keep_fc, keep_conv = \\\n            build_graph(cutoff_seq)\n\n        #ground_truth_input, learning_rate_input, train_step, confusion_matrix, cross_entropy_mean, loss, global_step \\\n        #    = build_loss_optimizer(logits=logits)\n\n    X_train, X_dev, glove_dict = get_train_df_glove_dict(train_file, glove_file,mmap_loc,reload_mmap=True)\n\n    valid_set_shape = X_dev.shape[0]\n    # Open the read mmap\n    mmap = np.memmap(mmap_loc, dtype='float32', mode='r', shape=(cutoff_shape + 2, glove_dim))\n\n    with tf.Session(graph=gr,config=tf.ConfigProto(log_device_placement=False)) as sess:\n\n        sess.run(tf.global_variables_initializer())\n        saver = tf.train.Saver()\n\n        # restore model and continue\n        ckpt = tf.train.get_checkpoint_state(chkpoint_dir)\n        if ckpt and ckpt.model_checkpoint_path:\n                saver.restore(sess, ckpt.model_checkpoint_path)\n\n\n        #train_writer = tf.summary.FileWriter(train_tensorboard_dir, sess.graph)\n        #valid_writer = tf.summary.FileWriter(valid_tensorboard_dir)\n\n        xent_counter = 0\n\n        for i in range(0,num_epochs):\n            \n            if (i == 4000):\n                learning_rate = learning_rate * 0.1\n\n            X_train_0_sample = X_train.loc[X_train['target'] == 0].sample(n=mini_batch_size)\n            X_train_1_sample = X_train.loc[X_train['target'] == 1].sample(n=round(mini_batch_size * 1.0))\n\n            train_sample = pd.concat([X_train_0_sample,X_train_1_sample],axis=0)\n            train_sample = shuffle(train_sample)\n            #train_sample = X_train\n            qn_npy, qn_batch_len,  sentence_len, sentence_batch_train_2, y_len_2 = process_questions(train_sample,glove_dict,mmap)\n            y_train = np.asarray(train_sample.loc[:,'target'])\n\n            ylen2_npy = np.asarray(y_len_2)\n\n\n            # Create a matrix with each row as sentence offsets\n            # That gets used to rollup the flattened sentences into their documents\n            sentence_offsets = np.cumsum(sentence_len)\n            sentence_offsets_2 = np.insert(sentence_offsets,0,0,axis=0)\n            sentence_offsets_3 = np.delete(sentence_offsets_2,sentence_offsets_2.shape[0] - 1)\n            np_offsets_len = np.column_stack([sentence_offsets_3,sentence_offsets])\n\n\n\n            prob, _, train_confusion_matrix, train_loss, y_2, pred_indices = \\\n                sess.run([final_probs,train_step,confusion_matrix, loss, tf_y_final_2 ,predicted_indices], feed_dict = {\n                    inputs : qn_npy,\n                    batch_sequence_lengths : qn_batch_len,\n                    sentence_batch_len : sentence_len,\n                    sentence_index_offsets : np_offsets_len,\n                    learning_rate_input : learning_rate,\n                    keep_fc : 0.5,\n                    keep_conv : 0.5,\n                    \n                    sentence_batch_length_2 : sentence_batch_train_2,\n                    ylen_2 : ylen2_npy\n            })\n\n            xent_counter += 1\n            \n            '''\n            loss_train_summary = tf.Summary(\n                value=[tf.Summary.Value(tag=\"loss_train_summary\", simple_value=train_loss)])\n            train_writer.add_summary(loss_train_summary, xent_counter)\n\n            acc_train_summary = tf.Summary(\n                value=[tf.Summary.Value(tag=\"acc_train_summary\", simple_value=float(true_pos / all_pos))])\n            train_writer.add_summary(acc_train_summary, xent_counter)\n\n            auc_train_summary = tf.Summary(\n                value=[tf.Summary.Value(tag=\"auc_train_summary\", simple_value=train_auc)])\n            train_writer.add_summary(auc_train_summary, xent_counter)\n\n            f1_train_summary = tf.Summary(\n                value=[tf.Summary.Value(tag=\"f1_train_summary\", simple_value=train_f1)])\n            train_writer.add_summary(f1_train_summary, xent_counter)\n            '''\n\n            if (i % 100 == 0):\n\n                #print('Saving checkpoint for epoch:' + str(i))\n                saver.save(sess=sess, save_path=chkpoint_dir + 'quora_insincere_qns.ckpt',\n                           global_step=global_step)\n\n            '''\n            # Validation machinery\n            if (i % 20 == 0):\n\n                valid_conf_matrix = None\n                validation_loss = None\n\n                #print ('Valid set shape')\n                #print (valid_set_shape)\n\n                #for j in range(0,valid_set_shape,mini_batch_size):\n\n                    #valid_sample = X_dev[j:j+mini_batch_size]\n                    #y_valid = valid_sample.loc[:,'target']\n\n                y_valid = X_dev.loc[:, 'target']\n\n                qn_npy_valid, qn_batch_len_valid, sentence_len_valid, sentence_batch_valid_2, y_len_valid_2 = process_questions(X_dev, glove_dict,mmap)\n\n                ylen2_valid_npy = np.asarray(y_len_valid_2)\n\n                sentence_offsets = np.cumsum(sentence_len_valid)\n                sentence_offsets_2 = np.insert(sentence_offsets, 0, 0, axis=0)\n                sentence_offsets_3 = np.delete(sentence_offsets_2, sentence_offsets_2.shape[0] - 1)\n                np_offsets_len = np.column_stack([sentence_offsets_3, sentence_offsets])\n\n\n                valid_prob, conf_matrix, valid_loss, y_2_valid,val_pred_indices = \\\n                    sess.run([final_probs,confusion_matrix, loss,tf_y_final_2, predicted_indices], feed_dict={\n                        inputs: qn_npy_valid,\n                        batch_sequence_lengths: qn_batch_len_valid,\n                        sentence_batch_len: sentence_len_valid,\n                        sentence_index_offsets: np_offsets_len,\n                        #ground_truth_input: y_valid,\n                        sentence_batch_length_2 : sentence_batch_valid_2,\n                        ylen_2 : ylen2_valid_npy\n                    })\n\n                if valid_conf_matrix is None:\n                    valid_conf_matrix = conf_matrix\n                    validation_loss = valid_loss\n                else:\n                    valid_conf_matrix += conf_matrix\n                    validation_loss += valid_loss\n\n                #print ('Validation Conf matrix')\n                #print(valid_conf_matrix)\n                #print ('Validation Loss')\n                #print (validation_loss)\n\n                true_pos = np.sum(np.diag(valid_conf_matrix))\n                all_pos = np.sum(valid_conf_matrix)\n                #print('Valid Accuracy is: ' + str(float(true_pos / all_pos)))\n                #print('Total data points:' + str(all_pos))\n\n                valid_auc = roc_auc_score(y_2_valid, valid_prob,average=\"weighted\")\n                #print('Valid AUC')\n                #print(valid_auc)\n\n                valid_f1 = f1_score(y_2_valid,val_pred_indices,average='weighted')\n                #print('Valid F1')\n                #print(valid_f1)\n\n                loss_valid_summary = tf.Summary(\n                    value=[tf.Summary.Value(tag=\"loss_valid_summary\", simple_value=validation_loss)])\n                valid_writer.add_summary(loss_valid_summary, i / 20)\n\n                acc_valid_summary = tf.Summary(\n                    value=[tf.Summary.Value(tag=\"acc_valid_summary\", simple_value=float(true_pos / all_pos))])\n                valid_writer.add_summary(acc_valid_summary, i / 20)\n\n                auc_valid_summary = tf.Summary(\n                    value=[tf.Summary.Value(tag=\"auc_valid_summary\", simple_value=valid_auc)])\n                valid_writer.add_summary(auc_valid_summary, i / 20)\n\n                f1_valid_summary = tf.Summary(\n                    value=[tf.Summary.Value(tag=\"f1_valid_summary\", simple_value=valid_f1)])\n                valid_writer.add_summary(f1_valid_summary, i / 20)\n            '''\n\n\ndef inference(test_file, glove_file, mmap_loc, chkpoint_dir, out_file):\n\n    test_df = pd.read_csv(test_file,low_memory=False)\n    batch_final = np.zeros((1,2))\n    \n    # 'Normalizing' the ID because the dynamic sentence vector rollups in the graph expect an integer ID and not the kaggle provided hashes\n    test_df['qid_num'] = range(0,len(test_df))\n\n    meta_data = test_df[['qid_num','question_text']]\n    print ('Shape of the meta data')\n    print (meta_data.shape)\n\n    # Need to play around with this, this is unused now..\n    threshold = 0.5\n\n    # Build the graph and the optimizer and loss\n    with tf.Graph().as_default() as gr:\n        final_probs, logits,  inputs, batch_sequence_lengths, sentence_batch_len,\\\n        sentence_index_offsets, sentence_batch_length_2, tf_y_final_2, ylen_2, \\\n        learning_rate_input, train_step, confusion_matrix, cross_entropy_mean, \\\n        loss, global_step, predicted_indices, keep_fc , keep_conv = \\\n            build_graph(cutoff_seq)\n\n    # Does not re-write to the memory map during inference - use the same mmap generated during training\n    _, _, glove_dict = get_train_df_glove_dict(test_file, glove_file, mmap_loc,is_training=False,reload_mmap=False)\n    mmap = np.memmap(mmap_loc, dtype='float32', mode='r', shape=(cutoff_shape + 2, glove_dim))\n\n    test_len = test_df.shape[0]\n    \n    i = 0\n    test_batch = 10000\n\n    with tf.Session(graph=gr,config=tf.ConfigProto(log_device_placement=False)) as sess:\n\n        sess.run(tf.global_variables_initializer())\n        saver = tf.train.Saver()\n\n        # restore model and continue\n        ckpt = tf.train.get_checkpoint_state(chkpoint_dir)\n        if ckpt and ckpt.model_checkpoint_path:\n                saver.restore(sess, ckpt.model_checkpoint_path)\n\n        while (i <= test_len):\n\n            if (i + test_batch > test_len):\n                test_batch = test_len - i\n            else:\n                test_batch = 10000\n\n            test_sample = meta_data[i:i+test_batch]\n\n            # Y_len_2 is just 0s, needed to pump into the graph. Unused otherwise\n            qn_npy, qn_batch_len, sentence_len, sentence_batch_train_2, y_len_2 = process_questions(test_sample, glove_dict,\n                                                                                                mmap,is_training=False)\n            ylen2_npy = np.asarray(y_len_2)\n\n            # Create a matrix with each row as sentence offsets\n            # That gets used to rollup the flattened sentences into their documents\n            sentence_offsets = np.cumsum(sentence_len)\n            sentence_offsets_2 = np.insert(sentence_offsets, 0, 0, axis=0)\n            sentence_offsets_3 = np.delete(sentence_offsets_2, sentence_offsets_2.shape[0] - 1)\n            np_offsets_len = np.column_stack([sentence_offsets_3, sentence_offsets])\n\n            prob, y_2, pred_indices = \\\n                sess.run([final_probs, tf_y_final_2, predicted_indices], feed_dict={\n                    inputs: qn_npy,\n                    batch_sequence_lengths: qn_batch_len,\n                    sentence_batch_len: sentence_len,\n                    sentence_index_offsets: np_offsets_len,\n                    sentence_batch_length_2: sentence_batch_train_2,\n                    ylen_2: ylen2_npy,\n                    keep_fc : 1.0,\n                    keep_conv : 1.0\n                \n                })\n\n            y_2 = np.expand_dims(y_2,axis=1)\n            pred_indices = np.expand_dims(pred_indices,axis=1)\n            pred_indices = pred_indices.astype(int)\n            y_2 = y_2.astype(int)\n\n            batch_hstack = np.hstack((y_2,pred_indices))\n            batch_final = np.concatenate([batch_final,batch_hstack],axis=0)\n            \n            i += test_batch\n\n        batch_final = batch_final[1:,:]\n\n        batch_df = pd.DataFrame(batch_final)\n        batch_df.columns = ['qid_num','prediction']\n\n        batch_merge = batch_df.merge(test_df,on='qid_num')\n        batch_merge.drop('qid_num',axis=1,inplace=True)\n        batch_merge.drop('question_text', axis=1, inplace=True)\n\n        cols = ['qid','prediction']\n        batch_merge = batch_merge.reindex(columns=cols)\n        batch_merge[['prediction']] = batch_merge[['prediction']].astype(int)\n        \n        # Submissions file!\n        batch_merge.to_csv(out_file,index=False)\n\ndef main():\n\n    #chkpoint_dir = '/home/nitin/Desktop/kaggle_data/all/tensorboard/checkpoint/'\n    chkpoint_dir = '/kaggle/working/checkpoint/'\n\n    #out_file = '/home/nitin/Desktop/kaggle_data/all/test_inf.csv'\n    out_file = 'submission.csv'\n\n    #memmap_loc = '/home/nitin/Desktop/kaggle_data/all/memmap_file_embeddings.npy'\n    memmap_loc = 'memmap_file_embeddings.npy'\n\n    #glove_file = '/home/nitin/Desktop/kaggle_data/all/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    glove_file = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    \n    #train_file = '/home/nitin/Desktop/kaggle_data/all/train.csv'\n    #test_data = '/home/nitin/Desktop/kaggle_data/all/test.csv'\n    train_file = '../input/train.csv'\n    test_file = '../input/test.csv'\n    \n    #train_tensorboard_dir = '/home/nitin/Desktop/kaggle_data/all/tensorboard/train/'\n    #valid_tensorboard_dir = '/home/nitin/Desktop/kaggle_data/all/tensorboard/valid/'\n    train_tensorboard_dir = '/kaggle/working/train/'\n    valid_tensorboard_dir = '/kaggle/working/valid/'\n\n    # Does training\n    build_session(train_file,glove_file,memmap_loc,chkpoint_dir,train_tensorboard_dir,valid_tensorboard_dir)\n    \n    # Does inference\n    inference(test_file,glove_file,memmap_loc,chkpoint_dir,out_file)\n\nmain()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}