{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#in V34 we are eliminating the aux input. Correcting the training data.\n\n#Version 31 used Quora competition embedding data. The submit button did not work saying exterior \n# data not allowed. Can do internal analysis using gap-development, which is same as test.\n# Need to change training data source from gap-dev to gap-training. \n\n#Version 32 is the first attempted implementation of IsLayer.\n\nimport os\nimport csv\nimport json\nimport string\nimport keras\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import log_loss\nimport seaborn as sns\ncolor = sns.color_palette()\nfrom math import floor\nimport spacy\nimport zipfile\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\n\nimport time\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, Add, Flatten, CuDNNLSTM\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.engine.topology import Layer\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np # linear algebra\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nfrom wordcloud import WordCloud, STOPWORDS\nfrom collections import defaultdict\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9eb4372a4b591bf1b84973a66c45c4c359f66ad4"},"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm')\ndoc = nlp(u'A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels')\n\nfor token in doc:\n    print(token.text, token.pos_, token.tag_, token.dep_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1fb278795718ac3a298bf27ae34744b0da628af"},"cell_type":"code","source":"def word_locate(sentence, location): \n    count_words = 0\n    count_chars = 2 #2 is to count for the two spaces in the beginning\n    for word in sentence.split():\n        count_words += 1\n        if location == count_chars:\n            return word, count_words\n        count_chars += len(word)\n        count_chars += 1 #for space","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b73b30ec281e0587d5bd230903ab721ae954188d"},"cell_type":"code","source":"def curr_prev_sentence(sentence, loc):\n    current_sentence = \"\"\n    prev_sentence = \"\"\n    detect = 0\n    count = 0\n    for char in sentence:\n        count += 1\n        current_sentence += char\n        if char == \".\" and detect == 0:\n            prev_sentence = current_sentence \n            current_sentence = \"\"\n        if char == \".\" and detect == 1:\n            return current_sentence, prev_sentence\n        if count == loc:\n            detect = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66edc761e807b587a78847e81eb448f4efed644b"},"cell_type":"code","source":"def find_subject(sentence):\n    doc = nlp(sentence)\n    for token in doc:\n        if token.dep_ == \"nsubj\":\n            return token.text\n    return \"none\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9078fe9d5b8755cd23ed857cbb633faf14b8727"},"cell_type":"code","source":"text1 = \"After a few years of almost no work -- although he was a guest star on Lou Grant and Charlie's Angels in the late 1970s, he once summed up the 1970s as 'I cried and did a lot of gardening' -- he was hired in 1979 for his best-known role, self-made millionaire Palmer Cortlandt on ABC's long-running soap opera All My Children. Initially hired for only one year, he remained on contract through 2009. For much of his first decade on the show, Palmer was a ruthless villain, totally possessive of his daughter, Nina and violently threatening his ex-wife Daisy with being attacked by dobermans when she came back from the dead.\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5e707081c46804ce9e4dabfd180372ed7542e6f"},"cell_type":"code","source":"text2 = \"When onlookers expressed doubt, claiming that the Proctor family was well regarded in the community, the girl promptly came out of her trance and told them it was all for 'sport'. On March 29, 1692, Abigail Williams and Mercy Lewis again said they were being tormented by Elizabeth's spectre. A few days later, Abigail complained that Elizabeth was pinching her and tearing at her bowels, and said she saw Elizabeth's spectre as well as John's.\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a295443dede4624b9d965209e79df08991c50b0"},"cell_type":"code","source":"current, prev = curr_prev_sentence(text2, 360)\nprint(current)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc6f870198b7aa091e7f73124a4a72f3dd8c836a"},"cell_type":"code","source":"candidate = find_subject(current)\nprint(candidate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00c8e1b143837f77771231aeb730d2125eea6f7f"},"cell_type":"code","source":"word, loc = word_locate(text2, 360) \nprint(word, \" \", loc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51a6db6e45786dcfdfd9e09fe35d2bd9a7be3efd"},"cell_type":"code","source":"with open('../input/gap-other-training-data/gap-training.tsv') as tsvfile:\n    reader = csv.DictReader(tsvfile, dialect='excel-tab')\n    train_X = []\n    train_y = []\n    tr_label_A_list = []\n    tr_label_B_list = []\n    for row in reader:\n        text = row['Text']\n        text = text.lower()\n        new_textA = text\n        labelA = 0\n        labelB = 0\n        check = 0 #to allign the text in correct location after first insert\n        for char_idx in range(len(text)):\n            if char_idx == int(row['A-offset']):\n                new_textA = new_textA[:char_idx+check] + \"person_loc \" + new_textA[char_idx+check:]\n                check = 11\n            if char_idx == int(row['Pronoun-offset']):\n                new_textA = new_textA[:char_idx+check] + \"pronom_loc \" + new_textA[char_idx+check:]\n                check = 11\n        if row['A-coref'] == 'TRUE':\n            labelA = 1\n        train_X.append(new_textA)\n        train_y.append(labelA)\n        new_textB = text\n        label = 0\n        check = 0 #to allign the text in correct location after first insert\n        for char_idx in range(len(text)):\n            if char_idx == int(row['B-offset']):\n                new_textB = new_textB[:char_idx+check] + \"person_loc \" + new_textB[char_idx+check:]\n                check = 11\n            if char_idx == int(row['Pronoun-offset']):\n                new_textB = new_textB[:char_idx+check] + \"pronom_loc \" + new_textB[char_idx+check:]\n                check = 11\n        if row['B-coref'] == 'TRUE':\n            labelB = 1\n\n        train_X.append(new_textB)\n        train_y.append(labelB)\n        \n        tr_label_A_list.append(labelA)\n        tr_label_B_list.append(labelB)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c63bc70f54fc2ffbb6adbd6bf9e7d51bcddb2b6"},"cell_type":"code","source":"with open('../input/gap-other-training-data/gap-training.tsv') as tsvfile:\n    reader = csv.DictReader(tsvfile, dialect='excel-tab')\n    testA_X = []\n    testA_y = []\n    testB_X = []\n    testB_y = []\n    label_A_list = []\n    label_B_list = []\n    for row in reader:\n        text = row['Text']\n        text = text.lower()\n        new_textA = text\n        labelA = 0\n        labelB = 0\n        check = 0 #to allign the text in correct location after first insert\n        for char_idx in range(len(text)):\n            if char_idx == int(row['A-offset']):\n                new_textA = new_textA[:char_idx+check] + \"person_loc \" + new_textA[char_idx+check:]\n                check = 11\n            if char_idx == int(row['Pronoun-offset']):\n                new_textA = new_textA[:char_idx+check] + \"pronom_loc \" + new_textA[char_idx+check:]\n                check = 11\n        if row['A-coref'] == 'TRUE':\n            labelA = 1\n        testA_X.append(new_textA)\n        testA_y.append(labelA)\n        new_textB = text\n        label = 0\n        check = 0 #to allign the text in correct location after first insert\n        for char_idx in range(len(text)):\n            if char_idx == int(row['B-offset']):\n                new_textB = new_textB[:char_idx+check] + \"person_loc \" + new_textB[char_idx+check:]\n                check = 11\n            if char_idx == int(row['Pronoun-offset']):\n                new_textB = new_textB[:char_idx+check] + \"pronom_loc \" + new_textB[char_idx+check:]\n                check = 11\n        if row['B-coref'] == 'TRUE':\n            labelB = 1\n\n        testB_X.append(new_textB)\n        testB_y.append(labelB)\n        \n        label_A_list.append(labelA)\n        label_B_list.append(labelB)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c678ed34eb5d77e7b0de04943724a7aa87d7a1b"},"cell_type":"code","source":"print(len(train_X))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cea25a36ac6a1788e9940f5e697d392b525569e9"},"cell_type":"code","source":"maxlen = 120\nembed_size = 300\nmax_features = 5000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a06bf47e120dfaebc313916c128b4674a3fcc218"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer_list = list(train_X)\ntokenizer.fit_on_texts(tokenizer_list)\n\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntestA_X = tokenizer.texts_to_sequences(testA_X)\ntestB_X = tokenizer.texts_to_sequences(testB_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ntestA_X = pad_sequences(testA_X, maxlen=maxlen)\ntestB_X = pad_sequences(testB_X, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"707cf46fdcb1264f5070e994432804e27915942c"},"cell_type":"code","source":"word_index = tokenizer.word_index\nmax_features = len(word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"953adbafc496dcbe7d536dda22ce6cffb41d4bb0"},"cell_type":"code","source":"!wget http://nlp.stanford.edu/data/glove.840B.300d.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1ba0b32643c09348cd25d83c055af43f84713df"},"cell_type":"code","source":"with zipfile.ZipFile(\"glove.840B.300d.zip\",\"r\") as zip_ref:\n    zip_ref.extractall()\n!ls 'cased_glove_300'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e53d126512adb70789b24d92994e206a8a51a746"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if o.split(\" \")[0] in word_index)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if word == \"pronom_loc\":\n            embedding_vector = np.ones(300)*0.85 # the number is arbitrary, just some vector\n        if word == \"person_loc\":\n            embedding_vector = np.ones(300)*0.25 # the number is arbitrary, just some vector\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \n    return embedding_matrix ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29a89ba55a026942a997f713eb79e727c2ab5c88"},"cell_type":"code","source":"def dot_product(x, kernel):\n    \"\"\"\n    Wrapper for dot product operation, in order to be compatible with both\n    Theano and Tensorflow\n    Args:\n        x (): input\n        kernel (): weights\n    Returns:\n    \"\"\"\n    if K.backend() == 'tensorflow':\n        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n    else:\n        return K.dot(x, kernel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d571cbae35966419a19a3e2a7d0763c156f4103"},"cell_type":"code","source":"def dupli_row(x, num_copies, axis):\n    y = x\n    for i in range(num_copies-1):\n        y = np.concatenate((y,x), axis= axis)\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6ee96231564a5a7f756fafd5a9bf9bd4b52dff5"},"cell_type":"code","source":"class IsLayer(Layer):\n    #Layer to be used after a dense one. It will multiply all the elements with each other.\n    #In a sense, it allows the neurons to have a say on  each others' outputs. This layer, hopefully,\n    #compares the relative importance of neurons. A dominant neuron can have more say via its weight.\n    #The idea follows from attention layer, but is more basic than that. As it is multiplicative, it is \n    #an alternative to the vanilla additive layer where outputs are added at the next layer.\n    \n    def __init__(self, **kwargs):\n        super(IsLayer, self).__init__(**kwargs)\n    \n    def build(self, input_shape):\n        #Create a trainable weight variable for this layer.\n        self.W = self.add_weight(name='W', \n                                 shape=(input_shape[1], 1), \n                                 initializer='uniform',\n                                 trainable=True)\n        super(IsLayer, self).build(input_shape)\n        \n    def call(self, x):\n        x_W = K.dot(x, self.W)\n        x_new = x*x_W \n        return x_new\n    \n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa8ebc8290956f118c9863325bffac150d56bf3c"},"cell_type":"code","source":"class AttentionWithContext(Layer):\n    \"\"\"\n    Attention operation, with a context/query vector, for temporal data.\n    Supports Masking.\n    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n    \"Hierarchical Attention Networks for Document Classification\"\n    by using a context vector to assist the attention\n    # Input shape\n        3D tensor with shape: `(samples, steps, features)`.\n    # Output shape\n        2D tensor with shape: `(samples, features)`.\n    How to use:\n    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n    The dimensions are inferred based on the output shape of the RNN.\n    Note: The layer has been tested with Keras 2.0.6\n    Example:\n        model.add(LSTM(64, return_sequences=True))\n        model.add(AttentionWithContext())\n        # next add a Dense layer (for classification/regression) or whatever...\n    \"\"\"\n\n    def __init__(self,\n                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n                 W_constraint=None, u_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n\n\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.u_regularizer = regularizers.get(u_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.u_constraint = constraints.get(u_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        super(AttentionWithContext, self).__init__(**kwargs)\n\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        if self.bias:\n            self.b = self.add_weight((input_shape[-1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n\n        self.u = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_u'.format(self.name),\n                                 regularizer=self.u_regularizer,\n                                 constraint=self.u_constraint)\n\n        super(AttentionWithContext, self).build(input_shape)\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        uit = dot_product(x, self.W)\n\n        if self.bias:\n            uit += self.b\n\n        uit = K.tanh(uit)\n        ait = dot_product(uit, self.u)\n\n        a = K.exp(ait)\n\n        # apply mask after the exp. will be re-normalized next\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            a *= K.cast(mask, K.floatx())\n\n\n        # in some cases especially in the early stages of training the sum may be almost zero\n        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], input_shape[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f06972468cf9492bb1241253808ce6527832f59a"},"cell_type":"code","source":"train_y = np.asarray(train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45f9e21e970f8307e863b1206b31fe788ecea6cb"},"cell_type":"code","source":"embedding_matrix = load_glove(word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bffd67d714d5cb8b75559b444f99a39fdb65b18b"},"cell_type":"code","source":"inp1 = Input(shape=(maxlen,))\n\nmodel1_out = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp1)\nmodel1_out = Bidirectional(CuDNNGRU(256, return_sequences=True))(model1_out)\nmodel1_out = AttentionWithContext()(model1_out)\nmodel1_out = Dropout(0.1)(model1_out)\n\nmerged_out = Dense(32, activation=\"relu\")(model1_out)\nmerged_out = Dropout(0.1)(merged_out)\nmerged_out = Dense(1, activation=\"sigmoid\")(merged_out)\nmodel = Model(inputs=inp1, outputs=merged_out)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adee9dcc23851cf063ff33c3d828d1c8c691a3e4"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=20, validation_data=(train_X, train_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46dfe12b5c7bb75ba8ce76e596b09c66e8beed97"},"cell_type":"code","source":"pred_valA_y = model.predict(testA_X, batch_size=512, verbose=1)\npred_valB_y = model.predict(testB_X, batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcece3985792df7fbbb083965dda054432ee71bd"},"cell_type":"code","source":"print(len(pred_valB_y), \" \", len(pred_valA_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9ddbb1b1af6369b5ed12383977c1ce147d1307f"},"cell_type":"code","source":"print(\"Test score A:\", log_loss(pred_valA_y,testA_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64853cd10da9443dcb5fbf4bf4bda840f29b7021"},"cell_type":"code","source":"print(\"Test score B:\", log_loss(pred_valB_y,testB_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61eb913d49b4bd366ff3449e9f34e2f125beb6ff"},"cell_type":"code","source":"out_df = pd.DataFrame({\"ID\":test_ids})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4994d9544f504fbbb00fef6ed318968023c6c4b8"},"cell_type":"code","source":"out_df['A'] = [max(float(val),0.33) for val in list(pred_valA_y)]\nout_df['B'] = [max(float(val),0.33) for val in list(pred_valB_y)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8f90b9a3d278851927f47404b8353689b0562b1"},"cell_type":"code","source":"out_df['NEITHER'] = [max((1-float(val)),0.33) for val in list(pred_valA_y)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"157c02ca36df5746454359dbe120e410be45519e"},"cell_type":"code","source":"out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}