{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport tensorflow_hub as hub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a9046a1ec3208fd699b9ec7b17afb1d73a70cb7"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3cf2e50016666c8af74f5d2bed5bf1e2dc12c8"},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b7fd9aa27da06d02e50cd917abc563cef489262"},"cell_type":"code","source":"df['question_length'] = df['question_text'].str.split().apply(len)\ndf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"653b13351fdf4018a29861449edbc962ce5cc0d5"},"cell_type":"code","source":"print(df['question_text'][0])\nprint(df['question_length'][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d1e4053c4a1bd1d17d3f529e05705c067322e6b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, test_df = train_test_split(df, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fc2d1d92ca8e896ca61933e2e644cdfa3dc52e8"},"cell_type":"code","source":"print(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d51e261a09986ba8f883f104304bd7ebb4fc3bcd"},"cell_type":"code","source":"from gensim.models import KeyedVectors\nEMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nword2vec = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35d8bb4186e1725f52498d1684d8969ca799eabf"},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH=30\nWORD_EMBEDDING_LENGTH=300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eea7eb16e2e2e049159b31d6a6731b152866fd0"},"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.layers import Embedding, Dense, LSTM, Dropout, Conv2D, Flatten, MaxPool2D, Reshape, BatchNormalization, CuDNNLSTM, Bidirectional, MaxPool1D\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.activations import relu, sigmoid\nfrom tensorflow import keras as keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"163ef5ad5eb02cd0df9f0e61f14626a345b1a7d8"},"cell_type":"code","source":"tokenizer = Tokenizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a95a2dafab8ae4195ab5915665bd2f14bef02cb"},"cell_type":"code","source":"tokenizer.fit_on_texts(df['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9726966f36ca3e3639a2e895943a0b69e6af56f"},"cell_type":"code","source":"word_index = tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1737784f0130b2ab5e52caebe8191ddb13c06738"},"cell_type":"code","source":"embedding_matrix = np.zeros((len(word_index)+1, WORD_EMBEDDING_LENGTH))\nfor word, i in word_index.items():\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n#del word2vec\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bc6485243a67f1957913c88d19be8359ea7c77e"},"cell_type":"code","source":"embedding_layer = Embedding(len(word_index) + 1,\n                            WORD_EMBEDDING_LENGTH,\n                            weights=[embedding_matrix],\n                            input_length=MAX_SEQUENCE_LENGTH,\n                            trainable=False, \n                            mask_zero=True,\n                           name=\"embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84204577fabf1e794dd9b6281254a7641862b695"},"cell_type":"code","source":"# https://github.com/aravindsiv/dan_qa/blob/master/custom_layers.py\n\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras import backend as K\nimport tensorflow as tf\n\nclass AverageWords(Layer):\n    def __init__(self):\n        super(AverageWords,self).__init__()\n        self.supports_masking = True\n\n    def call(self, x, mask=None):\n        axis = K.ndim(x) - 2\n        if mask is not None:\n            summed = K.sum(x, axis=axis)\n            n_words = K.expand_dims(K.sum(K.cast(mask, 'float32'), axis=axis), axis)\n            return summed / n_words\n        else:\n            return K.mean(x, axis=axis)\n\n    def compute_mask(self, inputs, mask=None):\n        return None\n\n    def compute_output_shape(self, input_shape):\n        dimensions = list(input_shape)\n        n_dimensions = len(input_shape)\n        del dimensions[n_dimensions - 2]\n        return tuple(dimensions)\n\nclass WordDropout(Layer):\n    def __init__(self, rate):\n        super(WordDropout,self).__init__()\n        self.rate = min(1., max(0., rate))\n        self.supports_masking = True\n\n    def call(self, inputs, training=None, input_mask=None):\n        if 0. < self.rate < 1.0:\n            def dropped_inputs():\n                input_shape = K.shape(inputs)\n                batch_size = input_shape[0]\n                n_time_steps = input_shape[1]\n                mask = tf.random_uniform((batch_size, n_time_steps, 1)) >= self.rate\n                w_drop = K.cast(mask, 'float32') * inputs\n                if input_mask is not None:\n                    w_drop = input_mask*inputs\n                return w_drop\n            return K.in_train_phase(dropped_inputs, inputs, training=training)\n        return inputs\n\n    def get_config(self):\n        config = {'rate': self.rate}\n        base_config = super().get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n    \n\nclass ConsumeMask(Layer):\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"506efeec48d08a965078ab7e91c2d2485ce7d61c"},"cell_type":"code","source":"# https://www.aclweb.org/anthology/D14-1181\ndrop_out = 0.5\nword_drop = 0.3\nfilters = [3,4,5]\n\ninput1 = keras.Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"input\")\nembedding = embedding_layer(input1)\nword_drop = WordDropout(word_drop)(embedding)\nremove_mask = ConsumeMask()(word_drop)\nreshape = Reshape((MAX_SEQUENCE_LENGTH, WORD_EMBEDDING_LENGTH, 1), name='reshape')(remove_mask)\ncnn_layers = []\nfor f in filters:\n    conv = Conv2D(\n        100,\n        (f, WORD_EMBEDDING_LENGTH),\n        name=\"%s_conv\" % f,\n        input_shape=(None, MAX_SEQUENCE_LENGTH, WORD_EMBEDDING_LENGTH, 1))(reshape)\n    \n    cnn_layers.append(MaxPool2D((MAX_SEQUENCE_LENGTH + 1 - f, 1), name='%s_max' % f)(conv))\nconcat1 = keras.layers.concatenate(cnn_layers, axis=1, name=\"concat1\")\nflatten = Flatten(name=\"flatten\")(concat1)\navg = AverageWords()(word_drop)\nconcat2 = keras.layers.concatenate([flatten, avg], axis=1, name=\"concat2\")\n#drop1 = Dropout(drop_out, name=\"drop1\")(concat2)\ndense1 = Dense(512, activation=relu, name=\"dense1\")(concat2)\ndrop2 = Dropout(drop_out, name=\"drop2\")(dense1)\noutput = Dense(1, activation='sigmoid', name=\"output\")(drop2)\n\nmodel = keras.Model(inputs=input1, outputs=output)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aaf2b4eff578bd5b5a777dc7b60c7a1d933a7db9"},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef precision(y_true, y_pred):\n    \"\"\"Precision metric.\n    Only computes a batch-wise average of precision.\n    Computes the precision, a metric for multi-label classification of\n    how many selected items are relevant.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef recall(y_true, y_pred):\n    \"\"\"Recall metric.\n    Only computes a batch-wise average of recall.\n    Computes the recall, a metric for multi-label classification of\n    how many relevant items are selected.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\n\ndef f1(y_true, y_pred):\n    y_pred = K.round(y_pred)\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1 - K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5933cf636f35fe8a466cc50656fbbcf76b084521"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['acc', precision, recall, f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0996eeef1ba31c3da363e19751d75b3e0e26f6eb"},"cell_type":"code","source":"def process_text(array, tokenizer):\n    seq = tokenizer.texts_to_sequences(array)\n    return pad_sequences(seq, maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c8c425a097a25a5289848ea8ebdcade09941249"},"cell_type":"code","source":"import math\ndef batch_gen(train_df, batch_size=128):\n    while True:\n        n_batches = math.ceil(len(train_df) / batch_size)\n        for i in range(n_batches):\n            x = np.array(\n                process_text(\n                    train_df['question_text'][i*batch_size:(i+1)*batch_size].values,\n                    tokenizer)\n            )\n            yield x, np.array(train_df[\"target\"][i*batch_size:(i+1)*batch_size].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dc5a2a39d602f982364ad91e06ab981458abef5"},"cell_type":"code","source":"import math\ndef batch_gen_lang_model(train_df, batch_size=128):\n    while True:\n        n_batches = math.ceil(len(train_df) / batch_size)\n        for i in range(n_batches):\n            x = np.array(\n                process_text(\n                    train_df['question_text'][i*batch_size:(i+1)*batch_size].values,\n                    tokenizer)\n            )\n            y = []\n            for sample in train_df['question_text'][i*batch_size:(i+1)*batch_size].values:\n                y_sentence = []\n                for word in sample.split():\n                    y_sentence.append(word2vec.word_vec(word))\n                y_sentence+= [0*WORD_EMBEDDING_LENGTH]*(MAX_SEQUENCE_LENGTH - len(sample))\n                y.append(y_sentence)\n            y = np.array(y)\n            yield x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f0e5ae03f28f6b5724dbedf7620d05efbe89a38","scrolled":true},"cell_type":"code","source":"batch_size = 512\n\nx_v, y_v = np.array(process_text(test_df['question_text'].values,tokenizer)), np.array(test_df[\"target\"].values)\n\nloss = model.fit_generator(\n    batch_gen(train_df, batch_size),\n    epochs=3,\n    steps_per_epoch=2000,\n    validation_data=(x_v, y_v),#batch_gen(train_df, batch_size),\n    #validation_steps=2000,\n    shuffle=False,\n    use_multiprocessing=False\n).history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4a24732b4bdf8fbde04ef343239343aeea48e57"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nmetric = 'f1'\n\nplt.plot(\n    range(len(loss[metric])),loss[metric],\n    range(len(loss['val_' + metric])), loss['val_' + metric]\n)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a4d651f81d28718525f7aa5b3307b14462590c0"},"cell_type":"code","source":"\npred = model.predict(\n    x=process_text(np.array(train_df['question_text'].head(100).values), tokenizer)\n)\npred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c7744bb7a14c28e856e0d475f8a5aeb367342d3"},"cell_type":"code","source":"result = model.evaluate(\n    x=process_text(np.array(test_df['question_text'].values), tokenizer),\n    y=np.array(test_df['target'].values),\n    batch_size=128)\nresult","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c320ae69492219cf0ff13b9e4a4d338ac97e279c"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')\ndf_test.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46e1bd8d2649a964f671f5b698b0f8b6b7ec5a71"},"cell_type":"code","source":"test_x = process_text(np.array(df_test['question_text'].values), tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bfb3768b670388d96728f40f6840cfa2fa0c06d"},"cell_type":"code","source":"predictions_list = model.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78c17a869ec9b1ea40cae3323f5f05e3bb47db1c"},"cell_type":"code","source":"df_sub = df_test.drop('question_text', axis=1)\ndf_sub.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c674b182b6530235af22e88fc973eb74cf91707"},"cell_type":"code","source":"df_sub['prediction'] = np.array(predictions_list)\ndf_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4efc9296108e7f55b3c9af2a67ebe0d10c5bbf42"},"cell_type":"code","source":"df_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fee61f968149a83aaf6f3adba841c95bbb45d0b0"},"cell_type":"code","source":"df_sub['prediction'] = df_sub['prediction'].round(0).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d7e23a138c7a17332a4c233a1b778cb6af79fe2"},"cell_type":"code","source":"df_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c91d193af58f6262829a450db66213a44515cf19"},"cell_type":"code","source":"df_sub.loc[df_sub['prediction'] == 1].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8543fe703502cae6b49f9debbf65609a42f9a55"},"cell_type":"code","source":"df_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}