{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e75f411622a6f50985288b175fdd6ca0b0e568be"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\nX_train = train_df[\"question_text\"].fillna(\"dieter\").values\ntest_df = pd.read_csv(\"../input/test.csv\")\nX_test = test_df[\"question_text\"].fillna(\"dieter\").values\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f0073b11059320d40c0c8a85c8711c2b87564be"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4545308a2a69c6eb34b109e6a301adc45ff13d6"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn import metrics\n\nfrom keras.preprocessing import text, sequence\nfrom keras.layers import *\nfrom keras.layers import ReLU\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02ae95e055e90b20e0f96850c7c2b1851e8f4302"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"484150ec3bab41a0151b881828622a1891d73c7b"},"cell_type":"markdown","source":"## **Data cleaning**"},{"metadata":{"trusted":true,"_uuid":"7f8bf6c6f1f2a1b175fdc82f2d13b39da6cc7e12"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58ea516ef61192c9c8932a5882a8d3b383b109ca"},"cell_type":"code","source":"mispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59ea621a7e5c993926f3a6df889bfe3b081f3fd3"},"cell_type":"code","source":"def clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = str(x)\n    x = re.sub(r'[0-9]{5,}', '#####', x)\n    x = re.sub(r'[0-9]{4}', '####', x)\n    x = re.sub(r'[0-9]{3}', '###', x)\n    x = re.sub(r'[0-9]{2}', '##', x)\n    \n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    \n    text = mispellings_re.sub(replace, text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d42301e9c7616599bcb4a15ff494469ec1019ee"},"cell_type":"code","source":"#X_train[\"question_text\"] = X_train[\"question_text\"].apply(lambda x: x.lower())\n#X_test[\"question_text\"] = X_test[\"question_text\"].apply(lambda x: x.lower())\n\n#X_train[\"question_text\"] = X_train[\"question_text\"].map(clean_text)\n#X_test[\"question_text\"] = X_test[\"question_text\"].map(clean_text)\n\nX_train[\"question_text\"] = X_train[\"question_text\"].map(clean_numbers)\nX_test[\"question_text\"] = X_test[\"question_text\"].map(clean_numbers)\n\nX_train[\"question_text\"] = X_train[\"question_text\"].map(replace_typical_misspell)\nX_test[\"question_text\"] = X_test[\"question_text\"].map(replace_typical_misspell)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4325950a9a38f79134081b3825371eaa79453b98"},"cell_type":"code","source":"SEQ_LEN = 100  # magic number - length to truncate sequences of words\nmaxlen = 100\nmax_features = 50000\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7df48151fc43cbc61844cf40a830d2d12f263b33"},"cell_type":"code","source":"EMBEDDING_FILE_GLOVE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n\nembeddings_index_glove = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE_GLOVE))\n\nall_embs_glove = np.stack(embeddings_index_glove.values())\nemb_mean_glove,emb_std_glove = -0.005838499,0.48782197\nembed_size_glove = all_embs_glove.shape[1]\n\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_glove = np.random.normal(emb_mean_glove, emb_std_glove, (nb_words, embed_size_glove))\n\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector_glove = embeddings_index_glove.get(word)\n    if embedding_vector_glove is not None: embedding_matrix_glove[i] = embedding_vector_glove","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd5a14a6028a4fdbe802aa16bd115b7764198695"},"cell_type":"code","source":"# Capsule layer\n\ndef squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33e23e0cace9060ced2f024ad4d487efc20e0967"},"cell_type":"code","source":"def capsule():\n    K.clear_session()       \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size_glove, weights=[embedding_matrix_glove], trainable=False)(inp)\n    x = SpatialDropout1D(rate=0.2)(x)\n    x = Bidirectional(CuDNNLSTM(160, return_sequences=True, \n                                kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)\n    x = Capsule(num_capsule=15, dim_capsule=12, routings=8, share_weights=True)(x)\n    x = Flatten()(x)\n    x = Dense(100, activation=None, kernel_initializer=glorot_normal(seed=12300))(x)\n    x = Dropout(0.2)(x)\n    x = BatchNormalization()(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(),)\n    return model\n\nmodel_glove = capsule()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"814c6000f319a3660628f7a13b246b40069a3ea5"},"cell_type":"code","source":"batch_size = 128\nepochs = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf4da3d3f75253c30b74291c33d58450e5c9e759"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a687047fb739aeb682744872e803e04c82f7fbe"},"cell_type":"code","source":"hist_glove = model_glove.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\npred_glove_y = model_glove.predict(x_test, batch_size=1024)\npred_glove_y = (pred_glove_y[:,0] > 0.33).astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7480cb792b32ebdf33e933e21d8d39ad8cc472e6"},"cell_type":"code","source":"submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": pred_glove_y})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}