{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"740fcaa1d69d8a2dc231f9a5df7c45af1f66d19d"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\nfrom scipy.spatial.distance import cdist\nfrom tensorflow.python.keras.models import Sequential\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, SpatialDropout1D\nfrom keras.layers import *\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom tensorflow.python.keras.optimizers import Adam\nfrom tensorflow.python.keras.preprocessing.text import Tokenizer\nfrom tensorflow.python.keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom keras.callbacks import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4614edd904d61aefb8ef0cfc0ba825aa9900e76f","scrolled":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ntext_data = df_train[\"question_text\"].values\ntarget = df_train[\"target\"].values\ndf_test = pd.read_csv(\"../input/test.csv\")\ntest_data = df_test[\"question_text\"].values\nX_train, X_test, y_train, y_test = train_test_split(text_data,target, random_state = 23, test_size=0.0)\nnumWords = 20000\ntokenizer = Tokenizer(num_words = numWords)\ntokenizer.fit_on_texts(text_data)\nx_train_tokens = tokenizer.texts_to_sequences(X_train)\nx_test_tokens = tokenizer.texts_to_sequences(test_data)\n# len_features = [len(x) for x in x_train_tokens]\n# sum(x>50 for x in len_features)\npad = 'pre'\nmaxTokens = 50\nx_train_pad = pad_sequences(x_train_tokens, maxlen=maxTokens, padding=pad,\n                           truncating=pad)\nx_test_pad = pad_sequences(x_test_tokens, maxlen=maxTokens, padding=pad,\n                          truncating = pad)\ny_pred_lst = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"a78e5de1dad8538631beb570e13a122b708edcbb"},"cell_type":"code","source":"# from tqdm import tqdm\n# inverse_transform = dict(zip(tokenizer.word_index.values(),tokenizer.word_index.keys()))\n# def convertTokensToString(tokens):\n#     words = [inverse_transform[token] for token in tokens if token!=0]\n#     return words\n# x_train_tokens_words = [convertTokensToString(x) for x in tqdm(x_train_tokens)]\n# x_test_tokens_words = [convertTokensToString(x) for x in tqdm(x_test_tokens)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef73cddddbdcbe0bd96758fee15f22e420cef337"},"cell_type":"code","source":"def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\ndef generate_embeddings(EMBEDDING_FILE):\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    word_index = tokenizer.word_index\n    nb_words = min(numWords,len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= numWords: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embed_size, embedding_matrix\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b49256087d344a3507a12bbe7247497dc9fbfeb"},"cell_type":"code","source":"def squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n# A Capsule Implement with Pure Keras\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"942ee64229fcf4f4cf64fb7ec90bca2af6d8eb5b","scrolled":true},"cell_type":"code","source":"def build_model():\n    max_features = numWords\n    maxlen=50\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = SpatialDropout1D(rate=0.2)(x)\n    x = Bidirectional(CuDNNGRU(128, return_sequences=True,\n                               recurrent_initializer = orthogonal(gain=1.0,seed = 10000)))(x)\n    x = CuDNNGRU(26, return_sequences=True)(x)\n    x = Capsule(num_capsule = 10, dim_capsule = 10,\n            routings=4,\n            share_weights=True)(x)\n    x = Flatten()(x)\n    x = Dense(70, activation=\"relu\")(x)\n    x = Dropout(0.12)(x)\n    x = BatchNormalization()(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    print(model.summary())\n    model.fit(x_train_pad,y_train,validation_split=0.05,epochs=2,batch_size=512)\n    y_pred = model.predict(x_test_pad, verbose=1)\n    y_pred_training = model.predict(x_train_pad, verbose=1)\n    return (y_pred, y_pred_training)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b46a1e0e2ff6b1307ecc79abbe50cbf1628362c"},"cell_type":"code","source":"import gc\nEMBEDDING_FILES = ['../input/embeddings/glove.840B.300d/glove.840B.300d.txt',\n                   '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',\n                   '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt']\ny_pred_lst=[]\ny_pred_lst_training=[]\nfor EMBEDDING_FILE in EMBEDDING_FILES:\n    embed_size, embedding_matrix = generate_embeddings(EMBEDDING_FILE)\n    y_pred, y_pred_training = build_model()\n    y_pred_lst.append(y_pred)\n    y_pred_lst_training.append(y_pred_training)\n    del embedding_matrix\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0b68623a36d38bd8b8e6ebcf686e64c4207521d"},"cell_type":"code","source":"# y_pred = y_pred_lst[len(y_pred_lst)-1]\n# def get_thresh_f1(y_pred):\n#     f1score_max = 0\n#     thresh_max = 0\n#     for thresh in np.arange(0.1, 0.501, 0.01):\n#         thresh = np.round(thresh, 2)\n#         if(f1score_max<f1_score(y_test, (y_pred>thresh).astype(int))):\n#             f1score_max = f1_score(y_test, (y_pred>thresh).astype(int))\n#             thresh_max = thresh\n#     return f1score_max, thresh_max\n# from sklearn.metrics import accuracy_score, f1_score\n# get_thresh_f1(y_pred)\n# ## (0.6697490870538343, 0.36) cudnn 26\n\n##For final submission\n##{glove : 0.29, wiki: 0.27}\n#y_pred_final= (y_pred>0.36).astype(int).flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3569758d2a12df7b0132a485804dc0da5920215"},"cell_type":"code","source":"train_arr = np.concatenate((y_pred_lst_training[0],y_pred_lst_training[1],y_pred_lst_training[2]), axis=1)\ntest_arr = np.concatenate((y_pred_lst[0],y_pred_lst[1],y_pred_lst[2]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"274f0b930a546d0281391d1362e705f6a2c72860"},"cell_type":"code","source":"from keras.utils import to_categorical\nmodel = Sequential()\nmodel.add(Dense(10, input_dim=3, activation='relu'))\nmodel.add(Dense(4, input_dim=3, activation='relu'))\nmodel.add(Dense(2, activation='softmax'))\n# Compile model\ny_train_categorical = to_categorical(y_train)\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())\nmodel.fit(train_arr,y_train_categorical,epochs=3,batch_size=256)\ny_pred = model.predict(test_arr, verbose=1)\n#10, 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e0af162c45b292ca4a62df477ad6f3128208d0"},"cell_type":"code","source":"y_pred_final = np.argmax(y_pred,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb003f18289cc172cc1ddf3742460707cf432321"},"cell_type":"code","source":"df_submission = pd.DataFrame({\"qid\":df_test[\"qid\"].values, \"prediction\":y_pred_final})\ndf_submission = df_submission[[\"qid\",\"prediction\"]]\ndf_submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d4a5cbe925d4e31fa16b24c3c4e0e3f1277d729"},"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix\n# confusion_matrix(y_test, (y_pred>thresh).astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a746b1de126a5ded283d05c4334052045adadabc"},"cell_type":"code","source":"# y_pred = 0.33*y_pred_gnews + 0.33*y_pred_wiki + 0.33*y_pred_paragram","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a920a6359abd1ec04302b004a36418350117433c"},"cell_type":"code","source":"# max_features = numWords\n# maxlen=50\n# inp = Input(shape=(maxlen,))\n# x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n# x = Bidirectional(CuDNNGRU(14, return_sequences=True))(x)\n# x = CuDNNGRU(12, return_sequences=True)(x)\n# x = GlobalMaxPool1D()(x)\n# x = Dense(16, activation=\"relu\")(x)\n# x = Dropout(0.1)(x)\n# x = Dense(1, activation=\"sigmoid\")(x)\n# model = Model(inputs=inp, outputs=x)\n# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n# print(model.summary())\n# model.fit(x_train_pad,y_train,validation_split=0.05,epochs=2,batch_size=512)\n# y_pred = model.predict(x_test_pad, verbose=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}