{"cells":[{"metadata":{"_uuid":"693859c710488a003fb927654efe653d3a8f8a06"},"cell_type":"markdown","source":"some reference:\n\nhttps://www.kaggle.com/shujian/single-rnn-with-4-folds-clr by shujian\n\nhttps://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings by SRK\n\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings by Dieter\n\nhttps://www.kaggle.com/shujian/mix-of-nn-models-based-on-meta-embedding by shujian\n\n\nTell me if missed any"},{"metadata":{"trusted":true,"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train.shape)\nprint(\"Test shape : \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d336bfef799c16f12f2d02ffa4f3c2eaaf6ef34"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].str.lower()\ntest[\"question_text\"] = test[\"question_text\"].str.lower()\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\n\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b0cf1880df72c47d8a882e6441aaa07dacfd9c"},"cell_type":"code","source":"## some config values \nembed_size = 300 # how big is each word vector\nmax_features = None # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use #99.99%\n\n## fill up the missing values\nX = train[\"question_text\"].fillna(\"_na_\").values\nX_test = test[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features, filters='')\ntokenizer.fit_on_texts(list(X))\n\nX = tokenizer.texts_to_sequences(X)\nX_test = tokenizer.texts_to_sequences(X_test)\n\n## Pad the sentences \nX = pad_sequences(X, maxlen=maxlen)\nX_test = pad_sequences(X_test, maxlen=maxlen)\n\n## Get the target values\nY = train['target'].values\n\nsub = test[['qid']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"259d359c2fd45efd7ccd1a18db69fbb7fe7ad8d2"},"cell_type":"code","source":"del train, test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e110fe2c74167bf02fda50e7cd9cbb897dca57"},"cell_type":"code","source":"word_index = tokenizer.word_index\nmax_features = len(word_index)+1\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if o.split(\" \")[0] in word_index)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100 and o.split(\" \")[0] in word_index )\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100 and o.split(\" \")[0] in word_index)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    \n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2505794d8ebc30884ccc6a9a9a17a3a1af12bd29"},"cell_type":"code","source":"embedding_matrix_1 = load_glove(word_index)\n#embedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)\nembedding_matrix = np.mean((embedding_matrix_1, embedding_matrix_3), axis=0)  \ndel embedding_matrix_1, embedding_matrix_3\ngc.collect()\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b75ab7890f8d51c008b47d994011f88a35de36a7","trusted":true},"cell_type":"code","source":"def squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n# A Capsule Implement with Pure Keras\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d9ba67784a576692b6f38c931e3c012eff539d7"},"cell_type":"code","source":"def capsule():\n    K.clear_session()       \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(rate=0.2)(x)\n    x = Bidirectional(CuDNNGRU(100, return_sequences=True, \n                                kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)\n\n    x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n    x = Flatten()(x)\n\n    x = Dense(100, activation=\"relu\", kernel_initializer=glorot_normal(seed=12300))(x)\n    x = Dropout(0.12)(x)\n    x = BatchNormalization()(x)\n\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(),)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5a8d1d9ceff06d19f32ccefa62c601d08a4fd0c"},"cell_type":"code","source":"def f1_smart(y_true, y_pred):\n    args = np.argsort(y_pred)\n    tp = y_true.sum()\n    fs = (tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)\n    res_idx = np.argmax(fs)\n    return 2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d97ed2a521ec1bd2fc10c3ec9fdb7c5fc6f571c"},"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)\nbestscore = []\ny_test = np.zeros((X_test.shape[0], ))\nfor i, (train_index, valid_index) in enumerate(kfold.split(X, Y)):\n    X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]\n    filepath=\"weights_best.h5\"\n    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)\n    earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')\n    callbacks = [checkpoint, reduce_lr]\n    model = capsule()\n    if i == 0:print(model.summary()) \n    model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks, \n             )\n    model.load_weights(filepath)\n    y_pred = model.predict([X_val], batch_size=1024, verbose=2)\n    y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2))/5\n    f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))\n    print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))\n    bestscore.append(threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c75b99630279cdaf010878fb5b8d35be77765ad5"},"cell_type":"code","source":"y_test = y_test.reshape((-1, 1))\npred_test_y = (y_test>np.mean(bestscore)).astype(int)\nsub['prediction'] = pred_test_y\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}