{"cells":[{"metadata":{"_uuid":"83ffdc6c18f79b55ecf6c797931f4a9e316ed5c6"},"cell_type":"markdown","source":"Based on others'  awesome kernels:\n\nhttps://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings by SRK\n\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings by Dieter\n\nTell me if i missed any."},{"metadata":{"trusted":true,"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport os\nimport time\nimport gc\nimport re\nfrom unidecode import unidecode","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train.shape)\nprint(\"Test shape : \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d336bfef799c16f12f2d02ffa4f3c2eaaf6ef34"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].str.lower()\ntest[\"question_text\"] = test[\"question_text\"].str.lower()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afd570d1160b826a84c4c7af76950fd7a30e4471"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b47c945e6f02c2a9f72aedd72da4681695cb20dd"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b0cf1880df72c47d8a882e6441aaa07dacfd9c"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = None # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use #99.99%\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\n#### test #####(maximum is best )\ntokenizer = Tokenizer(num_words=None, filters='')\n#### test #####\ntokenizer.fit_on_texts(list(train_X))\n\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values\n\nsub = test[['qid']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"259d359c2fd45efd7ccd1a18db69fbb7fe7ad8d2"},"cell_type":"code","source":"del train, test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e110fe2c74167bf02fda50e7cd9cbb897dca57"},"cell_type":"code","source":"EMBEDDING_FILE_GLOVE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE_GLOVE) )\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nnp.random.seed(10)\nword_index = tokenizer.word_index\nmax_features = len(word_index)+1\nembedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7da1175268fbea36b8eac3ca4a18c4f2497f690a"},"cell_type":"code","source":"class AdamW(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,  # decoupled weight decay (1/4)\n                 epsilon=1e-8, decay=0., **kwargs):\n        super(AdamW, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.wd = K.variable(weight_decay, name='weight_decay') # decoupled weight decay (2/4)\n        self.epsilon = epsilon\n        self.initial_decay = decay\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n        wd = self.wd # decoupled weight decay (3/4)\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr *= (1. / (1. + self.decay * K.cast(self.iterations,\n                                                  K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        self.weights = [self.iterations] + ms + vs\n\n        for p, g, m, v in zip(params, grads, ms, vs):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) - lr * wd * p # decoupled weight decay (4/4)\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'weight_decay': float(K.get_value(self.wd)),\n                  'epsilon': self.epsilon}\n        base_config = super(AdamW, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b75ab7890f8d51c008b47d994011f88a35de36a7"},"cell_type":"markdown","source":"max_features = 10000\nembed_size = 300\nmaxlen = 72\nembedding_matrix = np.zeros((max_features, embed_size))"},{"metadata":{"trusted":true,"_uuid":"6d9ba67784a576692b6f38c931e3c012eff539d7"},"cell_type":"code","source":"%%time\nK.clear_session()       \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\nx = SpatialDropout1D(rate=0.2)(x)\nx = Bidirectional(CuDNNLSTM(128, return_sequences=True, \n                            kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)\n\n# 1D convolutions that can iterate over the word vectors\nx1 = Conv1D(filters=96, kernel_size=1, padding='same', activation='relu', kernel_initializer=glorot_normal(1000))(x)\nx2 = Conv1D(filters=72, kernel_size=2, padding='same', activation='relu', kernel_initializer=glorot_normal(2000))(x)\nx3 = Conv1D(filters=48, kernel_size=3, padding='same', activation='relu', kernel_initializer=glorot_normal(3000))(x)\nx4 = Conv1D(filters=24, kernel_size=5, padding='same', activation='relu', kernel_initializer=glorot_normal(4000))(x)\n\nx1 = GlobalMaxPool1D()(x1)\nx2 = GlobalMaxPool1D()(x2)\nx3 = GlobalMaxPool1D()(x3)\nx4 = GlobalMaxPool1D()(x4)\n\nmerge1 = concatenate([x1, x2, x3, x4])\n#x = Dropout(0.22)(merge1)\nx = Dense(200, activation=\"relu\", kernel_initializer=he_uniform(seed=12300))(merge1)\nx = Dropout(0.22)(x)\nx = BatchNormalization()(x)\nx = Dense(200, activation=\"relu\", kernel_initializer=he_uniform(seed=12300))(x)\nx = Dropout(0.22)(x)\nx = BatchNormalization()(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=0.08),)\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d5b550ec41ee26e65e3e72667522b3abc0b4970"},"cell_type":"markdown","source":"#### Noise ####\ndef noise_measurement(train, noise_level):\n    noised_train = train.copy()\n    to_transform = np.random.random(train.shape) < noise_level\n    transform_y = np.random.randint(0, max_features, size=train.shape)\n    noised_train[to_transform] = transform_y[to_transform]\n    return noised_train\n#### Noise ####"},{"metadata":{"trusted":true,"_uuid":"9b187ac6f92b513a925619e1cffb486c71a01955"},"cell_type":"code","source":"#LSTM Epoch 5/7\n# - 338s - loss: 0.1012 - val_loss: 0.1045\n#GRU Epoch 5/7\n# - 342s - loss: 0.1032 - val_loss: 0.1086\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d97ed2a521ec1bd2fc10c3ec9fdb7c5fc6f571c"},"cell_type":"code","source":"%%time\nfilepath=\"weights_best.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')\n#reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=0.0001, verbose=2)\n#earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=2, mode='auto')\ncallbacks = [checkpoint]\nmodel.fit(train_X, train_y, batch_size=512, epochs=5, \n          validation_data=(val_X, val_y), verbose=2, callbacks=callbacks, \n         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c75b99630279cdaf010878fb5b8d35be77765ad5"},"cell_type":"code","source":"model.load_weights(filepath)\npred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=2)\npred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=2)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_glove_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\npred_test_y = (pred_glove_test_y>best_thresh).astype(int)\nsub['prediction'] = pred_test_y\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69a36d5440b66a95d6e64326188835f3616d7004"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}