# BASED ON https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings import os import time import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) from tqdm import tqdm import math from sklearn import metrics from keras import backend as K from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, GlobalAvgPool1D, concatenate from keras.layers import Bidirectional, GlobalMaxPool1D from keras.models import Model from keras import initializers, regularizers, constraints, optimizers, layers from keras.optimizers import Optimizer from keras.callbacks import * import tensorflow as tf train_df = pd.read_csv("../input/train.csv") test_df = pd.read_csv("../input/test.csv") ## Parameters embed_size = 300 # how big is each word vector max_features = 55000 # how many unique words to use (i.e num rows in embedding vector) maxlen = 27 # max number of words in a question to use batch_size = 10000 drop_ratio = 0.13 opt = 'adam' ## fill up the missing values train_X = train_df["question_text"].fillna("_na_").values test_X = test_df["question_text"].fillna("_na_").values ## Tokenize the sentences tokenizer = Tokenizer(num_words=max_features) tokenizer.fit_on_texts(list(train_X)) train_X = tokenizer.texts_to_sequences(train_X) test_X = tokenizer.texts_to_sequences(test_X) ## Pad the sentences train_X = pad_sequences(train_X, maxlen=maxlen) test_X = pad_sequences(test_X, maxlen=maxlen) ## Get the target values train_y = train_df['target'].values class CyclicLR(Callback): """This callback implements a cyclical learning rate policy (CLR). The method cycles the learning rate between two boundaries with some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186). The amplitude of the cycle can be scaled on a per-iteration or per-cycle basis. This class has three built-in policies, as put forth in the paper. "triangular": A basic triangular cycle w/ no amplitude scaling. "triangular2": A basic triangular cycle that scales initial amplitude by half each cycle. "exp_range": A cycle that scales initial amplitude by gamma**(cycle iterations) at each cycle iteration. For more detail, please see paper. # Example ```python clr = CyclicLR(base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular') model.fit(X_train, Y_train, callbacks=[clr]) ``` Class also supports custom scaling functions: ```python clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.)) clr = CyclicLR(base_lr=0.001, max_lr=0.006, step_size=2000., scale_fn=clr_fn, scale_mode='cycle') model.fit(X_train, Y_train, callbacks=[clr]) ``` # Arguments base_lr: initial learning rate which is the lower boundary in the cycle. max_lr: upper boundary in the cycle. Functionally, it defines the cycle amplitude (max_lr - base_lr). The lr at any cycle is the sum of base_lr and some scaling of the amplitude; therefore max_lr may not actually be reached depending on scaling function. step_size: number of training iterations per half cycle. Authors suggest setting step_size 2-8 x training iterations in epoch. mode: one of {triangular, triangular2, exp_range}. Default 'triangular'. Values correspond to policies detailed above. If scale_fn is not None, this argument is ignored. gamma: constant in 'exp_range' scaling function: gamma**(cycle iterations) scale_fn: Custom scaling policy defined by a single argument lambda function, where 0 <= scale_fn(x) <= 1 for all x >= 0. mode paramater is ignored scale_mode: {'cycle', 'iterations'}. Defines whether scale_fn is evaluated on cycle number or cycle iterations (training iterations since start of cycle). Default is 'cycle'. """ def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular', gamma=1., scale_fn=None, scale_mode='cycle'): super(CyclicLR, self).__init__() self.base_lr = base_lr self.max_lr = max_lr self.step_size = step_size self.mode = mode self.gamma = gamma if scale_fn == None: if self.mode == 'triangular': self.scale_fn = lambda x: 1. self.scale_mode = 'cycle' elif self.mode == 'triangular2': self.scale_fn = lambda x: 1/(2.**(x-1)) self.scale_mode = 'cycle' elif self.mode == 'exp_range': self.scale_fn = lambda x: gamma**(x) self.scale_mode = 'iterations' else: self.scale_fn = scale_fn self.scale_mode = scale_mode self.clr_iterations = 0. self.trn_iterations = 0. self.history = {} self._reset() def _reset(self, new_base_lr=None, new_max_lr=None, new_step_size=None): """Resets cycle iterations. Optional boundary/step size adjustment. """ if new_base_lr != None: self.base_lr = new_base_lr if new_max_lr != None: self.max_lr = new_max_lr if new_step_size != None: self.step_size = new_step_size self.clr_iterations = 0. def clr(self): cycle = np.floor(1+self.clr_iterations/(2*self.step_size)) x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1) if self.scale_mode == 'cycle': return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle) else: return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations) def on_train_begin(self, logs={}): logs = logs or {} if self.clr_iterations == 0: K.set_value(self.model.optimizer.lr, self.base_lr) else: K.set_value(self.model.optimizer.lr, self.clr()) def on_batch_end(self, epoch, logs=None): logs = logs or {} self.trn_iterations += 1 self.clr_iterations += 1 self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr)) self.history.setdefault('iterations', []).append(self.trn_iterations) for k, v in logs.items(): self.history.setdefault(k, []).append(v) K.set_value(self.model.optimizer.lr, self.clr()) def f1(y_true, y_pred): ''' metric from here https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras ''' def recall(y_true, y_pred): """Recall metric. Only computes a batch-wise average of recall. Computes the recall, a metric for multi-label classification of how many relevant items are selected. """ true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + K.epsilon()) return recall def precision(y_true, y_pred): """Precision metric. Only computes a batch-wise average of precision. Computes the precision, a metric for multi-label classification of how many selected items are relevant. """ true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) return precision precision = precision(y_true, y_pred) recall = recall(y_true, y_pred) return 2*((precision*recall)/(precision+recall+K.epsilon())) clr = CyclicLR(base_lr=0.001, max_lr=0.007, step_size=300., mode='exp_range', gamma=0.99994) inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size)(inp) x = Bidirectional(CuDNNGRU(200, return_sequences=True))(x) x1 = GlobalMaxPool1D()(x) x2 = GlobalAvgPool1D()(x) x = concatenate([x1,x2]) x = Dense(16, activation="elu")(x) x = Dropout(drop_ratio)(x) x = Dense(1, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1]) model.fit(train_X, train_y, batch_size=batch_size, epochs=2, callbacks = [clr,]) pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1) del model, inp, x import gc; gc.collect() time.sleep(10) #### Glove EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt' def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)) all_embs = np.stack(embeddings_index.values()) emb_mean,emb_std = all_embs.mean(), all_embs.std() embed_size = all_embs.shape[1] word_index = tokenizer.word_index nb_words = min(max_features, len(word_index)) embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size)) for word, i in word_index.items(): if i >= max_features: continue embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp) x = Bidirectional(CuDNNGRU(200, return_sequences=True))(x) x1 = GlobalMaxPool1D()(x) x2 = GlobalAvgPool1D()(x) x = concatenate([x1,x2]) x = Dense(16, activation="elu")(x) x = Dropout(drop_ratio)(x) x = Dense(1, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1]) model.fit(train_X, train_y, batch_size=batch_size, epochs=2, callbacks = [clr,]) pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1) del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x import gc; gc.collect() time.sleep(10) #### WIKI EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec' def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE) if len(o)>100) all_embs = np.stack(embeddings_index.values()) emb_mean,emb_std = all_embs.mean(), all_embs.std() embed_size = all_embs.shape[1] word_index = tokenizer.word_index nb_words = min(max_features, len(word_index)) embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size)) for word, i in word_index.items(): if i >= max_features: continue embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp) x = Bidirectional(CuDNNGRU(200, return_sequences=True))(x) x1 = GlobalMaxPool1D()(x) x2 = GlobalAvgPool1D()(x) x = concatenate([x1,x2]) x = Dense(16, activation="elu")(x) x = Dropout(drop_ratio)(x) x = Dense(1, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1]) model.fit(train_X, train_y, batch_size=batch_size, epochs=2, callbacks = [clr,]) pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1) del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x import gc; gc.collect() time.sleep(10) ### Paragram Embeddings: EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore') if len(o)>100) all_embs = np.stack(embeddings_index.values()) emb_mean,emb_std = all_embs.mean(), all_embs.std() embed_size = all_embs.shape[1] word_index = tokenizer.word_index nb_words = min(max_features, len(word_index)) embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size)) for word, i in word_index.items(): if i >= max_features: continue embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp) x = Bidirectional(CuDNNGRU(200, return_sequences=True))(x) x1 = GlobalMaxPool1D()(x) x2 = GlobalAvgPool1D()(x) x = concatenate([x1,x2]) x = Dense(16, activation="elu")(x) x = Dropout(drop_ratio)(x) x = Dense(1, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1]) model.fit(train_X, train_y, batch_size=batch_size, epochs=2, callbacks = [clr,]) pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1) del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x import gc; gc.collect() time.sleep(10) #### pred_test_y = 0.32*pred_glove_test_y + 0.32*pred_fasttext_test_y + 0.32*pred_paragram_test_y + 0.04*pred_noemb_test_y pred_test_y = (pred_test_y>0.31).astype(int) out_df = pd.DataFrame({"qid":test_df["qid"].values}) out_df['prediction'] = pred_test_y out_df.to_csv("submission.csv", index=False)