from pathlib import Path import numpy as np import pandas as pd import os, gc, time import tensorflow as tf from tensorflow import keras from nltk.stem import PorterStemmer, LancasterStemmer, SnowballStemmer, WordNetLemmatizer from gensim import models import random import time # Any results you write to the current directory are saved as output. RANDOM_STATE = 42 TRAIN_DATA_PATH = '../input/quora-insincere-questions-classification/train.csv' TEST_DATA_PATH = '../input/quora-insincere-questions-classification/test.csv' GLOVE_PATH = Path('../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt') PARAGRAM_PATH = Path('../input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt') WIKI_NEWS_PATH = Path('../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec') GOOGLE_NEWS_PATH = Path('../input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin') LABEL_COLUMNS = ['target'] contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have", "o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" } punct = "/-'?!.,#$%\'()*+-/:;<=>@[\\]^_`{|}~" + '""“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\×™√²—–&' # for adding space before and after in words containing punct punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', } # for glove mispell_mapping = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'} def clean_special_chars(text, punct=punct, mapping=punct_mapping): for p in mapping: text = text.replace(p, mapping[p]) for p in punct: text = text.replace(p, f' {p} ') # f is formatted string literal sifn specials = {'\u200b': ' ', '…': ' ... ', '\ufeff': '', 'करना': '', 'है': ''} # Other special characters for s in specials: text = text.replace(s, specials[s]) return text def correct_spelling(text, mapping=mispell_mapping): for word in mapping.keys(): text = text.replace(word, mapping[word]) return text def load_train_data(file_path, train_column, label_columns): print('Loading train data...') data_frame = pd.read_csv(file_path) return data_frame[train_column].tolist(), data_frame[label_columns].values def load_test_data(file_path, train_column): print('Loading test data...') data_frame = pd.read_csv(file_path) return data_frame[train_column].tolist(), data_frame['qid'] def clean_contractions(text, mapping=contraction_mapping): specials = ["’", "‘", "´", "`"] for s in specials: text = text.replace(s, "'") text_new = [] for t in text.split(' '): if t.lower() in mapping: t_new = mapping[t.lower()] if t.isupper(): t_new = t_new.upper() elif t[0].isupper() and len(t_new) > 1: t_new = t_new[0].upper() + t_new[1:] text_new.append(t_new) else: text_new.append(t) return ' '.join(text_new) def index_word_embeddings(embeddings_path, embedding_dim, skip_first_line=False): print('Indexing word embeddings from', str(embeddings_path), '...') embeddings_index = {} skipped = False with open(embeddings_path, encoding='utf-8', errors='ignore') as f: for line in f: if skip_first_line and not skipped: skipped = True continue splitted = line.split(' ') phrase_len = len(splitted)-embedding_dim coefs = ' '.join(splitted[phrase_len:len(splitted)]) phrase_parts = splitted[0:phrase_len] if set(phrase_parts) == set(['']): phrase_parts.append('') word = ' '.join(phrase_parts) coefs = np.fromstring(coefs, 'f', sep=' ') assert coefs.size == embedding_dim embeddings_index[word] = coefs print('Found', len(embeddings_index), 'word vectors.') return embeddings_index def index_keyed_embeddings(keyed_vectors): print('Indexing embeddings from keyed vectors...') embedding_index = {} for word, vector in zip(keyed_vectors.index2word, keyed_vectors.vectors): embedding_index[word] = vector return embedding_index # creates Tokenizer instance and fits it def fit_tokenizer(X_texts, hparams): print('Fitting tokenizer...') tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=hparams['max_words'], filters=hparams['tokenizer_filters'], lower=hparams['tokenizer_lower'], split=hparams['tokenizer_split'], char_level=hparams['tokenizer_char_level'], oov_token=hparams['tokenizer_oov_token']) tokenizer.fit_on_texts(X_texts) return tokenizer def prepare_embedding_matrix(max_words, embedding_dim, word_index, embeddings_index, hparams, lower_only=False): print('Preparing embedding matrix...') np.random.seed(hparams['random_state']) porter = PorterStemmer() lancaster = LancasterStemmer() snowball = SnowballStemmer('english') lemmatizer = WordNetLemmatizer() count = 0 embedding_matrix = np.zeros((max_words, embedding_dim)) random_vector = np.random.random(embedding_dim) for word, i in word_index.items(): if i >= max_words: continue if word in embeddings_index and word.lower() not in embeddings_index: embeddings_index[word.lower()] = embeddings_index[word] embedding_vector = embeddings_index.get(word.lower()) if lower_only else embeddings_index.get(word) # https://www.kaggle.com/wowfattie/3rd-place if embedding_vector is None: embedding_vector = embeddings_index.get(word.lower()) if embedding_vector is None: embedding_vector = embeddings_index.get(word.upper()) if embedding_vector is None: embedding_vector = embeddings_index.get(word.capitalize()) if embedding_vector is None: embedding_vector = embeddings_index.get(porter.stem(word)) if embedding_vector is None: embedding_vector = embeddings_index.get(lancaster.stem(word)) if embedding_vector is None: embedding_vector = embeddings_index.get(snowball.stem(word)) if embedding_vector is None: embedding_vector = embeddings_index.get(lemmatizer.lemmatize(word)) if word == hparams['tokenizer_oov_token'] or embedding_vector is None: embedding_matrix[i] = random_vector else: embedding_matrix[i] = embedding_vector count += 1 print('Word vectors coverage:', count / max_words) print('Embedding matrix shape:', embedding_matrix.shape) return embedding_matrix def create_padded_sequences(X_texts, tokenizer, hparams): print('Converting texts to sequences...') X_sequences = tokenizer.texts_to_sequences(X_texts) print('Padding sequences...') X_padded = tf.keras.preprocessing.sequence.pad_sequences(X_sequences, maxlen=hparams['max_length'], padding=hparams['padding'], truncating=hparams['truncating']) return X_padded #https://www.tensorflow.org/tutorials/text/nmt_with_attention#write_the_encoder_and_decoder_model class BahdanauAttention(tf.keras.Model): def __init__(self, units): super(BahdanauAttention, self).__init__() self.W1 = tf.keras.layers.Dense(units) self.W2 = tf.keras.layers.Dense(units) self.V = tf.keras.layers.Dense(1) def call(self, query, values): # values shape == (batch_size, max_length, hidden size) # hidden (query?) shape == (batch_size, hidden size) # hidden_with_time_axis shape == (batch_size, 1, hidden size) # we are doing this to perform addition to calculate the score (with broadcasting) hidden_with_time_axis = tf.expand_dims(query, 1) W1_out = self.W1(values) # (batch_size, max_length, units) W2_out = self.W2(hidden_with_time_axis) # (batch_size, 1, units) # the shape of the tensor (W_out_sum) before applying self.V is (batch_size, max_length, units) W_out_sum = W1_out + W2_out # (..., 1, units) broadcasted to (..., max_length, units) W_out_sum_tanh = tf.nn.tanh(W_out_sum) # score shape == (batch_size, max_length, 1) # we get 1 at the last axis because we are applying score to self.V score = self.V(W_out_sum_tanh) # attention_weights shape == (batch_size, max_length, 1) attention_weights = tf.nn.softmax(score, axis=1) context_vector = attention_weights * values # (batch_size, max_length, hidden_size) # context_vector shape after sum == (batch_size, hidden_size) context_vector = tf.reduce_sum(context_vector, axis=1) return context_vector, attention_weights def build_model(hparams, embedding_matrix=None): print('Building model...') np.random.seed(hparams['random_state']) random.seed(hparams['random_state']) tf.random.set_seed(hparams['random_state']) input_ = keras.layers.Input(shape=(hparams['max_length'],)) if embedding_matrix is not None: x = keras.layers.Embedding(input_dim=hparams['max_words'], output_dim=hparams['emb_out_dim'], embeddings_initializer=keras.initializers.Constant(embedding_matrix), trainable=False)(input_) else: x = keras.layers.Embedding(input_dim=hparams['max_words'], output_dim=hparams['emb_out_dim'])(input_) x = keras.layers.SpatialDropout1D(rate=hparams['dropout_rate'])(x) x, x_h_state_1_1, _, x_h_state_1_2, _ = keras.layers.Bidirectional(keras.layers.LSTM(256, return_sequences=True, return_state=True))(x) # x.shape = None, 256, len avg_1 = keras.layers.GlobalAveragePooling1D()(x) max_1 = keras.layers.GlobalMaxPooling1D()(x) x_ctx_1_1, _ = BahdanauAttention(256)(x_h_state_1_1, x[:,:,0:256]) x_ctx_1_2, _ = BahdanauAttention(256)(x_h_state_1_2, x[:,:,256:]) x_seq_2_1, x_state_2_1 = keras.layers.GRU(128, return_sequences=True, return_state=True)(x) avg_2_1 = keras.layers.GlobalAveragePooling1D()(x_seq_2_1) max_2_1 = keras.layers.GlobalMaxPooling1D()(x_seq_2_1) x_ctx_2_1, _ = BahdanauAttention(128)(x_state_2_1, x_seq_2_1) x_seq_2_2, x_state_2_2 = keras.layers.GRU(128, return_sequences=True, return_state=True, go_backwards=True)(x) avg_2_2 = keras.layers.GlobalAveragePooling1D()(x_seq_2_2) max_2_2 = keras.layers.GlobalMaxPooling1D()(x_seq_2_2) x_ctx_2_2, _ = BahdanauAttention(128)(x_state_2_2, x_seq_2_2) c_out = keras.layers.concatenate([avg_1, max_1, x_ctx_1_1, x_ctx_1_2, avg_2_1, max_2_1, x_ctx_2_1, avg_2_2, max_2_2, x_ctx_2_2, ]) output = keras.layers.Dense(1, activation='sigmoid')(c_out) model = keras.Model(inputs=[input_], outputs=[output]) model.compile(optimizer=hparams['optimizer'], loss='binary_crossentropy', metrics=[]) print('Parameters:', model.count_params()) return model class CustomAveraging(tf.keras.callbacks.Callback): def __init__(self, save_epoch): self.save_epoch = save_epoch - 1 self.saved_weights = None def on_epoch_end(self, epoch, logs={}): if epoch == self.save_epoch: self.saved_weights = self.model.get_weights() print('Model weights saved.') if epoch == (self.save_epoch + 1): model_weights = self.model.get_weights() new_weights = [(model_weights[i] + self.saved_weights[i]) / 2 for i in range(len(model_weights))] self.model.set_weights(new_weights) print('Model weights averaged.') # main start = time.time() X_train_texts, y_train = load_train_data(file_path=TRAIN_DATA_PATH, train_column='question_text', label_columns=LABEL_COLUMNS) X_test_texts, qids = load_test_data(TEST_DATA_PATH, train_column='question_text') hparams = { 'max_words': None, # for Tokenizer 'max_length': 40, 'batch_size': 512, 'emb_out_dim': 600, 'dropout_rate': 0.4, 'epochs': 4, 'optimizer': 'nadam', 'tokenizer_filters': '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', 'tokenizer_lower': False, 'tokenizer_split': " ", 'tokenizer_char_level': False, 'padding': 'post', 'truncating': 'post', 'tokenizer_oov_token': '', # not a real hyperparameter 'n_classes': y_train.shape[1], # not a real hyperparameter 'random_state': RANDOM_STATE # not a real hyperparameter } print('Cleaning contractions...') X_train_texts = [clean_contractions(text) for text in X_train_texts] X_test_texts = [clean_contractions(text) for text in X_test_texts] print('Cleaning special characters...') X_train_texts = [clean_special_chars(text) for text in X_train_texts] X_test_texts = [clean_special_chars(text) for text in X_test_texts] print('Correcting spelling...') X_train_texts = [correct_spelling(text) for text in X_train_texts] X_test_texts = [correct_spelling(text) for text in X_test_texts] keyed_vectors_google = models.KeyedVectors.load_word2vec_format(GOOGLE_NEWS_PATH, binary=True) #index_word_embeddings(GOOGLE_NEWS_PATH, 300, True) embeddings_index_google = index_keyed_embeddings(keyed_vectors_google) print('Deleting keyed vectors...') del keyed_vectors_google gc.collect() embeddings_index_wiki = index_word_embeddings(WIKI_NEWS_PATH, 300, True) tokenizer = fit_tokenizer(X_train_texts + X_test_texts, hparams) VOCAB_SIZE = len(tokenizer.word_index) if hparams['max_words'] is None: hparams['max_words'] = VOCAB_SIZE + 1 # 1 for padding value 0 else: hparams['max_words'] += 1 print('Found', VOCAB_SIZE, 'unique train tokens.') print('MAX WORDS:', hparams['max_words']) embedding_matrix_wiki = prepare_embedding_matrix(hparams['max_words'], 300, tokenizer.word_index, embeddings_index_wiki, hparams) embedding_matrix_google = prepare_embedding_matrix(hparams['max_words'], 300, tokenizer.word_index, embeddings_index_google, hparams) embedding_matrix_2 = 0.7 * embedding_matrix_wiki + 0.3 * embedding_matrix_google print('Deleting used matrices...') del embeddings_index_wiki, embeddings_index_google, embedding_matrix_wiki, embedding_matrix_google gc.collect() embeddings_index_glove = index_word_embeddings(GLOVE_PATH, 300) embeddings_index_paragram = index_word_embeddings(PARAGRAM_PATH, 300) embedding_matrix_glove = prepare_embedding_matrix(hparams['max_words'], 300, tokenizer.word_index, embeddings_index_glove, hparams) embedding_matrix_paragram = prepare_embedding_matrix(hparams['max_words'], 300, tokenizer.word_index, embeddings_index_paragram, hparams, lower_only=True) embedding_matrix_1 = 0.7 * embedding_matrix_glove + 0.3 * embedding_matrix_paragram print('Deleting used matrices...') del embeddings_index_glove, embeddings_index_paragram, embedding_matrix_glove, embedding_matrix_paragram gc.collect() embedding_matrix_final = np.concatenate((embedding_matrix_1, embedding_matrix_2), axis=1) X_train_padded = create_padded_sequences(X_train_texts, tokenizer, hparams) X_test_padded = create_padded_sequences(X_test_texts, tokenizer, hparams) model = build_model(hparams, embedding_matrix_final) model.fit(X_train_padded, y_train, epochs=hparams['epochs'], batch_size=hparams['batch_size'], shuffle=True, callbacks=[CustomAveraging(3)], verbose=2) print('\nPredicting...') y_pred = np.squeeze(model.predict(X_test_padded, batch_size=4096)) y_pred = (y_pred >= 0.3780).astype(int) submission = pd.DataFrame({'qid': qids}) submission['prediction'] = y_pred submission.to_csv('submission.csv', index=False) print('done')