#### Fold - 5 #### Epochs - [5,5,5,5,5] #### early_stop and best - False #### regularization - 0.0001 l1 #### preprocessing - use space punct, remove specific indexes from train df, replace multi space with single space #### set max_length - None #### f1 - averaging ( 0.31 - 0.32 = 0.32 ) #### lstm - unidirectional #### meta embedddings = True import os # os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' # os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) from sklearn.model_selection import StratifiedKFold from sklearn.metrics import f1_score from datetime import datetime print(os.listdir("../input")) import sys import warnings warnings.filterwarnings('ignore') import tensorflow as tf from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops import time import gc from multiprocessing import Pool import re from sklearn.preprocessing import StandardScaler from scipy.sparse import hstack ACTIVATION_DICT = {'tanh': tf.nn.tanh, 'relu': tf.nn.relu, 'sigmoid': tf.nn.sigmoid} tf.reset_default_graph() def evaluate_extra_projection(W_projection, b_projection, projection_layer_activations, initial_state, scope='evaluate_extra_projection'): ''' :param W_projection: a dict of matrices :param b_projection: a dict of biases :param projection_layer_activations: a list of activations :param initial_state: hidden state of previous network or the starting state :return: final hidden state ( generated by passing the initial state through the network ) ''' assert (len(W_projection) == len(b_projection) == len(projection_layer_activations)) with tf.name_scope(scope) as vs1: for iter_ in range(len(W_projection)): actv_fn = ACTIVATION_DICT[projection_layer_activations[iter_][1]] hidden_projection = actv_fn( tf.matmul(initial_state, W_projection[iter_]) + b_projection[iter_]) initial_state = hidden_projection return hidden_projection def initialize_projection_weight(projection_layer_dimensions, start_dimension, initializer=tf.contrib.layers.xavier_initializer(), scope='extra_projection' ): ''' :param projection_layer_dimensions: a list of dimensions excluding start_dimension eg: [ 200 , 50 , 128 ] :param projection_layer_activations: a list of activations , total number should be same as projection_layer_dimensions eg: [ tanh , relu , tanh ] :param start_dimension: :return: ''' with tf.variable_scope(scope) as vs2: W_projection = {} b_projection = {} for iter_ in range(len(projection_layer_dimensions)): W_temp = tf.get_variable("W_projection_{}".format(iter_), shape=[start_dimension, projection_layer_dimensions[iter_][0]], initializer=initializer) b_temp = tf.Variable(tf.zeros(shape=[projection_layer_dimensions[iter_][0]]), name='bias_projection_{}'.format(iter_)) start_dimension = projection_layer_dimensions[iter_][0] W_projection[iter_] = W_temp b_projection[iter_] = b_temp return W_projection, b_projection def initialize_output_weight(start_dimension, n_classes, scope='output', initializer=tf.contrib.layers.xavier_initializer()): ''' :param start_dimension: last dimension from projection layer or prenultimate network output dim :return: ''' with tf.variable_scope(scope): W_out = tf.get_variable("W_out", shape=[start_dimension, n_classes], initializer=initializer) b_out = tf.Variable(tf.zeros(shape=[n_classes]), name='bias_out') return W_out, b_out ### Preprocess ####################################################################################################################### ####################################################################################################################### import re remove_multi_dot = re.compile(r"""(?<=[a-z, A-Z])[\.]{2,}""", re.MULTILINE | re.IGNORECASE | re.VERBOSE | re.UNICODE) remove_multi_question = re.compile(r"""(?<=[a-z, A-Z])[\?]{2,}""", re.MULTILINE | re.IGNORECASE | re.VERBOSE | re.UNICODE) remove_multi_punctuation = re.compile(r"""(?<=[a-z, A-Z])[\!]{2,}""", re.MULTILINE | re.IGNORECASE | re.VERBOSE | re.UNICODE) remove_mulcti_comma = re.compile(r"""(?<=[a-z, A-Z])[\,]{2,}""", re.MULTILINE | re.IGNORECASE | re.VERBOSE | re.UNICODE) replace_email = re.compile(r"\b[\w.-]+?@\w+?\.\w+?\b") puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\', '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ] def clean_text(x): x = str(x) for punct in puncts: x = x.replace(punct, f' {punct} ') return x contractions = {"'cause": 'because', ',cause': 'because', ';cause': 'because', "ain't": 'am not', 'ain,t': 'am not', 'ain;t': 'am not', 'ain´t': 'am not', 'ain’t': 'am not', "aren't": 'are not', 'aren,t': 'are not', 'aren;t': 'are not', 'aren´t': 'are not', 'aren’t': 'are not', "can't": 'cannot', "can't've": 'cannot have', 'can,t': 'cannot', 'can,t,ve': 'cannot have', 'can;t': 'cannot', 'can;t;ve': 'cannot have', 'can´t': 'cannot', 'can´t´ve': 'cannot have', 'can’t': 'cannot', 'can’t’ve': 'cannot have', "could've": 'could have', 'could,ve': 'could have', 'could;ve': 'could have', "couldn't": 'could not', "couldn't've": 'could not have', 'couldn,t': 'could not', 'couldn,t,ve': 'could not have', 'couldn;t': 'could not', 'couldn;t;ve': 'could not have', 'couldn´t': 'could not', 'couldn´t´ve': 'could not have', 'couldn’t': 'could not', 'couldn’t’ve': 'could not have', 'could´ve': 'could have', 'could’ve': 'could have', "didn't": 'did not', 'didn,t': 'did not', 'didn;t': 'did not', 'didn´t': 'did not', 'didn’t': 'did not', "doesn't": 'does not', 'doesn,t': 'does not', 'doesn;t': 'does not', 'doesn´t': 'does not', 'doesn’t': 'does not', "don't": 'do not', 'don,t': 'do not', 'don;t': 'do not', 'don´t': 'do not', 'don’t': 'do not', "hadn't": 'had not', "hadn't've": 'had not have', 'hadn,t': 'had not', 'hadn,t,ve': 'had not have', 'hadn;t': 'had not', 'hadn;t;ve': 'had not have', 'hadn´t': 'had not', 'hadn´t´ve': 'had not have', 'hadn’t': 'had not', 'hadn’t’ve': 'had not have', "hasn't": 'has not', 'hasn,t': 'has not', 'hasn;t': 'has not', 'hasn´t': 'has not', 'hasn’t': 'has not', "haven't": 'have not', 'haven,t': 'have not', 'haven;t': 'have not', 'haven´t': 'have not', 'haven’t': 'have not', "he'd": 'he would', "he'd've": 'he would have', "he'll": 'he will', "he's": 'he is', 'he,d': 'he would', 'he,d,ve': 'he would have', 'he,ll': 'he will', 'he,s': 'he is', 'he;d': 'he would', 'he;d;ve': 'he would have', 'he;ll': 'he will', 'he;s': 'he is', 'he´d': 'he would', 'he´d´ve': 'he would have', 'he´ll': 'he will', 'he´s': 'he is', 'he’d': 'he would', 'he’d’ve': 'he would have', 'he’ll': 'he will', 'he’s': 'he is', "how'd": 'how did', "how'll": 'how will', "how's": 'how is', 'how,d': 'how did', 'how,ll': 'how will', 'how,s': 'how is', 'how;d': 'how did', 'how;ll': 'how will', 'how;s': 'how is', 'how´d': 'how did', 'how´ll': 'how will', 'how´s': 'how is', 'how’d': 'how did', 'how’ll': 'how will', 'how’s': 'how is', "i'd": 'i would', "i'll": 'i will', "i'm": 'i am', "i've": 'i have', 'i,d': 'i would', 'i,ll': 'i will', 'i,m': 'i am', 'i,ve': 'i have', 'i;d': 'i would', 'i;ll': 'i will', 'i;m': 'i am', 'i;ve': 'i have', "isn't": 'is not', 'isn,t': 'is not', 'isn;t': 'is not', 'isn´t': 'is not', 'isn’t': 'is not', "it'd": 'it would', "it'll": 'it will', "it's": 'it is', 'it,d': 'it would', 'it,ll': 'it will', 'it,s': 'it is', 'it;d': 'it would', 'it;ll': 'it will', 'it;s': 'it is', 'it´d': 'it would', 'it´ll': 'it will', 'it´s': 'it is', 'it’d': 'it would', 'it’ll': 'it will', 'it’s': 'it is', 'i´d': 'i would', 'i´ll': 'i will', 'i´m': 'i am', 'i´ve': 'i have', 'i’d': 'i would', 'i’ll': 'i will', 'i’m': 'i am', 'i’ve': 'i have', "let's": 'let us', 'let,s': 'let us', 'let;s': 'let us', 'let´s': 'let us', 'let’s': 'let us', "ma'am": 'madam', 'ma,am': 'madam', 'ma;am': 'madam', "mayn't": 'may not', 'mayn,t': 'may not', 'mayn;t': 'may not', 'mayn´t': 'may not', 'mayn’t': 'may not', 'ma´am': 'madam', 'ma’am': 'madam', "might've": 'might have', 'might,ve': 'might have', 'might;ve': 'might have', "mightn't": 'might not', 'mightn,t': 'might not', 'mightn;t': 'might not', 'mightn´t': 'might not', 'mightn’t': 'might not', 'might´ve': 'might have', 'might’ve': 'might have', "must've": 'must have', 'must,ve': 'must have', 'must;ve': 'must have', "mustn't": 'must not', 'mustn,t': 'must not', 'mustn;t': 'must not', 'mustn´t': 'must not', 'mustn’t': 'must not', 'must´ve': 'must have', 'must’ve': 'must have', "needn't": 'need not', 'needn,t': 'need not', 'needn;t': 'need not', 'needn´t': 'need not', 'needn’t': 'need not', "oughtn't": 'ought not', 'oughtn,t': 'ought not', 'oughtn;t': 'ought not', 'oughtn´t': 'ought not', 'oughtn’t': 'ought not', "sha'n't": 'shall not', 'sha,n,t': 'shall not', 'sha;n;t': 'shall not', "shan't": 'shall not', 'shan,t': 'shall not', 'shan;t': 'shall not', 'shan´t': 'shall not', 'shan’t': 'shall not', 'sha´n´t': 'shall not', 'sha’n’t': 'shall not', "she'd": 'she would', "she'll": 'she will', "she's": 'she is', 'she,d': 'she would', 'she,ll': 'she will', 'she,s': 'she is', 'she;d': 'she would', 'she;ll': 'she will', 'she;s': 'she is', 'she´d': 'she would', 'she´ll': 'she will', 'she´s': 'she is', 'she’d': 'she would', 'she’ll': 'she will', 'she’s': 'she is', "should've": 'should have', 'should,ve': 'should have', 'should;ve': 'should have', "shouldn't": 'should not', 'shouldn,t': 'should not', 'shouldn;t': 'should not', 'shouldn´t': 'should not', 'shouldn’t': 'should not', 'should´ve': 'should have', 'should’ve': 'should have', "that'd": 'that would', "that's": 'that is', 'that,d': 'that would', 'that,s': 'that is', 'that;d': 'that would', 'that;s': 'that is', 'that´d': 'that would', 'that´s': 'that is', 'that’d': 'that would', 'that’s': 'that is', "there'd": 'there had', "there's": 'there is', 'there,d': 'there had', 'there,s': 'there is', 'there;d': 'there had', 'there;s': 'there is', 'there´d': 'there had', 'there´s': 'there is', 'there’d': 'there had', 'there’s': 'there is', "they'd": 'they would', "they'll": 'they will', "they're": 'they are', "they've": 'they have', 'they,d': 'they would', 'they,ll': 'they will', 'they,re': 'they are', 'they,ve': 'they have', 'they;d': 'they would', 'they;ll': 'they will', 'they;re': 'they are', 'they;ve': 'they have', 'they´d': 'they would', 'they´ll': 'they will', 'they´re': 'they are', 'they´ve': 'they have', 'they’d': 'they would', 'they’ll': 'they will', 'they’re': 'they are', 'they’ve': 'they have', "wasn't": 'was not', 'wasn,t': 'was not', 'wasn;t': 'was not', 'wasn´t': 'was not', 'wasn’t': 'was not', "we'd": 'we would', "we'll": 'we will', "we're": 'we are', "we've": 'we have', 'we,d': 'we would', 'we,ll': 'we will', 'we,re': 'we are', 'we,ve': 'we have', 'we;d': 'we would', 'we;ll': 'we will', 'we;re': 'we are', 'we;ve': 'we have', "weren't": 'were not', 'weren,t': 'were not', 'weren;t': 'were not', 'weren´t': 'were not', 'weren’t': 'were not', 'we´d': 'we would', 'we´ll': 'we will', 'we´re': 'we are', 'we´ve': 'we have', 'we’d': 'we would', 'we’ll': 'we will', 'we’re': 'we are', 'we’ve': 'we have', "what'll": 'what will', "what're": 'what are', "what's": 'what is', "what've": 'what have', 'what,ll': 'what will', 'what,re': 'what are', 'what,s': 'what is', 'what,ve': 'what have', 'what;ll': 'what will', 'what;re': 'what are', 'what;s': 'what is', 'what;ve': 'what have', 'what´ll': 'what will', 'what´re': 'what are', 'what´s': 'what is', 'what´ve': 'what have', 'what’ll': 'what will', 'what’re': 'what are', 'what’s': 'what is', 'what’ve': 'what have', "where'd": 'where did', "where's": 'where is', 'where,d': 'where did', 'where,s': 'where is', 'where;d': 'where did', 'where;s': 'where is', 'where´d': 'where did', 'where´s': 'where is', 'where’d': 'where did', 'where’s': 'where is', "who'll": 'who will', "who's": 'who is', 'who,ll': 'who will', 'who,s': 'who is', 'who;ll': 'who will', 'who;s': 'who is', 'who´ll': 'who will', 'who´s': 'who is', 'who’ll': 'who will', 'who’s': 'who is', "won't": 'will not', 'won,t': 'will not', 'won;t': 'will not', 'won´t': 'will not', 'won’t': 'will not', "wouldn't": 'would not', 'wouldn,t': 'would not', 'wouldn;t': 'would not', 'wouldn´t': 'would not', 'wouldn’t': 'would not', "you'd": 'you would', "you'll": 'you will', "you're": 'you are', 'you,d': 'you would', 'you,ll': 'you will', 'you,re': 'you are', 'you;d': 'you would', 'you;ll': 'you will', 'you;re': 'you are', 'you´d': 'you would', 'you´ll': 'you will', 'you´re': 'you are', 'you’d': 'you would', 'you’ll': 'you will', 'you’re': 'you are', '´cause': 'because', '’cause': 'because', "you've": "you have", "could'nt": 'could not', "havn't": 'have not', "here’s": "here is", 'i""m': 'i am', "i'am": 'i am', "i'l": "i will", "i'v": 'i have', "wan't": 'want', "was'nt": "was not", "who'd": "who would", "who're": "who are", "who've": "who have", "why'd": "why would", "would've": "would have", "y'all": "you all", "y'know": "you know", "you.i": "you i", "your'e": "you are", "arn't": "are not", "agains't": "against", "c'mon": "common", "doens't": "does not", 'don""t': "do not", "dosen't": "does not", "dosn't": "does not", "shoudn't": "should not", "that'll": "that will", "there'll": "there will", "there're": "there are", "this'll": "this all", "u're": "you are", "ya'll": "you all", "you'r": "you are", "you’ve": "you have", "d'int": "did not", "did'nt": "did not", "din't": "did not", "dont't": "do not", "gov't": "government", "i'ma": "i am", "is'nt": "is not", } def replace_contractions(text): text = text.split() new_text = [] for word in text: if word in contractions: new_text.append(contractions[word]) continue if word.lower() in contractions: new_text.append(contractions[word.lower()]) continue else: new_text.append(word) text = " ".join(new_text) return text def fn_simple_tokenizer(text): text = text.lower() text = ' '.join(text.split()).split() return text def preprocess_before(text, replace_dict): for word, alternative in replace_dict.items(): text = text.replace(word, alternative) return text def special_bracket_handling_fw(word, vocab, all_alphabets): main_word = word for index, char in enumerate(word): if char in all_alphabets + "(": word = word[index:] break try: if word[0] == '(': if word[2] == ')': word_ = word[1] + word[3:] if word_ in vocab: return word_ else: if word_[-1] == ')': if word_[-3] == '(': word__ = word_[:-3] if word__ in vocab: return word__ except: pass return main_word def special_bracket_handling_bw(word, vocab, all_alphabets): main_word = word for index, char in enumerate(word): if char in all_alphabets + "(": word = word[index:] break try: if word[-1] == ')': if word[-3] == '(': word__ = word[:-3] if word__ in vocab: return word__ except: pass return main_word def special_bracket_handling_fw_square(word, vocab, all_alphabets): main_word = word for index, char in enumerate(word): if char in all_alphabets + "[": word = word[index:] break try: if word[0] == '[': if word[2] == ']': word_ = word[1] + word[3:] if word_ in vocab: return word_ else: if word_[-1] == '[': if word_[-3] == ']': word__ = word_[:-3] if word__ in vocab: return word__ except: pass return main_word def special_bracket_handling_bw_square(word, vocab, all_alphabets): main_word = word for index, char in enumerate(word): if char in all_alphabets + "[": word = word[index:] break try: if word[-1] == '[': if word[-3] == ']': word__ = word[:-3] if word__ in vocab: return word__ except: pass return main_word def replace_multiple_to_single_german(word): regex = r"(['ä', 'é','ö', 'ü', 'ß','a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'])\1{2,}" matches = re.finditer(regex, word, re.MULTILINE) matched = [] for match in matches: matched.append(match.group()) if matched: for char_ in matched: char = char_[0] word = word.replace(char_, char) return word else: return word def extract_character(word, vocab, all_alphabets): start = 0 end = len(word) - 1 for index, char in enumerate(word): if char in all_alphabets: start = index break word_rev = word[::-1] counter = len(word_rev) for index, char in enumerate(word_rev): if char in all_alphabets: end = counter - index break word_found = word[start:end] return word_found def extract_character_2side(word, vocab, all_alphabets): word_found = extract_character(word, vocab, all_alphabets) words = [] start = 0 for index, char in enumerate(word_found): if char not in all_alphabets: if word_found[start:index] in vocab and len(word_found[start:index]) >= 3: words.append(word_found[start:index]) start = index + 1 if index == len(word_found) - 1: if word_found[start:index + 1] in vocab and len(word_found[start:index + 1]) >= 3: words.append(word_found[start:index + 1]) return words def common_regex_match(word): match = re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', word) if match: return "email" match = re.match(r'[0-9]+(?:\.[0-9]+){3}', word) if match: return "ipaddress" match = re.match(r'\d{1,2}:\d{1,2}', word) if match: return "time" match = re.match(r'\d+-\d+-\d+', word) if match: return "date" match = re.match(r'\d+\/\d+\d+/\d+', word) if match: return "date" return None def special_preprocessing(word, vocab, all_alphabets, unknown_to_known, exceptions={}): word = word.strip() main_word = word if word in vocab: return unknown_to_known, word else: if word in exceptions: return unknown_to_known, word if word[-2:] == "'s": word_ = word[:-2] if word_ in vocab: unknown_to_known[main_word] = word_ return unknown_to_known, word_ else: word = word_ if word[-2:] == "’s": word_ = word[:-2] if word_ in vocab: unknown_to_known[main_word] = word_.strip() return unknown_to_known, word_.strip() else: word = word_.strip() word = replace_multiple_to_single_german(word) if word in vocab: unknown_to_known[main_word] = word.strip() return unknown_to_known, word.strip() word = special_bracket_handling_fw(word, vocab, all_alphabets) if word in vocab: unknown_to_known[main_word] = word.strip() return unknown_to_known, word.strip() word = special_bracket_handling_bw(word, vocab, all_alphabets) if word in vocab: unknown_to_known[main_word] = word.strip() return unknown_to_known, word.strip() word = special_bracket_handling_fw_square(word, vocab, all_alphabets) if word in vocab: unknown_to_known[main_word] = word.strip() return unknown_to_known, word.strip() word = special_bracket_handling_bw_square(word, vocab, all_alphabets) if word in vocab: unknown_to_known[main_word] = word.strip() return unknown_to_known, word.strip() word_copy = word for char in word: if char in all_alphabets: word_found = extract_character(word, vocab, all_alphabets) if word_found in vocab: unknown_to_known[main_word] = word_found.strip() return unknown_to_known, word_found.strip() else: if word_found[-2:] == "'s": word_ = word_found[:-2] if word_ in vocab: unknown_to_known[main_word] = word_.strip() return unknown_to_known, word_.strip() else: word_found = word_.strip() if word_found[-2:] == "’s": word_ = word_found[:-2] if word_ in vocab: unknown_to_known[main_word] = word_.strip() return unknown_to_known, word_.strip() else: word_found = word_.strip() word_found = special_bracket_handling_fw(word_found, vocab, all_alphabets) if word_found in vocab: unknown_to_known[main_word] = word_found.strip() return unknown_to_known, word_found.strip() word_found = special_bracket_handling_bw(word_found, vocab, all_alphabets) if word_found in vocab: unknown_to_known[main_word] = word_found.strip() return unknown_to_known, word_found.strip() word_found = special_bracket_handling_fw_square(word_found, vocab, all_alphabets) if word_found in vocab: unknown_to_known[main_word] = word_found.strip() return unknown_to_known, word_found.strip() word_found = special_bracket_handling_bw_square(word_found, vocab, all_alphabets) if word_found in vocab: unknown_to_known[main_word] = word_found.strip() return unknown_to_known, word_found.strip() return unknown_to_known, main_word def start_preprocess(text, lower=False): if lower: text = text.lower() text = replace_contractions(text) text = clean_text(text) ##### replacing punctuations with a spaces on either sides text = text.strip() text = text.strip('\n') text = text.strip('\r') text = text.strip('\t') text = text.strip('\r\n') text = text.replace(r"\r\n", r" ") text = text.replace(r"\n", r" ") text = text.replace(r"\r", r" ") text = text.replace(r"\t", r" ") text = ' '.join(text.split()) # more than one space to single space return text ### Preprocess , vocab , batch generator ####################################################################################################################### ###################################################################################################################### import numpy as np import pandas as pd import json class TextPreprocessor: def __init__(self, lower=True): """Text Preprocessor base class. corpus: a list of sentences """ self.lower = lower def fit_sentence(self, sentence, lower=True): """ strip each sentence , lowercase it and split by space # sentence.strip().lower().split() sentence: a string """ if lower: return sentence.strip().lower().split() else: return sentence.strip().split() def fit(self, sentence_list, lower=True): """ strip each sentence , lowercase it and split by space # sentence.strip().lower().split() sentence: a string """ preprocessed_data = [] word_tokens_len = [] for sentence in sentence_list: tokens = self.fit_sentence(sentence) word_tokens_len.append(len(tokens)) preprocessed_data.append(tokens) return word_tokens_len, preprocessed_data def transform_sentence(self, sentence): """ same as preprocess function, but can be used for test data, just a convenient notational convention sentence: a string """ return self.fit_sentence(sentence) def transform(self, sentence_list): """ same as preprocess function, but can be used for test data, just a convenient notational convention sentence: a string """ preprocessed_data = [] word_tokens_len = [] for sentence in sentence_list: tokens = self.transform_sentence(sentence, lower=self.lower) preprocessed_data.append(tokens) word_tokens_len.append(len(tokens)) return word_tokens_len, preprocessed_data ###################################################################################################################### ###################################################################################################################### class VocabProcessor: def __init__(self, word_min_count=1, replace_unknown_word=True, keep_random_embedding_for_unknwon_word=False, remove_unknown_words=False): """ Vocabprocessor base class """ self.SPECIAL_TOKENS_words = ['', '', '', ''] self.SPECIAL_TOKENS_chars = ['<_PAD_>', '<_UNK_>', '<_START_>', '<_END>_'] self.word_min_count = word_min_count self.replace_unknown_word = replace_unknown_word self.keep_random_embedding_for_unknwon_word = keep_random_embedding_for_unknwon_word self.remove_unknown_words = remove_unknown_words def fit_word(self, sentence_tokens_list): vocab_word_counter = {} for index, tokens in enumerate(sentence_tokens_list): for word in tokens: if word not in vocab_word_counter: vocab_word_counter[word] = 1.0 else: vocab_word_counter[word] += 1.0 counter = 0 self.vocab_word = [] for word in self.SPECIAL_TOKENS_words: self.vocab_word.append(word) vocab_word_counter[word] = 1.0 counter += 1 for word, counter_ in sorted(vocab_word_counter.items(), key=lambda x: x[1], reverse=True): if counter >= self.word_min_count: self.vocab_word.append(word) counter += 1 self.vocab_counter = vocab_word_counter def transform_word(self, word_tokens, unk=False): transform_holder = [] unk_count = 0 for word in word_tokens: if word in self.vocab_word: transform_holder.append(self.vocab_word[word]) continue if self.replace_unknown_word or self.keep_random_embedding_for_unknwon_word: transform_holder.append(self.vocab_word['']) unk_count += 1 continue if self.remove_unknown_words: unk_count += 1 continue if unk: return np.round(unk_count / len(word_tokens), 2), transform_holder else: if transform_holder: # to account for all UNK in the sentence return transform_holder else: transform_holder = [self.vocab_word['']] return transform_holder def cross_check_embedding_vocab(self, vocab, embedding_vocab): self.known_words = self.SPECIAL_TOKENS_words[:] self.unknown_words = [] for word in vocab: if word in self.SPECIAL_TOKENS_words: continue if word in embedding_vocab: self.known_words.append(word) else: if self.replace_unknown_word: self.unknown_words.append(word) if self.keep_random_embedding_for_unknwon_word: self.known_words.append(word) if self.remove_unknown_words: self.unknown_words.append(word) ####################################################################################################################### ####################################################################################################################### class EmbeddingProcessor: @staticmethod def generate_train_emebdding_and_vocab(vocab_processor, original_emebdding, original_emebdding_vocab): embedding_dimension = original_emebdding.shape[1] emb_mean, emb_std = original_emebdding.mean(), original_emebdding.std() train_embeddings = [] train_vocab = {} for index, word in enumerate(vocab_processor.known_words): if word == '': array = np.zeros((embedding_dimension)).tolist() train_embeddings.append(array) train_vocab[word] = index continue if word in original_emebdding_vocab: array = original_emebdding[original_emebdding_vocab[word]].tolist() train_embeddings.append(array) train_vocab[word] = index else: try: word_ = word.strip().lower() seed = abs(hash(word_)) % (10 ** 8) except: seed = 0 np.random.seed(seed) array = np.random.normal(emb_mean, emb_std, (embedding_dimension)).tolist() train_embeddings.append(array) train_vocab[word] = index return np.array(train_embeddings), train_vocab ##################################################################################################################### ##################################################################################################################### class TextBatchProcessor: # @staticmethod def pad_the_batch(self, batch_data, pad_index=0): batch_len = np.array([len(array) for array in batch_data]) max_seq_len = np.max(batch_len) sentence_padded_list = [] for array in batch_data: difference = max_seq_len - len(array) if difference == 0: sentence_padded_list.append(array) continue else: array = array + [pad_index] * difference sentence_padded_list.append(array) return np.array(sentence_padded_list), np.array(batch_len) def batch_train(self, sentence_tokens, label_list, batch_size, pad_index=0, shuffle=False, shuffle_counter=1): assert (len(sentence_tokens) == len(label_list)) if isinstance(sentence_tokens, pd.core.series.Series): sentence_tokens = sentence_tokens.values if isinstance(label_list, pd.core.series.Series): label_list = label_list.values if isinstance(sentence_tokens, list): sentence_tokens = np.array(sentence_tokens) if isinstance(label_list, list): label_list = np.array(label_list) total_data_points = len(sentence_tokens) if shuffle: np.random.seed(shuffle_counter) shuffled_indices = np.random.permutation(total_data_points) sentence_tokens = sentence_tokens[shuffled_indices] label_list = label_list[shuffled_indices] remainder = total_data_points % batch_size if remainder == 0.0: iterations = int(total_data_points / batch_size) else: iterations = int((total_data_points / batch_size) + 1) for iter_ in range(iterations): start_index = iter_ * batch_size end_index = (iter_ + 1) * batch_size padded_batch, padded_batch_len = self.pad_the_batch(sentence_tokens[start_index:end_index], pad_index) yield (padded_batch, padded_batch_len, label_list[start_index:end_index]) def batch_train_pairs(self, sentence_tokens_1, sentence_tokens_2, label_list, batch_size, pad_index=0, shuffle=False, shuffle_counter=1): assert (len(sentence_tokens_1) == len(sentence_tokens_2) == len(label_list)) if isinstance(sentence_tokens_1, pd.core.series.Series): sentence_tokens_1 = sentence_tokens_1.values if isinstance(sentence_tokens_2, pd.core.series.Series): sentence_tokens_2 = sentence_tokens_2.values if isinstance(label_list, pd.core.series.Series): label_list = label_list.values if isinstance(sentence_tokens_1, list): sentence_tokens = np.array(sentence_tokens) if isinstance(sentence_tokens_2, list): sentence_tokens = np.array(sentence_tokens) if isinstance(label_list, list): label_list = np.array(label_list) total_data_points = len(sentence_tokens_1) if shuffle: np.random.seed(shuffle_counter) shuffled_indices = np.random.permutation(total_data_points) sentence_tokens_1 = sentence_tokens_1[shuffled_indices] sentence_tokens_2 = sentence_tokens_2[shuffled_indices] label_list = label_list[shuffled_indices] remainder = total_data_points % batch_size if remainder == 0.0: iterations = int(total_data_points / batch_size) else: iterations = int((total_data_points / batch_size) + 1) for iter_ in range(iterations): start_index = iter_ * batch_size end_index = (iter_ + 1) * batch_size s1 = sentence_tokens_1[start_index:end_index] s2 = sentence_tokens_1[start_index:end_index] s = [] for i in range(len(s1)): s.append(s1[i]) s.append(s2[i]) padded_batch, padded_batch_len = self.pad_the_batch(s, pad_index) padded_batch = padded_batch.reshape(-1, 2, padded_batch.shape[-1]) padded_batch_len_mod = [] for i in range(len(padded_batch_len)): if i % 2 == 0.0: padded_batch_len_mod.append(padded_batch_len[i]) yield (padded_batch, np.array(padded_batch_len_mod), label_list[start_index:end_index]) def batch_test_pairs(self, sentence_tokens_1, sentence_tokens_2, label_list, batch_size, pad_index=0): assert (len(sentence_tokens_1) == len(sentence_tokens_2) == len(label_list)) if isinstance(sentence_tokens_1, pd.core.series.Series): sentence_tokens_1 = sentence_tokens_1.values if isinstance(sentence_tokens_2, pd.core.series.Series): sentence_tokens_2 = sentence_tokens_2.values if isinstance(label_list, pd.core.series.Series): label_list = label_list.values if isinstance(sentence_tokens_1, list): sentence_tokens = np.array(sentence_tokens) if isinstance(sentence_tokens_2, list): sentence_tokens = np.array(sentence_tokens) if isinstance(label_list, list): label_list = np.array(label_list) total_data_points = len(sentence_tokens_1) remainder = total_data_points % batch_size if remainder == 0.0: iterations = int(total_data_points / batch_size) else: iterations = int((total_data_points / batch_size) + 1) for iter_ in range(iterations): start_index = iter_ * batch_size end_index = (iter_ + 1) * batch_size s1 = sentence_tokens_1[start_index:end_index] s2 = sentence_tokens_1[start_index:end_index] s = [] for i in range(len(s1)): s.append(s1[i]) s.append(s2[i]) padded_batch, padded_batch_len = self.pad_the_batch(s, pad_index) padded_batch = padded_batch.reshape(-1, 2, padded_batch.shape[-1]) padded_batch_len_mod = [] for i in range(len(padded_batch_len)): if i % 2 == 0.0: padded_batch_len_mod.append(padded_batch_len[i]) yield (padded_batch, np.array(padded_batch_len_mod), ) def batch_test(self, sentence_tokens, batch_size, pad_index=0): if isinstance(sentence_tokens, pd.core.series.Series): sentence_tokens = sentence_tokens.values if isinstance(sentence_tokens, list): sentence_tokens = np.array(sentence_tokens) total_data_points = len(sentence_tokens) remainder = total_data_points % batch_size if remainder == 0.0: iterations = int(total_data_points / batch_size) else: iterations = int((total_data_points / batch_size) + 1) for iter_ in range(iterations): start_index = iter_ * batch_size end_index = (iter_ + 1) * batch_size padded_batch, padded_batch_len = self.pad_the_batch(sentence_tokens[start_index:end_index], pad_index) yield (padded_batch, padded_batch_len ) ###################################################################################################################### ###################################################################################################################### ### Load embeddings def load_glove(): EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt' def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)) embedding_vocab = {} counter = 0 for word in embeddings_index.keys(): embedding_vocab[word] = counter counter += 1 embedding_matrix = np.stack(embeddings_index.values()) del embeddings_index return embedding_vocab, embedding_matrix def load_fasttext(): EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec' def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE) if len(o) > 100) embedding_vocab = {} counter = 0 for word in embeddings_index.keys(): embedding_vocab[word] = counter counter += 1 embedding_matrix = np.stack(embeddings_index.values()) del embeddings_index return embedding_vocab, embedding_matrix def load_para(): EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32') embeddings_index = dict( get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore') if len(o) > 100) embedding_vocab = {} counter = 0 for word in embeddings_index.keys(): embedding_vocab[word] = counter counter += 1 embedding_matrix = np.stack(embeddings_index.values()) del embeddings_index return embedding_vocab, embedding_matrix ##################################################################################################################### ##################################################################################################################### class SubPreprocess(TextPreprocessor): def __init__(self): self.correction_vocab = {} self.max_length = 150 # any text words which are greater than 150 words will be trimmed to tokens[:self.max_length] def fit_sentence(self, sentence, lower=True, reverse=False): ''' ''' alphabets_numbers = 'abcdefghijklmnopqrstuvwxyz0123456789' all_alphabets = 'abcdefghijklmnopqrstuvwxyz' if sentence[-1] not in alphabets_numbers: # spacing out the last character sentence = sentence[:-1] + ' ' + sentence[-1] text = start_preprocess(sentence, lower=False) temp = [] for word in text.split(): word_match = common_regex_match(word) if word_match: temp.append(word_match) continue if word in embedding_vocab: temp.append(word) continue if word.lower() in embedding_vocab: temp.append(word.lower()) continue if word in self.correction_vocab: temp.append(self.correction_vocab[word]) continue if word.lower() in self.correction_vocab: temp.append(self.correction_vocab[word.lower()]) continue self.correction_vocab, word = special_preprocessing(word, embedding_vocab, all_alphabets, self.correction_vocab, exceptions={}) if word in self.correction_vocab: temp.append(self.correction_vocab[word]) continue self.correction_vocab, word = special_preprocessing(word.lower(), embedding_vocab, all_alphabets, self.correction_vocab, exceptions={}) if word in self.correction_vocab: temp.append(self.correction_vocab[word]) continue else: temp.append(word) if self.max_length: temp = temp[:self.max_length] return temp ### Load train and test data start_time = time.time() train_df = pd.read_csv("../input/train.csv") test_df = pd.read_csv("../input/test.csv") print("Train shape : ", train_df.shape) print("Test shape : ", test_df.shape) end_time = time.time() print("Time taken seconds", end_time - start_time) # In[ ]: #### Fill NA ( :-) ) train_df["question_text"].fillna("insincere", inplace=True) test_df["question_text"].fillna("insincere", inplace=True) #### Load embeddings use_meta_emebddings = True if use_meta_emebddings == True: embedding_vocab_glove, embedding_matrix_glove = load_glove() embedding_vocab_para, embedding_matrix_para = load_para() embedding_vocab = embedding_vocab_glove.copy() for w in embedding_vocab_para: if w not in embedding_vocab: embedding_vocab[w] = 1 else: start_time = time.time() embedding_vocab, embedding_matrix = load_glove() print("Loaded embeddings") end_time = time.time() print("Time taken seconds", end_time - start_time) start_time = time.time() train_processor = SubPreprocess() train_sentence_len, train_word_tokens = train_processor.fit(train_df['question_text']) test_word_lens, test_word_tokens = train_processor.fit(test_df['question_text']) train_df['sen_len'] = train_sentence_len train_df['tokens'] = train_word_tokens test_df['sen_len'] = test_word_lens test_df['tokens'] = test_word_tokens print("1. Preprocessing done") end_time = time.time() print("Time taken {} seconds".format(end_time - start_time)) print() start_time = time.time() train_sample = train_df[train_df['target'] == 1] index_to_delete = train_sample[train_sample['sen_len'] < 4].index train_df.drop(index_to_delete, inplace=True) train_df.index = range(len(train_df.index)) del train_sample print("2. Removed some data from training data done", train_df.shape) end_time = time.time() print("Time taken {} seconds".format(end_time - start_time)) print() start_time = time.time() vocab_processor = VocabProcessor(word_min_count=1, replace_unknown_word=True, keep_random_embedding_for_unknwon_word=False, remove_unknown_words=False) all_tokens = [] for tokens in train_df['tokens']: all_tokens.append(tokens) for tokens in test_df['tokens']: all_tokens.append(tokens) vocab_processor.fit_word(all_tokens) vocab_processor.cross_check_embedding_vocab(vocab_processor.vocab_word, embedding_vocab) emb_processor = EmbeddingProcessor() if use_meta_emebddings: train_embedding_glove, train_vocab_glove = emb_processor.generate_train_emebdding_and_vocab(vocab_processor, embedding_matrix_glove, embedding_vocab_glove) train_embedding_para, train_vocab_para = emb_processor.generate_train_emebdding_and_vocab(vocab_processor, embedding_matrix_para, embedding_vocab_para) assert (train_vocab_glove == train_vocab_para) train_vocab = train_vocab_glove.copy() train_vocab_reverse = {v: k for k, v in train_vocab.items()} train_embedding = np.mean([train_embedding_glove, train_embedding_para], axis=0) #### To avoid meaningless averaging for index, word in train_vocab_reverse.items(): if word in embedding_vocab_glove and word in embedding_vocab_para: continue if word in embedding_vocab_glove: if word not in embedding_vocab_para: train_embedding[index] = train_embedding_glove[index] continue if word in embedding_vocab_para: if word not in embedding_vocab_glove: train_embedding[index] = train_embedding_para[index] continue del embedding_matrix_glove del embedding_vocab_glove del embedding_matrix_para del embedding_vocab_para # del train_embedding_glove # del train_embedding_para # del train_vocab_glove # del train_vocab_para else: train_embedding, train_vocab = emb_processor.generate_train_emebdding_and_vocab(vocab_processor, embedding_matrix, embedding_vocab) del embedding_matrix del embedding_vocab vocab_processor.vocab_word = train_vocab # del embedding_matrix # del embedding_vocab print("3. Vocab and embedding has been created") print("Train emebdding shape {}".format(train_embedding.shape)) print("Train vocab shape {}".format(len(train_vocab))) end_time = time.time() print("Time taken {} seconds".format(end_time - start_time)) print() start_time = time.time() def get_len(tokens): return len(tokens) train_df['tokens_index'] = train_df['tokens'].map(vocab_processor.transform_word) train_df['sen_len'] = train_df['tokens_index'].map(get_len) test_df['tokens_index'] = test_df['tokens'].map(vocab_processor.transform_word) test_df['sen_len'] = test_df['tokens_index'].map(get_len) train_null_tokens = [index for index, row in train_df.iterrows() if row['tokens_index'] == [1]] #### those indexes where we have all UNK test_null_tokens = [index for index, row in test_df.iterrows() if row['tokens_index'] == [1]] #### those indexes where we have all UNK print("4. Convert words to indexes has been done") print("Train complete unknown tokens is", len(train_null_tokens)) print("Test complete unknown tokens is", len(test_null_tokens)) end_time = time.time() print("Time taken {} seconds".format(end_time - start_time)) print() gc.collect() ########## Keras style # from keras.preprocessing.sequence import pad_sequences MAXLEN = MAX_LEN = 72 MAX_FEATURES = len(train_embedding) EMBED_SIZE = 300 def trim_sentence(token_list): return token_list[:MAX_LEN] def reverse_sentence(token_list): return token_list[::-1] train_df['tokens_index_keras'] = train_df['tokens_index'].map(trim_sentence) test_df['tokens_index_keras'] = test_df['tokens_index'].map(trim_sentence) train_df['tokens_index_reverse'] = train_df['tokens_index'].map(reverse_sentence) test_df['tokens_index_reverse'] = test_df['tokens_index'].map(reverse_sentence) train_df['tokens_index_keras'] = train_df['tokens_index'].map(trim_sentence) test_df['tokens_index_keras'] = test_df['tokens_index'].map(trim_sentence) train_df['tokens_index_keras_reverse'] = train_df['tokens_index_keras'].map(reverse_sentence) test_df['tokens_index_keras_reverse'] = test_df['tokens_index_keras'].map(reverse_sentence) gc.collect() ######################################################################################################################################### ######################################### Best f1 score ################################################################################# from sklearn.metrics import roc_curve, precision_recall_curve def pick_the_best(y_true, y_proba, plot=False): precision, recall, thresholds = precision_recall_curve(y_true, y_proba) thresholds = np.append(thresholds, 0.99) F = 2 / (1 / precision + 1 / recall) best_score = np.max(F) best_th = thresholds[np.argmax(F)] res = dict(zip(thresholds, F)) if plot: plt.plot(thresholds, F, '-b') plt.plot([best_th], [best_score], '*r') plt.show() search_result = {'threshold': best_th, 'f1': best_score} return sorted(res.items(), key=lambda x: x[1], reverse=True) class Model_2: def __init__(self, loss_op, train_op, prediction_op, input_x, input_x_len, input_y, dropout_keep_prob, model_dir, early_stop=False, ema_and_assign=None, use_ema=False, pred_before_ema=False): # model_dir = os.path.join("/kaggle","saved_models", model_dir) # if os.path.exists(model_dir): # raise ValueError("A Model already found directory {}, change the path".format(model_dir)) self.model_dir = model_dir session_conf = tf.ConfigProto( allow_soft_placement=True, log_device_placement=True) self.sess = tf.Session(config=session_conf) self.saver = tf.train.Saver(max_to_keep=100) self.loss_op = loss_op self.train_op = train_op self.prediction_prob = prediction_op self.best_epoch = 0 self.sess.run(tf.global_variables_initializer()) self.input_x = input_x self.input_x_len = input_x_len self.input_y = input_y self.input_dropout_keep_prob = dropout_keep_prob self.batch_processor = TextBatchProcessor() self.the_train_loss = [] self.the_val_loss = [] self.best_val_f1 = -0.01 self.best_val_thresh = 0.0 self.all_score = [] self.early_stop = early_stop self.ema = ema_and_assign[0] # ema self.ema_assign_op = ema_and_assign[1] # assign_op self.use_ema = use_ema, self.pred_before_ema = pred_before_ema def one_loop_train(self, X_train, y_train, train_vocab, batch_size, dropout_keep_prob, epoch): batch_processor_train = self.batch_processor.batch_train(X_train, y_train, pad_index=train_vocab[''], batch_size=batch_size, shuffle=True, shuffle_counter=epoch) epoch_loss = 0.0 epoch_time = 0.0 train_loss = [] total_batches = int(len(X_train) / batch_size) for batch_count, (X_batch, X_batch_len, Y_batch) in enumerate(batch_processor_train): start_time = datetime.now() feed_dict_train = {self.input_x: X_batch, self.input_x_len: X_batch_len, self.input_y: Y_batch, self.input_dropout_keep_prob: dropout_keep_prob} batch_loss, _ = self.sess.run((self.loss_op, self.train_op), feed_dict=feed_dict_train) end_time = datetime.now() difference = (end_time - start_time).total_seconds() epoch_time += difference epoch_loss += batch_loss if batch_count % 500 == 0.0: print("Epoch {} , batch {} / {} , per batch cost {:5.5f} in {:5.5f} seconds".format(epoch, batch_count, total_batches, batch_loss, epoch_time)) epoch_loss = epoch_loss / (batch_count + 1) train_loss.append(epoch_loss) print("Epoch {} total cost {:5.5f} in {:5.5f} seconds".format(epoch, np.mean(train_loss), epoch_time)) self.the_train_loss.append(np.mean(train_loss)) def one_loop_validate(self, X_val, y_val, train_vocab, batch_size): batch_processor_validate = self.batch_processor.batch_train(X_val, y_val, pad_index=train_vocab[''], batch_size=batch_size, shuffle=False ) validation_batches = int(len(X_val) / batch_size) validation_probability = [] epoch_time = 0.0 validation_loss = [] print("Validation proceeding wait . . . . . . . . . . . .") for batch_count, (X_val_batch, X_val_batch_len, Y_val_batch) in enumerate(batch_processor_validate): start_time = datetime.now() feed_dict_validate = {self.input_x: X_val_batch, self.input_x_len: X_val_batch_len, self.input_y: Y_val_batch, self.input_dropout_keep_prob: 1.0} val_loss, val_pred_prob = self.sess.run((self.loss_op, self.prediction_prob), feed_dict=feed_dict_validate) end_time = datetime.now() difference = (end_time - start_time).total_seconds() epoch_time += difference validation_probability.extend(val_pred_prob) validation_loss.append(val_loss) print("Validation done in {} with loss {}".format(epoch_time, np.mean(validation_loss))) val_score = pick_the_best(y_val, validation_probability) temp_thresh = val_score[0][0] temp_f1 = val_score[0][1] self.the_val_loss.append(np.mean(validation_loss)) self.all_score.append(val_score[0]) return np.array(validation_probability) , {'thresh': temp_thresh, 'f1_score': temp_f1} def train(self, train_data, train_vocab, dropout_keep_prob, val_data=None, epochs=20, validate_every=1, batch_size=512, do_validation=True): X_train, y_train = train_data if val_data: X_val, y_val = val_data for iter_ in range(1, epochs + 1): self.one_loop_train(X_train, y_train, train_vocab, batch_size=batch_size, dropout_keep_prob=dropout_keep_prob, epoch=iter_) print("Epoch {} done".format(iter_)) print("Training finished") def evaluate(self): pass def predict(self, X_test, train_vocab, batch_size): test_batch_processor = self.batch_processor.batch_test(X_test, pad_index=train_vocab[''], batch_size=batch_size) test_predictions = [] for (X_batch, X_batch_len) in test_batch_processor: feed_dict = {self.input_x: X_batch, self.input_x_len: X_batch_len, self.input_dropout_keep_prob: 1.0} t_p = self.sess.run(self.prediction_prob, feed_dict=feed_dict) test_predictions.extend(t_p) return test_predictions def neural_blender_simple(neural_model, model_name, network_config, train_X, train_y, val_X, val_Y, test_X, epoch_per_fold=None, n_folds=5, cv_epoch=None, use_cv=True, pred_before_ema=False, batch_size = 1024 ): ''' All neural network model are peformed here, with CV , OOF based on conditions ''' blue_print = {} if not cv_epoch: sys.exit("When use_cv=True, cv_epoch must be defined as an integer") if not isinstance(cv_epoch, int): sys.exit("When use_cv=True, cv_epoch must be defined as an integer") loss, train_op, prediction_prob, input_x, input_x_len, input_y, input_dropout_keep_prob, dropout_keep_prob, ema = neural_model(network_config) model_dir = 'model_name' model = Model_2(loss_op=loss, train_op=train_op, prediction_op=prediction_prob, input_x=input_x, input_x_len=input_x_len, input_y=input_y, dropout_keep_prob=input_dropout_keep_prob, model_dir=model_dir, early_stop=False, ema_and_assign=ema, use_ema=True, pred_before_ema=pred_before_ema) model.train((train_X, train_y), train_vocab, dropout_keep_prob, epochs=cv_epoch) model.sess.run(model.ema_assign_op) valid_predictions = np.array(model.predict(val_X, train_vocab, batch_size=batch_size)) test_predictions = np.array(model.predict(test_X, train_vocab, batch_size=batch_size)) blue_print = {} blue_print['model_name'] = model_name blue_print['valid_predictions'] = valid_predictions blue_print['test_predictions'] = test_predictions val_scores = pick_the_best(val_Y, valid_predictions) blue_print['f1_score'] = val_scores[0][1] blue_print['thresh'] = val_scores[0][0] return blue_print def neural_blender_no_val(neural_model, model_name, network_config, train_X, train_y, test_X, cv_epoch=None, batch_size = 1024 ): ''' All neural network model are peformed here, with CV , OOF based on conditions ''' blue_print = {} if not cv_epoch: sys.exit("When use_cv=True, cv_epoch must be defined as an integer") if not isinstance(cv_epoch, int): sys.exit("When use_cv=True, cv_epoch must be defined as an integer") loss, train_op, prediction_prob, input_x, input_x_len, input_y, input_dropout_keep_prob, dropout_keep_prob, ema = neural_model(network_config) model_dir = 'model_name' model = Model_2(loss_op=loss, train_op=train_op, prediction_op=prediction_prob, input_x=input_x, input_x_len=input_x_len, input_y=input_y, dropout_keep_prob=input_dropout_keep_prob, model_dir=model_dir, early_stop=False, ema_and_assign=ema, use_ema=True, pred_before_ema=False) model.train((train_X, train_y), train_vocab, dropout_keep_prob, epochs=cv_epoch) model.sess.run(model.ema_assign_op) train_predictions = np.array(model.predict(train_X, train_vocab, batch_size=batch_size)) test_predictions = np.array(model.predict(test_X, train_vocab, batch_size=batch_size)) blue_print = {} blue_print['model_name'] = model_name blue_print['train_predictions'] = train_predictions blue_print['test_predictions'] = test_predictions val_scores = pick_the_best(train_y, train_predictions) blue_print['f1_score'] = val_scores[0][1] blue_print['thresh'] = val_scores[0][0] return blue_print def neural_blender_oof(neural_model, model_name, network_config, train_X, train_y, test_X, epoch_per_fold, n_folds=5, batch_size = 1024, DATA_SPLIT_SEED=2018 ): ''' All neural network model are peformed here, with CV , OOF based on conditions ''' blue_print = {} oof_prob = np.zeros(train_y.shape) test_prob = np.zeros(test_X.shape[0]) f1_per_fold = [] thresh_per_fold = [] blue_print['model_name'] = model_name splits = list( StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=DATA_SPLIT_SEED).split(train_X, train_y)) validate_per_fold = epoch_per_fold for idx, (train_idx, valid_idx) in enumerate(splits): X_train = train_X[train_idx] y_train = train_y[train_idx] X_val = train_X[valid_idx] y_val = train_y[valid_idx] loss, train_op, prediction_prob, input_x, input_x_len, input_y, input_dropout_keep_prob, dropout_keep_prob, ema = neural_model(network_config) model_dir = 'model_name' model = Model_2(loss_op=loss, train_op=train_op, prediction_op=prediction_prob, input_x=input_x, input_x_len=input_x_len, input_y=input_y, dropout_keep_prob=input_dropout_keep_prob, model_dir=model_dir, early_stop=False, ema_and_assign=ema, use_ema=True, pred_before_ema=False) model.train((X_train, y_train), train_vocab, dropout_keep_prob, epochs=epoch_per_fold[idx]) model.sess.run(model.ema_assign_op) valid_preds, val_scores = model.one_loop_validate(X_val, y_val, train_vocab, batch_size=batch_size) oof_prob[valid_idx] = valid_preds test_predictions = np.array(model.predict(test_X, train_vocab, batch_size=batch_size)) test_prob += test_predictions/len(splits) f1_per_fold.append(val_scores['f1_score']) thresh_per_fold.append(val_scores['thresh']) print("Best f1 score fold {} is {} at threshhold {}".format(idx, val_scores['f1_score'], val_scores['thresh'])) print("="*15) thresh_mean = np.mean(thresh_per_fold) f1_mean = f1_score((oof_prob > thresh_mean).astype(np.int), train_y ) oof_scores = pick_the_best(train_y, oof_prob) thresh_oof = oof_scores[0][0] f1_oof = oof_scores[0][1] blue_print['f1_mean'] = f1_mean blue_print['thresh_mean'] = thresh_mean blue_print['thresh_oof'] = thresh_oof blue_print['f1_oof'] = f1_oof blue_print['oof_prob'] = oof_prob blue_print['test_prob'] = test_prob print("Threshhold", thresh_per_fold) print("F1 score", f1_per_fold) return blue_print def neural_blender_oof_mod(neural_model, model_name, network_config, train_X, train_y, test_X, epoch_per_fold, n_folds=5, batch_size = 1024 ): ''' All neural network model are peformed here, with CV , OOF based on conditions ''' blue_print = {} oof_prob = np.zeros(train_y.shape) test_prob = np.zeros(test_X.shape[0]) f1_per_fold = [] thresh_per_fold = [] the_models = [] blue_print['model_name'] = model_name splits = list( StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=DATA_SPLIT_SEED).split(train_X, train_y)) validate_per_fold = epoch_per_fold for idx, (train_idx, valid_idx) in enumerate(splits): X_train = train_X[train_idx] y_train = train_y[train_idx] X_val = train_X[valid_idx] y_val = train_y[valid_idx] loss, train_op, prediction_prob, input_x, input_x_len, input_y, input_dropout_keep_prob, dropout_keep_prob, ema = neural_model(network_config) model_dir = 'model_name' model = Model_2(loss_op=loss, train_op=train_op, prediction_op=prediction_prob, input_x=input_x, input_x_len=input_x_len, input_y=input_y, dropout_keep_prob=input_dropout_keep_prob, model_dir=model_dir, early_stop=False, ema_and_assign=ema, use_ema=True, pred_before_ema=False) model.train((X_train, y_train), train_vocab, dropout_keep_prob, epochs=epoch_per_fold[idx]) model.sess.run(model.ema_assign_op) valid_preds, val_scores = model.one_loop_validate(X_val, y_val, train_vocab, batch_size=batch_size) oof_prob[valid_idx] = valid_preds test_predictions = np.array(model.predict(test_X, train_vocab, batch_size=batch_size)) test_prob += test_predictions/len(splits) f1_per_fold.append(val_scores['f1_score']) thresh_per_fold.append(val_scores['thresh']) the_models.append(model) thresh_mean = np.mean(thresh_per_fold) f1_mean = f1_score((oof_prob > thresh_mean).astype(np.int), train_y ) oof_scores = pick_the_best(train_y, oof_prob) thresh_oof = oof_scores[0][0] f1_oof = oof_scores[0][1] blue_print['f1_mean'] = f1_mean blue_print['thresh_mean'] = thresh_mean blue_print['thresh_oof'] = thresh_oof blue_print['f1_oof'] = f1_oof blue_print['oof_prob'] = oof_prob blue_print['the_models'] = the_models blue_print['test_prob'] = test_prob return blue_print #################### Attention Model ################################################## def attention(inputs, attention_size, mode='normal', sentence_sequence_length = None, initializer = tf.initializers.random_normal(stddev=0.1) ): ''' mode: normal , before , after ''' hidden_size = inputs.shape[2].value # D value - hidden size of the RNN layer # Trainable parameters w_omega = tf.get_variable("w_omega", shape=[hidden_size, attention_size],initializer=initializer) b_omega = tf.get_variable("b_omega", shape=[attention_size],initializer=initializer) u_omega = tf.get_variable("u_omega", shape=[attention_size],initializer=initializer) with tf.name_scope('v'): # Applying fully connected layer with non-linear activation to each of the B*T timestamps; # the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size v = tf.tanh(tf.tensordot(inputs, w_omega, axes=1) + b_omega) # For each of the timestamps its vector of size A from `v` is reduced with `u` vector vu = tf.tensordot(v, u_omega, axes=1, name='vu') # (B,T) shape if mode == 'normal': alphas = tf.nn.softmax(vu, name='alphas') # (B,T) shape output = inputs * tf.expand_dims(alphas, -1) return output, alphas if mode == 'before': print("Before softmax") alpha_mask = tf.sequence_mask( sentence_sequence_length, maxlen=tf.shape(vu)[1]) res_after_masking = tf.exp(vu) * tf.cast(alpha_mask, tf.float32) # Numerator of softmax fn row_sum = tf.reduce_sum(res_after_masking, axis=1) row_sum = tf.reshape(row_sum, shape=(-1,1)) # Denominator of softmax function new_alphas = res_after_masking/row_sum output = inputs * tf.expand_dims(new_alphas, -1) return output, new_alphas if mode == 'after': print("After softmax") alphas = tf.nn.softmax(vu, name='alphas') # (B,T) shape zeros = tf.zeros(shape=tf.shape(alphas)) alpha_mask = tf.sequence_mask( sentence_sequence_length, maxlen=tf.shape(alphas)[1]) new_alphas = tf.where(alpha_mask, alphas, zeros) output = inputs * tf.expand_dims(new_alphas, -1) return output, new_alphas def call_my_model(network_config): ######### Define placeholders tf.reset_default_graph() n_classes = network_config['n_classes'] input_x = tf.placeholder(tf.int32, [None, None], name="input_x") input_x_len = tf.placeholder(tf.int32, [None], name="input_x_len") input_y = tf.placeholder(tf.float32, [None], name="input_y") input_dropout_keep_prob = tf.placeholder(tf.float32, name="input_dropout_keep_prob") train_embedding = network_config["train_embedding"] with tf.device('/cpu:0'): Embedding_Matrix = tf.get_variable( name='embedding_weights', shape=train_embedding.shape, initializer=tf.constant_initializer(train_embedding), trainable=False) input_train_embedding = tf.nn.embedding_lookup(Embedding_Matrix, input_x) if network_config["network"] == "rnn" or network_config["network"] == "rnn_cnn": ################################################################################################################# ################################# CUDA RNN ##################################################################### ################################################################################################################# cell_dir = network_config["cell_dir"] # bi and one_side cell_type = network_config["cell_type"] # lstm , num_layers = network_config["rnn_layers"] pooling = network_config["pooling"] rnn_hidden_dimension = network_config["rnn_hidden_dimension"] if cell_type == "lstm": if cell_dir == 'bi': the_cell = cudnn_rnn.CudnnLSTM(num_layers=num_layers, num_units=rnn_hidden_dimension, direction=cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) else: the_cell = cudnn_rnn.CudnnLSTM(num_layers=num_layers, num_units=rnn_hidden_dimension, direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) if cell_type == "gru": if cell_dir == 'bi': the_cell = cudnn_rnn.CudnnGRU(num_layers=1, num_units=rnn_hidden_dimension, direction=cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) else: the_cell = cudnn_rnn.CudnnGRU(num_layers=1, num_units=rnn_hidden_dimension, direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) rnn_outputs, _ = the_cell(tf.transpose(input_train_embedding, perm=[1, 0, 2])) rnn_outputs = tf.transpose(rnn_outputs, perm=[1, 0, 2]) rnn_outputs = tf.layers.dropout(rnn_outputs, rate=input_dropout_keep_prob) rnn_outputs = tf.identity(rnn_outputs, name="rnn_outputs") if network_config["use_attention"]: conc_attn = network_config["conc_attn"] attention_size = network_config["attention_size"] rnn_outputs_attention , attention_vector = attention(rnn_outputs, attention_size, mode = 'before', sentence_sequence_length = input_x_len) if cell_dir == 'one_side': rnn_state_dimension = rnn_hidden_dimension else: rnn_state_dimension = 2 * rnn_hidden_dimension rnn_states_attention = tf.reduce_sum(rnn_outputs_attention, axis=1) if conc_attn == False: # we use only attention as final feature rnn_states = rnn_states_attention else: if pooling == 'max_pool': rnn_states = tf.reduce_max(rnn_outputs, axis=1) rnn_states = tf.concat([rnn_states, rnn_states_attention], axis=1) rnn_state_dimension = rnn_state_dimension + rnn_state_dimension if pooling == 'average_pool': rnn_states = tf.reduce_mean(rnn_outputs, axis=1) rnn_states = tf.concat([rnn_states, rnn_states_attention], axis=1) rnn_state_dimension = rnn_state_dimension + rnn_state_dimension if pooling == 'max_average': rnn_states_1 = tf.reduce_max(rnn_outputs, axis=1) rnn_states_2 = tf.reduce_mean(rnn_outputs, axis=1) rnn_states = tf.concat([rnn_states_1, rnn_states_2, rnn_states_attention], axis=1) rnn_state_dimension = 2*rnn_state_dimension + rnn_state_dimension penultimate_hidden_dimension = rnn_state_dimension penultimate_hidden_state = rnn_states if network_config["network"] == "rnn": if cell_dir == 'bi': rnn_state_dimension = 2*rnn_hidden_dimension else: rnn_state_dimension = rnn_hidden_dimension if pooling == 'max_pool': rnn_states = tf.reduce_max(rnn_outputs, axis=1) rnn_state_dimension = rnn_state_dimension if pooling == 'average_pool': rnn_states = tf.reduce_mean(rnn_outputs, axis=1) rnn_state_dimension = rnn_state_dimension if pooling == 'max_average': rnn_states_1 = tf.reduce_max(rnn_outputs, axis=1) rnn_states_2 = tf.reduce_mean(rnn_outputs, axis=1) rnn_states = tf.concat([rnn_states_1, rnn_states_2], axis=1) rnn_state_dimension = 2*rnn_state_dimension penultimate_hidden_dimension = rnn_state_dimension penultimate_hidden_state = rnn_states if network_config["network"] == "cnn" or network_config["network"] == "rnn_cnn": if network_config["network"] == "rnn_cnn": if network_config["use_attention"]: cnn_input_train_embedding = rnn_outputs_attention embedding_size = rnn_state_dimension else: cnn_input_train_embedding = rnn_outputs if cell_dir == 'bi': embedding_size = 2*rnn_hidden_dimension else: embedding_size = rnn_hidden_dimension if network_config["network"] == "cnn": cnn_input_train_embedding = input_train_embedding embedding_size = 300 layers = [] n_filters = network_config["n_filters"] kernel_sizes = network_config["kernel_sizes"] conv_pooling = network_config["conv_pooling"] pooled_outputs = [] for i in range(len(n_filters)): filter_size = kernel_sizes[i] num_filters = n_filters[i] with tf.variable_scope("conv-maxpool-%s" % filter_size): filter_shape = [filter_size, embedding_size, num_filters] W = tf.get_variable("W_{}".format(filter_size), shape=filter_shape, initializer=tf.contrib.layers.xavier_initializer()) b = tf.get_variable("bias_{}".format(filter_size), shape=[num_filters], initializer=tf.initializers.constant(0.1)) conv = tf.nn.conv1d( cnn_input_train_embedding, W, stride=1, padding="SAME", name="conv_valid") conv1 = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu") if conv_pooling == 'max_pool': pooled = tf.reduce_max(conv1 , axis=1) pooled_outputs.append(pooled) if conv_pooling == 'average_pool': pooled = tf.reduce_mean(conv1 , axis=1) pooled_outputs.append(pooled) if conv_pooling == 'max_average': pooled_max = tf.reduce_max(conv1 , axis=1) pooled_mean = tf.reduce_max(conv1 , axis=1) pooled_outputs.append(tf.concat([pooled_max, pooled_mean], axis=1)) if conv_pooling == "max_pool": conv_features = tf.concat(pooled_outputs, axis=1) penultimate_hidden_dimension = np.sum(n_filters)# total filter shape sum penultimate_hidden_state = tf.nn.dropout(conv_features, keep_prob = input_dropout_keep_prob) if conv_pooling == "average_pool": conv_features = tf.concat(pooled_outputs, axis=1) penultimate_hidden_dimension = np.sum(n_filters)# total filter shape sum penultimate_hidden_state = tf.nn.dropout(conv_features, keep_prob = input_dropout_keep_prob) if conv_pooling == "max_average": conv_features = tf.concat(pooled_outputs, axis=1) penultimate_hidden_dimension = 2*(np.sum(n_filters))# total filter shape sum penultimate_hidden_state = tf.nn.dropout(conv_features, keep_prob = input_dropout_keep_prob) # print(rnn_states, rnn_states_attention, penultimate_hidden_dimension, penultimate_hidden_state) if network_config['projection_config']: projection_layer_dimensions = network_config['projection_config']['extra_projection'] W_projection, b_projection = initialize_projection_weight(projection_layer_dimensions, penultimate_hidden_dimension, scope='extra_projection') penultimate_hidden_state = evaluate_extra_projection(W_projection, b_projection, projection_layer_dimensions, penultimate_hidden_state) penultimate_hidden_dimension = projection_layer_dimensions[-1][0] W_out, b_out = initialize_output_weight(penultimate_hidden_dimension, n_classes, scope='output', initializer=tf.contrib.layers.xavier_initializer()) ##################### Selecting weights l2 / l1 if network_config['beta'] > 0.0: # if beta = 0 , regularization = 0 if network_config['apply_reg_rnn'] is False: reg_applying_weights = [] for tf_var_ in tf.trainable_variables(): found = 0 for var_ in ['rnn' , 'lstm' , 'gru' , 'cnn' , 'bias']: if var_ in tf_var_.name: found = 1 break if found == 0: reg_applying_weights.append(tf_var_) else: reg_applying_weights = [tf_var for tf_var in tf.trainable_variables() if 'bias' not in tf_var.name] with tf.name_scope("output"): logits = tf.nn.xw_plus_b(penultimate_hidden_state, W_out, b_out, name='logits') if network_config['beta'] == 0: loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss')) else: if network_config['regularization'] == 'l2': loss_logits = tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss') l2_loss_calculated = network_config['beta'] * tf.add_n([tf.nn.l2_loss(var_) for var_ in reg_applying_weights]) loss = tf.reduce_mean(loss_logits + l2_loss_calculated) if network_config['regularization'] == 'l1': loss_logits = tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss') l1_loss_tf = tf.contrib.layers.l1_regularizer( scale=network_config['beta'], scope=None ) l1_loss_calculated = network_config['beta'] * tf.add_n([l1_loss_tf(var_) for var_ in reg_applying_weights]) loss = tf.reduce_mean(loss_logits + l1_loss_calculated) prediction_prob = tf.nn.sigmoid(tf.squeeze(logits), name='prediction_prob') with tf.name_scope("otpimizer"): global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) gradients, variables = zip(*optimizer.compute_gradients(loss)) gradients, _ = tf.clip_by_global_norm(gradients, 5.0) train_op_opt = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) with tf.name_scope("ema_"): tvar = tf.trainable_variables() MOVING_AVERAGE_DECAY = 0.99 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tvar) train_op = tf.group(train_op_opt, ema_op) # with tf.control_dependencies([train_op]): ########## Hack for assign ema values to original values before prediction tf_all_var_dict = {var_.name: var_ for var_ in tf.all_variables()} all_assign_ops = [] for var_ in tvar: var_raw = var_.name.replace(':0', '') var_looking = var_raw + '/ExponentialMovingAverage:0' if var_looking in tf_all_var_dict: print("Found" , var_raw , var_looking) all_assign_ops.append(tf.assign(var_ , tf_all_var_dict[var_looking])) all_assign_op = tf.group(all_assign_ops) return loss, train_op, prediction_prob , input_x, input_x_len, input_y, input_dropout_keep_prob ,network_config['dropout_keep_prob'], (ema, all_assign_op) def call_my_model_2(network_config): ######### Define placeholders tf.reset_default_graph() n_classes = 1 input_x = tf.placeholder(tf.int32, [None, None], name="input_x") input_x_len = tf.placeholder(tf.int32, [None], name="input_x_len") input_y = tf.placeholder(tf.float32, [None], name="input_y") input_dropout_keep_prob = tf.placeholder(tf.float32, name="input_dropout_keep_prob") train_embedding = network_config["train_embedding"] with tf.device('/cpu:0'): Embedding_Matrix = tf.get_variable( name='embedding_weights', shape=train_embedding.shape, initializer=tf.constant_initializer(train_embedding), trainable=False) input_train_embedding = tf.nn.embedding_lookup(Embedding_Matrix, input_x) lstm_cell = cudnn_rnn.CudnnLSTM(num_layers=1, num_units=40, direction=cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) gru_cell = cudnn_rnn.CudnnGRU(num_layers=1, num_units=40, direction=cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) rnn_outputs_1, _ = lstm_cell(tf.transpose(input_train_embedding, perm=[1, 0, 2])) rnn_outputs_1 = tf.transpose(rnn_outputs_1, perm=[1, 0, 2]) rnn_outputs_1 = tf.layers.dropout(rnn_outputs_1, rate=input_dropout_keep_prob) rnn_outputs_1 = tf.identity(rnn_outputs_1, name="rnn_outputs_1") rnn_outputs_2, _ = gru_cell(tf.transpose(input_train_embedding, perm=[1, 0, 2])) rnn_outputs_2 = tf.transpose(rnn_outputs_2, perm=[1, 0, 2]) rnn_outputs_2 = tf.layers.dropout(rnn_outputs_2, rate=input_dropout_keep_prob) rnn_outputs_2 = tf.identity(rnn_outputs_2, name="rnn_outputs_2") with tf.variable_scope("attention"): rnn_outputs_attention_1 , attention_vector_1 = attention(rnn_outputs_1, 40, mode = 'before', sentence_sequence_length = input_x_len) rnn_outputs_attention_1 = tf.reduce_sum(rnn_outputs_attention_1, axis=1) with tf.variable_scope("attention", reuse=True): rnn_outputs_attention_2 , attention_vector_2 = attention(rnn_outputs_2, 40, mode = 'before', sentence_sequence_length = input_x_len) rnn_outputs_attention_2 = tf.reduce_sum(rnn_outputs_attention_2, axis=1) max_gru = tf.reduce_max(rnn_outputs_2, axis=1) avg_gru = tf.reduce_mean(rnn_outputs_2, axis=1) penultimate_hidden_dimension = 320 penultimate_hidden_state = tf.concat([rnn_outputs_attention_1, rnn_outputs_attention_2, max_gru, avg_gru], axis=1) projection_layer_dimensions = [(16, 'relu')] W_projection, b_projection = initialize_projection_weight(projection_layer_dimensions, penultimate_hidden_dimension, scope='extra_projection') penultimate_hidden_state = evaluate_extra_projection(W_projection, b_projection, projection_layer_dimensions, penultimate_hidden_state) penultimate_hidden_dimension = projection_layer_dimensions[-1][0] W_out, b_out = initialize_output_weight(penultimate_hidden_dimension, n_classes, scope='output', initializer=tf.contrib.layers.xavier_initializer()) ##################### Selecting weights l2 / l1 if network_config['beta'] > 0.0: # if beta = 0 , regularization = 0 if network_config['apply_reg_rnn'] is False: reg_applying_weights = [] for tf_var_ in tf.trainable_variables(): found = 0 for var_ in ['rnn' , 'lstm' , 'gru' , 'cnn' , 'bias']: if var_ in tf_var_.name: found = 1 break if found == 0: reg_applying_weights.append(tf_var_) else: reg_applying_weights = [tf_var for tf_var in tf.trainable_variables() if 'bias' not in tf_var.name] with tf.name_scope("output"): logits = tf.nn.xw_plus_b(penultimate_hidden_state, W_out, b_out, name='logits') if network_config['beta'] == 0: loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss')) else: if network_config['regularization'] == 'l2': loss_logits = tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss') l2_loss_calculated = network_config['beta'] * tf.add_n([tf.nn.l2_loss(var_) for var_ in reg_applying_weights]) loss = tf.reduce_mean(loss_logits + l2_loss_calculated) if network_config['regularization'] == 'l1': loss_logits = tf.nn.sigmoid_cross_entropy_with_logits(labels=input_y, logits=tf.squeeze(logits), name='loss') l1_loss_tf = tf.contrib.layers.l1_regularizer( scale=network_config['beta'], scope=None ) l1_loss_calculated = network_config['beta'] * tf.add_n([l1_loss_tf(var_) for var_ in reg_applying_weights]) loss = tf.reduce_mean(loss_logits + l1_loss_calculated) prediction_prob = tf.nn.sigmoid(tf.squeeze(logits), name='prediction_prob') with tf.name_scope("otpimizer"): global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) gradients, variables = zip(*optimizer.compute_gradients(loss)) gradients, _ = tf.clip_by_global_norm(gradients, 5.0) train_op_opt = optimizer.apply_gradients(zip(gradients, variables), global_step=global_step) with tf.name_scope("ema_"): tvar = tf.trainable_variables() MOVING_AVERAGE_DECAY = 0.99 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tvar) train_op = tf.group(train_op_opt, ema_op) # with tf.control_dependencies([train_op]): ########## Hack for assign ema values to original values before prediction tf_all_var_dict = {var_.name: var_ for var_ in tf.all_variables()} all_assign_ops = [] for var_ in tvar: var_raw = var_.name.replace(':0', '') var_looking = var_raw + '/ExponentialMovingAverage:0' if var_looking in tf_all_var_dict: print("Found" , var_raw , var_looking) all_assign_ops.append(tf.assign(var_ , tf_all_var_dict[var_looking])) all_assign_op = tf.group(all_assign_ops) return loss, train_op, prediction_prob , input_x, input_x_len, input_y, input_dropout_keep_prob ,network_config['dropout_keep_prob'], (ema, all_assign_op) from sklearn.model_selection import train_test_split DATA_SPLIT_SEED = 2018 # train_df_sampled, val_df = train_test_split(train_df, test_size=0.08, random_state=2018) # train_X = train_df_sampled['tokens_index'] # train_y = train_df_sampled['target'] # val_X = val_df['tokens_index'] # val_Y = val_df['target'] test_X = test_df['tokens_index'] train_X = train_df['tokens_index'] train_y = train_df['target'] test_X_Keras = test_df['tokens_index_keras'] train_X_Keras = train_df['tokens_index_keras'] test_X_reverse = test_df['tokens_index_reverse'] train_X_reverse = train_df['tokens_index_reverse'] test_X_Keras_reverse = test_df['tokens_index_keras_reverse'] train_X_Keras_reverse = train_df['tokens_index_keras_reverse'] train_y = train_df['target'] all_models = [] network_config = { "network" : "rnn" , # [rnn , cnn , rnn_cnn] "cell_type" : "gru" , # ['lstm' , 'gru'] "rnn_layers" : 1, "cell_dir" : "one_side", # ['one_side', 'bi'], "rnn_hidden_dimension" : 256, "dropout_keep_prob" : 0.7, "pool":"max_average", # max_pool , avergae_pool "projection_config" : {}, #{'use_extra_projection' : True , # bool 'extra_projection' : [(256, 'relu'),(128, 'relu')]} "batch_size" : 512, "n_classes" : 1, "regularization" : 'l1', "beta" : 0.0001, "apply_reg_rnn": False, "train_embedding": train_embedding, "use_attention": False, "attention_size": 256, "conc_attn": False, "pooling": "max_pool", "n_filters" : [100, 100, 100], "kernel_sizes": [1,2,3], "conv_pooling": "max_pool" } ##### LSTM Model mean emebdding start_time = time.time() model_name = 'gru_model_mean' epoch_per_fold = [5,5,4,4,4] model = neural_blender_oof(neural_model=call_my_model, model_name=model_name, network_config=network_config, train_X = train_X, train_y = train_y, test_X = test_X, n_folds=5, epoch_per_fold=epoch_per_fold, batch_size = 1024 ) end_time = time.time() print('Best F1: {:.4f} at threshold: {:.4f}'.format(model['f1_mean'], model['thresh_mean'])) print("Model timing {} seconds".format(end_time-start_time)) all_models.append(model) gc.collect() if model['thresh_mean'] > 0.33: model['thresh_mean'] = 0.32 #### Compressing the best model sub = pd.read_csv('../input/sample_submission.csv') test_predictions_class = (model['test_prob'] > model['thresh_mean']).astype(np.int) sub.prediction = test_predictions_class sub.to_csv("submission.csv", index=False)