{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nimport random\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom nltk.stem.wordnet import WordNetLemmatizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-31T04:19:30.937355Z","iopub.execute_input":"2021-05-31T04:19:30.937765Z","iopub.status.idle":"2021-05-31T04:19:30.945665Z","shell.execute_reply.started":"2021-05-31T04:19:30.937695Z","shell.execute_reply":"2021-05-31T04:19:30.944748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#设置随机种子保证可重复性\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\nseed_everything()","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-05-31T04:19:30.947207Z","iopub.execute_input":"2021-05-31T04:19:30.947529Z","iopub.status.idle":"2021-05-31T04:19:30.96223Z","shell.execute_reply.started":"2021-05-31T04:19:30.947467Z","shell.execute_reply":"2021-05-31T04:19:30.961367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntrain, test_df= train_test_split(train, test_size=0.1, random_state=2018)\ntrain, val_df= train_test_split(train, test_size=0.1, random_state=2018)\n\ntrain[\"question_text\"] = train[\"question_text\"].str.lower()\ntest[\"question_text\"] = test[\"question_text\"].str.lower()\ntest_df[\"question_text\"] = test_df[\"question_text\"].str.lower()\ndef clean_tag(text):\n    if '[math]' in text:\n        text = re.sub('\\[math\\].*?math\\]', '[formula]', text)\n    if 'http' in text or 'www' in text:\n        text = re.sub('(?:(?:https?|ftp):\\/\\/)?[\\w/\\-?=%.]+\\.[\\w/\\-?=%.]+', '[url]', text)\n    return text\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_tag(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_tag(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_tag(x))","metadata":{"_uuid":"630b748c199117eda0f44974cdf3430bdac2db99","execution":{"iopub.status.busy":"2021-05-31T04:19:30.963633Z","iopub.execute_input":"2021-05-31T04:19:30.964114Z","iopub.status.idle":"2021-05-31T04:19:37.829723Z","shell.execute_reply.started":"2021-05-31T04:19:30.964066Z","shell.execute_reply":"2021-05-31T04:19:37.828899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ntrain_y = train['target']\ntest_y=test_df['target']\ntrain_y.value_counts().plot(kind='pie')\n","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:19:37.83097Z","iopub.execute_input":"2021-05-31T04:19:37.831237Z","iopub.status.idle":"2021-05-31T04:19:37.935151Z","shell.execute_reply.started":"2021-05-31T04:19:37.831193Z","shell.execute_reply":"2021-05-31T04:19:37.934407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"puncts=[',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']","metadata":{"_uuid":"f897897a7db3b0e3d58618ab4585262786fb117d","execution":{"iopub.status.busy":"2021-05-31T04:19:37.936465Z","iopub.execute_input":"2021-05-31T04:19:37.936929Z","iopub.status.idle":"2021-05-31T04:19:38.170571Z","shell.execute_reply.started":"2021-05-31T04:19:37.936879Z","shell.execute_reply":"2021-05-31T04:19:38.169625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef clean_punct(x):\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_punct(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_punct(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_punct(x))","metadata":{"_uuid":"1532cfc0a14ca4e77a95a985a3efd503bc7a853b","execution":{"iopub.status.busy":"2021-05-31T04:19:38.171776Z","iopub.execute_input":"2021-05-31T04:19:38.172039Z","iopub.status.idle":"2021-05-31T04:20:21.47024Z","shell.execute_reply.started":"2021-05-31T04:19:38.171998Z","shell.execute_reply":"2021-05-31T04:20:21.469497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 200000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use #99.99%\n\n## fill up the missing values\nX = train[\"question_text\"].fillna(\"_####_\").values\nX_test = test_df[\"question_text\"].fillna(\"_####_\").values\nprint(X)\nprint(X[0])\nprint(X.shape)\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features, filters='')\ntokenizer.fit_on_texts(list(X)+list(X_test))\n\nX = tokenizer.texts_to_sequences(X)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nprint(X[0])\nprint(len(X))\n## Pad the sentences \nX = pad_sequences(X, maxlen=maxlen)\nX_test = pad_sequences(X_test, maxlen=maxlen)\n\n## Get the target values\nY = train['target'].values\n\nsub = test[['qid']]\ndel train, test,test_df\ngc.collect()","metadata":{"_uuid":"1a19b23e79ad011078b200196dbc9a2a905347a8","execution":{"iopub.status.busy":"2021-05-31T04:20:21.471401Z","iopub.execute_input":"2021-05-31T04:20:21.471851Z","iopub.status.idle":"2021-05-31T04:21:10.6134Z","shell.execute_reply.started":"2021-05-31T04:20:21.471801Z","shell.execute_reply":"2021-05-31T04:21:10.612224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lem = WordNetLemmatizer()\nword_index = tokenizer.word_index\nmax_features = len(word_index)+1\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/glove840b300dtxt/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) \n                            for o in open(EMBEDDING_FILE) \n                            if o.split(\" \")[0] in word_index or o.split(\" \")[0].lower() in word_index)\n\n    emb_mean, emb_std = -0.005838499, 0.48782197\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n        elif embeddings_index.get(word.capitalize()) is not None:\n            embedding_matrix[i] = embeddings_index.get(word.capitalize())\n        elif embeddings_index.get(word.upper()) is not None:\n            embedding_matrix[i] = embeddings_index.get(word.upper())\n    del embeddings_index\n    gc.collect()        \n    return embedding_matrix \n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/paragram-300-sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) \n                            for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') \n                            if len(o)>100 and (o.split(\" \")[0] in word_index or o.split(\" \")[0].lower() in word_index))\n\n    emb_mean, emb_std = -0.005838499, 0.48782197\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n        elif embeddings_index.get(word.capitalize()) is not None:\n            embedding_matrix[i] = embeddings_index.get(word.capitalize())\n        elif embeddings_index.get(word.upper()) is not None:\n            embedding_matrix[i] = embeddings_index.get(word.upper())\n        \n    del embeddings_index\n    gc.collect()\n    return embedding_matrix\n\nseed_everything()\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_3 = load_para(word_index)\nembedding_matrix = np.mean((1.28*embedding_matrix_1, 0.72*embedding_matrix_3), axis=0)\ndel embedding_matrix_1, embedding_matrix_3\ngc.collect()\nnp.shape(embedding_matrix)","metadata":{"_uuid":"e14b6f8d1220022039461d2ce1b41b5f07ae13c2","execution":{"iopub.status.busy":"2021-05-31T04:21:10.614875Z","iopub.execute_input":"2021-05-31T04:21:10.615311Z","iopub.status.idle":"2021-05-31T04:24:47.065756Z","shell.execute_reply.started":"2021-05-31T04:21:10.615124Z","shell.execute_reply":"2021-05-31T04:24:47.0643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttentionWeightedAverage(Layer):\n    \"\"\"\n    Computes a weighted average of the different channels across timesteps.\n    Uses 1 parameter pr. channel to compute the attention value for a single timestep.\n    \"\"\"\n\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.RandomUniform(seed=10000)\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        self.trainable_weights = [self.W]\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        # computes a probability distribution over the timesteps\n        # uses 'max trick' for numerical stability\n        # reshape is done to avoid issue with Tensorflow\n        # and 1-dimensional weights\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        # masked timesteps have zero weight\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None\nclass AdamW(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,  # decoupled weight decay (1/4)\n                 epsilon=1e-8, decay=0., **kwargs):\n        super(AdamW, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.wd = K.variable(weight_decay, name='weight_decay') # decoupled weight decay (2/4)\n        self.epsilon = epsilon\n        self.initial_decay = decay\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n        wd = self.wd # decoupled weight decay (3/4)\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr *= (1. / (1. + self.decay * K.cast(self.iterations,\n                                                  K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        self.weights = [self.iterations] + ms + vs\n\n        for p, g, m, v in zip(params, grads, ms, vs):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) - lr * wd * p # decoupled weight decay (4/4)\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'weight_decay': float(K.get_value(self.wd)),\n                  'epsilon': self.epsilon}\n        base_config = super(AdamW, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"_uuid":"7ad4ebd22ef80fb18cb9ace15a08ffad96b27272","execution":{"iopub.status.busy":"2021-05-31T04:24:47.067062Z","iopub.execute_input":"2021-05-31T04:24:47.067321Z","iopub.status.idle":"2021-05-31T04:24:47.090522Z","shell.execute_reply.started":"2021-05-31T04:24:47.067276Z","shell.execute_reply":"2021-05-31T04:24:47.089567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def LSTM_GRU(spatialdropout=0.20, rnn_units=64, weight_decay=0.07):\n    K.clear_session()       \n    x_input = Input(shape=(maxlen,))\n    \n    emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding')(x_input)\n    emb = SpatialDropout1D(spatialdropout, seed=1024)(emb)\n\n    rnn1 = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111100), \n                           recurrent_initializer=Orthogonal(gain=1.0, seed=123000)))(emb)\n    rnn2 = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111000), \n                           recurrent_initializer=Orthogonal(gain=1.0, seed=1203000)))(rnn1)\n\n    x = concatenate([rnn1, rnn2])\n    x = GlobalMaxPooling1D()(x)  \n    x_output = Dense(1, activation='sigmoid', kernel_initializer=glorot_uniform(seed=111100))(x)\n    \n    model = Model(inputs=x_input, outputs=x_output)\n    model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay),)\n    return model","metadata":{"_uuid":"537c1d32911461003ef6136908612e1a5df92440","execution":{"iopub.status.busy":"2021-05-31T04:24:47.091892Z","iopub.execute_input":"2021-05-31T04:24:47.092216Z","iopub.status.idle":"2021-05-31T04:24:47.104891Z","shell.execute_reply.started":"2021-05-31T04:24:47.092154Z","shell.execute_reply":"2021-05-31T04:24:47.104223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM_GRU(spatialdropout=0.20, rnn_units=64, weight_decay=0.07)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:24:47.106108Z","iopub.execute_input":"2021-05-31T04:24:47.106586Z","iopub.status.idle":"2021-05-31T04:24:48.118816Z","shell.execute_reply.started":"2021-05-31T04:24:47.106481Z","shell.execute_reply":"2021-05-31T04:24:48.118048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#epoch=7\ndef poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04):\n    K.clear_session()\n    inp = Input(shape=(maxlen,))\n    embedding_layer = Embedding(max_features,\n                                embed_size,\n                                weights=[embedding_matrix],\n                                input_length=maxlen,\n                                trainable=False)(inp)\n    embedding_layer = SpatialDropout1D(spatialdropout, seed=1024)(embedding_layer)\n\n    rnn_1 = Bidirectional(CuDNNGRU(gru_units, return_sequences=True, \n                                   kernel_initializer=glorot_uniform(seed=10000), \n                                   recurrent_initializer=Orthogonal(gain=1.0, seed=123000)))(embedding_layer)\n\n    last = Lambda(lambda t: t[:, -1], name='last')(rnn_1)\n    maxpool = GlobalMaxPooling1D()(rnn_1)\n    attn = AttentionWeightedAverage()(rnn_1)\n    average = GlobalAveragePooling1D()(rnn_1)\n\n    c = concatenate([last, maxpool, attn], axis=1)\n    c = Reshape((3, -1))(c)\n    c = Lambda(lambda x:K.sum(x, axis=1))(c)\n    x = BatchNormalization()(c)\n    x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(x)\n    x = Dropout(0.2, seed=1024)(x)\n    x = BatchNormalization()(x)\n    output_layer = Dense(1, activation=\"sigmoid\", kernel_initializer=glorot_uniform(seed=111000))(x)\n    model = Model(inputs=inp, outputs=output_layer)\n    model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))\n    return model","metadata":{"_uuid":"57af6fcd3fe1d3cd87e6f10f0f51c25fb7779e24","execution":{"iopub.status.busy":"2021-05-31T04:24:48.120065Z","iopub.execute_input":"2021-05-31T04:24:48.120321Z","iopub.status.idle":"2021-05-31T04:24:48.130465Z","shell.execute_reply.started":"2021-05-31T04:24:48.120279Z","shell.execute_reply":"2021-05-31T04:24:48.129711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04)\nmodel.save('modell.h5')\n\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:24:48.132218Z","iopub.execute_input":"2021-05-31T04:24:48.13267Z","iopub.status.idle":"2021-05-31T04:24:49.631921Z","shell.execute_reply.started":"2021-05-31T04:24:48.132483Z","shell.execute_reply":"2021-05-31T04:24:49.631313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 80, 30, 12], weight_decay=0.10):\n    K.clear_session()       \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(rate=spatialdropout, seed=10000)(x)\n    x = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, \n                               kernel_initializer=glorot_uniform(seed=111000), \n                               recurrent_initializer=Orthogonal(gain=1.0, seed=123000)))(x)\n\n    x1 = Conv1D(filters=filters[0], activation='relu', kernel_size=1, \n                padding='same', kernel_initializer=glorot_uniform(seed=110000))(x)\n    x2 = Conv1D(filters=filters[1], activation='relu', kernel_size=2, \n                padding='same', kernel_initializer=glorot_uniform(seed=120000))(x)\n    x3 = Conv1D(filters=filters[2], activation='relu', kernel_size=3, \n                padding='same', kernel_initializer=glorot_uniform(seed=130000))(x)\n    x4 = Conv1D(filters=filters[3], activation='relu', kernel_size=5, \n                padding='same', kernel_initializer=glorot_uniform(seed=140000))(x)\n\n    \n    x1 = GlobalMaxPool1D()(x1)\n    x2 = GlobalMaxPool1D()(x2)\n    x3 = GlobalMaxPool1D()(x3)\n    x4 = GlobalMaxPool1D()(x4)\n\n    c = concatenate([x1, x2, x3, x4])\n    x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(c)\n    x = Dropout(0.2, seed=10000)(x)\n    x = BatchNormalization()(x)\n    x = Dense(1, activation=\"sigmoid\", kernel_initializer=glorot_uniform(seed=110000))(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))\n    return model","metadata":{"_uuid":"537c1d32911461003ef6136908612e1a5df92440","execution":{"iopub.status.busy":"2021-05-31T04:24:49.633164Z","iopub.execute_input":"2021-05-31T04:24:49.633452Z","iopub.status.idle":"2021-05-31T04:24:49.64338Z","shell.execute_reply.started":"2021-05-31T04:24:49.633389Z","shell.execute_reply":"2021-05-31T04:24:49.642611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 90, 30, 12], weight_decay=0.10)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:24:49.644481Z","iopub.execute_input":"2021-05-31T04:24:49.644747Z","iopub.status.idle":"2021-05-31T04:24:50.822659Z","shell.execute_reply.started":"2021-05-31T04:24:49.644704Z","shell.execute_reply":"2021-05-31T04:24:50.821956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#epoch=7\ndef singleRNN(spatialdropout=0.20, rnn_units=120, weight_decay=0.08):\n    K.clear_session()\n    inp = Input(shape=(maxlen,))\n    embedding_layer = Embedding(max_features,\n                                embed_size,\n                                weights=[embedding_matrix],\n                                input_length=maxlen,\n                                trainable=False)(inp)\n    embedding_layer = SpatialDropout1D(spatialdropout, seed=1024)(embedding_layer)\n\n    x = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, \n                                   kernel_initializer=glorot_uniform(seed=111000), \n                                   recurrent_initializer=Orthogonal(gain=1.0, seed=123000)))(embedding_layer)\n\n    x = AttentionWeightedAverage()(x)\n    x = Dense(100, kernel_initializer=glorot_uniform(seed=111000))(x)\n    x = Dropout(0.12, seed=1024)(x)\n    x = BatchNormalization()(x)\n    x = ReLU()(x)\n    output_layer = Dense(1, activation=\"sigmoid\", kernel_initializer=glorot_uniform(seed=111000))(x)\n    model = Model(inputs=inp, outputs=output_layer)\n    model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))\n    return model","metadata":{"_uuid":"e847781bd37df1dc9b2a2d5c9993430403283d42","execution":{"iopub.status.busy":"2021-05-31T04:24:50.823899Z","iopub.execute_input":"2021-05-31T04:24:50.824187Z","iopub.status.idle":"2021-05-31T04:24:50.834484Z","shell.execute_reply.started":"2021-05-31T04:24:50.824143Z","shell.execute_reply":"2021-05-31T04:24:50.831951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f1_smart(y_true, y_pred):\n    args = np.argsort(y_pred)\n    tp = y_true.sum()\n    fs = (tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)\n    print(fs)\n    print(np.arange(y_true.shape[0] + tp - 1, tp, -1))\n    res_idx = np.argmax(fs)\n    print(2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2)\n    return 2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2","metadata":{"_uuid":"0ae2e6c8459a69b7279de65ce43287f2393360d6","execution":{"iopub.status.busy":"2021-05-31T05:55:52.03246Z","iopub.execute_input":"2021-05-31T05:55:52.032738Z","iopub.status.idle":"2021-05-31T05:55:52.038672Z","shell.execute_reply.started":"2021-05-31T05:55:52.032689Z","shell.execute_reply":"2021-05-31T05:55:52.037856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_smart(np.array([0,1,1]),np.array([0.2,0.4,0.5]))","metadata":{"execution":{"iopub.status.busy":"2021-05-31T05:55:54.70321Z","iopub.execute_input":"2021-05-31T05:55:54.703506Z","iopub.status.idle":"2021-05-31T05:55:54.711267Z","shell.execute_reply.started":"2021-05-31T05:55:54.703445Z","shell.execute_reply":"2021-05-31T05:55:54.710477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=7, random_state=10, shuffle=True)\nbestscore = []\nbestloss = []\ny_test = np.zeros((X_test.shape[0], ))\noof = np.zeros((X.shape[0], ))\nepochs = [8,8,7,6]\nval_list = []\nfor i, (train_index, valid_index) in enumerate(kfold.split(X, Y)):\n    val_list += list(valid_index)\n    print('FOLD%s'%(i+1))\n    X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]\n    \n    filepath=\"weights_best.h5\"\n    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1, min_lr=0.0, verbose=0)\n    callbacks = [checkpoint, reduce_lr]\n    if i == 0:\n        model = singleRNN(spatialdropout=0.20, rnn_units=120, weight_decay=0.08)\n        print('LSTM_GRU(spatialdropout=0.20, rnn_units=64, weight_decay=0.07)')\n    elif i == 1:\n        break\n        model = poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04)\n        print('poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04)')\n    elif i == 2:\n        break\n        model = BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 90, 30, 12], weight_decay=0.10)\n        print('BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 90, 30, 12], weight_decay=0.10)')\n    model.fit(X_train, Y_train, batch_size=512, epochs=8, \n              validation_data=(X_val, Y_val), verbose=0, callbacks=callbacks, \n              #class_weight={0:1, 1:1.25}\n             )\n    final=model.predict([X_test], batch_size=1024, verbose=2)\n    final=final.reshape((-1,1))\n    \n    print(\"train logloss:%s\"%model.history.history['loss'])\n    print(\"val logloss:%s\"%model.history.history['val_loss'])\n    y_pred = model.predict([X_val], batch_size=1024, verbose=2)\n    y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2))/3\n    oof[valid_index] = np.squeeze(y_pred)\n    f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))\n    final=(final>threshold).astype(int)\n    print(\"Accuracy on test data created\",metrics.f1_score(test_y,final))\n    print('Optimal F1: {:.4f} at threshold: {:.4f}\\n'.format(f1, threshold))\n    bestscore.append(threshold)\n    if i == 2:break","metadata":{"_uuid":"ce8a7220d850952c6d4e984fb03f5004a20f9f0d","execution":{"iopub.status.busy":"2021-05-31T04:40:15.339023Z","iopub.execute_input":"2021-05-31T04:40:15.339306Z","iopub.status.idle":"2021-05-31T05:08:15.245355Z","shell.execute_reply.started":"2021-05-31T04:40:15.339249Z","shell.execute_reply":"2021-05-31T05:08:15.244506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1, threshold = f1_smart(np.squeeze(Y[val_list]), np.squeeze(oof[val_list]))\nprint('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))","metadata":{"_uuid":"9f0b5ef9202d7067b786146cfe2b30147ed9c4b1","execution":{"iopub.status.busy":"2021-05-31T05:08:58.332819Z","iopub.execute_input":"2021-05-31T05:08:58.333099Z","iopub.status.idle":"2021-05-31T05:08:58.508765Z","shell.execute_reply.started":"2021-05-31T05:08:58.333047Z","shell.execute_reply":"2021-05-31T05:08:58.507992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = y_test.reshape((-1, 1)) \npred_test_y = (y_test>threshold).astype(int) \nprint(\"Accuracy on test data created\",metrics.f1_score(test_y,pred_test_y))\n#sub['prediction'] = pred_test_y \n#sub.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"612dd64eb677c2a55c9369dbd60476bd0e34bd40","execution":{"iopub.status.busy":"2021-05-31T05:09:29.380104Z","iopub.execute_input":"2021-05-31T05:09:29.38039Z","iopub.status.idle":"2021-05-31T05:09:29.405975Z","shell.execute_reply.started":"2021-05-31T05:09:29.380336Z","shell.execute_reply":"2021-05-31T05:09:29.404992Z"},"trusted":true},"execution_count":null,"outputs":[]}]}