{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30021,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, LSTM, Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.callbacks import *\n#from keras import backend as K\n#from keras.engine.topology import Layer\nfrom zipfile import ZipFile\nimport os\nfrom gensim.models import KeyedVectors\nimport re\nimport io\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.optimizers import Optimizer\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport os\nfrom tensorflow.python.keras.layers import Layer\nfrom tensorflow.python.keras import backend as K","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1 = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1['question_text'].isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1['target'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d2 = d1[d1['target'] == 1]\nd2.head()\nx = d2['question_text'].values\nx","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d2 = d1[d1['target'] == 0]\nd2.head()\nx = d2['question_text'].values\nx","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(d1, test_size=0.2, random_state=2018)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train_df['question_text'].values\nval_X = val_df['question_text'].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y = train_df['target'].values\nval_y = val_df['target'].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_X)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=10, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings.zip'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zf = ZipFile(EMBEDDING_FILE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(zf.namelist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Glove Embeddings","metadata":{}},{"cell_type":"code","source":"embeddings_index = dict()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f=zf.open('glove.840B.300d/glove.840B.300d.txt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for line in f:\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:],dtype='float32')\n    embeddings_index[word] = coefs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#n = 300\nn = coefs.shape[0]\nn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = np.zeros((nb_words, n))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for word, i in tokenizer.word_index.items():\n\tembedding_vector = embeddings_index.get(word)\n\tif embedding_vector is not None:\n\t\tembedding_matrix[i] = embedding_vector","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_size = n\nembed_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Wiki News FastText Embeddings","metadata":{}},{"cell_type":"code","source":"embeddings_index = dict()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = zf.open('wiki-news-300d-1M/wiki-news-300d-1M.vec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for line in f:\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:],dtype = 'float32')\n    embeddings_index[word] = coefs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = coefs.shape[0]\nn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = np.zeros((nb_words, n))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        count = count + 1\n        embedding_matrix[i] = embedding_vector","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_size = n\nembed_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=1024, epochs=4, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**How to get words getting matched with the word embeddings**","metadata":{}},{"cell_type":"code","source":"def build_vocab(sentences):\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = d1['question_text'].apply(lambda x: x.split()).values\nsentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab = build_vocab(sentences)\nvocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vocab)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings.zip'\nzf = ZipFile(EMBEDDING_FILE)\nprint(zf.namelist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = zf.open('GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings_index = KeyedVectors.load_word2vec_format(f, binary=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in vocab:\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n\n    return oov","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1[\"question_text\"] = d1[\"question_text\"].apply(lambda x: clean_text(x))\nsentences = d1[\"question_text\"].apply(lambda x: x.split())\nvocab = build_vocab(sentences)\nvocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1[\"question_text\"] = d1[\"question_text\"].apply(lambda x: clean_numbers(x))\nsentences = d1[\"question_text\"].apply(lambda x: x.split())\nvocab = build_vocab(sentences)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n                }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mispellings, mispellings_re = _get_mispell(mispell_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mispellings_re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_typical_misspell(text):\n    \n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1[\"question_text\"] = d1[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\nsentences = d1[\"question_text\"].apply(lambda x: x.split())\nto_remove = ['a','to','of','and']\nsentences = [[word for word in sentence if not word in to_remove] for sentence in sentences]\nvocab = build_vocab(sentences)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Checking on Glove after words improvement**","metadata":{}},{"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings.zip'\nzf = ZipFile(EMBEDDING_FILE)\nf=zf.open('glove.840B.300d/glove.840B.300d.txt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings_index = dict()\nfor line in f:\n    line = line.decode('latin')\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:],dtype='float32')\n    embeddings_index[word] = coefs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = coefs.shape[0]\nn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_vocab(sentences):\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = d1['question_text'].apply(lambda x: x.split()).values\nsentences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab = build_vocab(sentences)\nvocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in vocab:\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n\n    return oov","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**CLR**","metadata":{}},{"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.007,\n                        step_size=300., mode='exp_range',\n                        gamma=0.99994)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = 'adam'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Attention Layer**","metadata":{}},{"cell_type":"code","source":"!pip install keras-self-attention\nimport keras\nfrom keras_self_attention import SeqSelfAttention","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = 'adam'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clr = CyclicLR(base_lr=0.001, max_lr=0.007,\n                        step_size=300., mode='exp_range',\n                        gamma=0.99994)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = SeqSelfAttention(attention_activation='sigmoid')(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Transformer**","metadata":{}},{"cell_type":"code","source":"class MultiHeadSelfAttention(layers.Layer):\n    def __init__(self, embed_dim, num_heads=8):\n        super(MultiHeadSelfAttention, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if embed_dim % num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}\"\n            )\n        self.projection_dim = embed_dim // num_heads\n        self.query_dense = layers.Dense(embed_dim)\n        self.key_dense = layers.Dense(embed_dim)\n        self.value_dense = layers.Dense(embed_dim)\n        self.combine_heads = layers.Dense(embed_dim)\n\n    def attention(self, query, key, value):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, inputs):\n        # x.shape = [batch_size, seq_len, embedding_dim]\n        batch_size = tf.shape(inputs)[0]\n        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        key = self.key_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        query = self.separate_heads(\n            query, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        key = self.separate_heads(\n            key, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        value = self.separate_heads(\n            value, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        attention, weights = self.attention(query, key, value)\n        attention = tf.transpose(\n            attention, perm=[0, 2, 1, 3]\n        )  # (batch_size, seq_len, num_heads, projection_dim)\n        concat_attention = tf.reshape(\n            attention, (batch_size, -1, self.embed_dim)\n        )  # (batch_size, seq_len, embed_dim)\n        output = self.combine_heads(\n            concat_attention\n        )  # (batch_size, seq_len, embed_dim)\n        return output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = MultiHeadSelfAttention(embed_dim, num_heads)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_heads = 2  # Number of attention heads\nff_dim = 32  # Hidden layer size in feed forward network inside transformer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\ntransformer_block = TransformerBlock(embed_size, num_heads, ff_dim)\nx = transformer_block(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}