{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Bi-LSTM","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Natural Language Processing(NLP) is one of the main usages for the Neural networks-Deep learning model wither it was speech recognition, ChatBots or even predict the next words in a sentence, this all will not be achieved throughout using simple NN there is model's developed in order to overcome these obstacles one of these models is RNN.\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"BiLSTM - (Bidirectional LSTMs) it's an extension of traditional LSTMs. It trains two instead of one LSTMs on the input sequence, The first on the input sequence as-is, and the second on a reversed copy of the input sequence. This can provide additional context to the network and result in faster and even fuller learning on the problem.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Note: This file is created to experment the run for explined jupter file ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## The Code","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom numba import jit, cuda \nfrom timeit import default_timer as timer    \n\nimport os,pickle\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfrom keras.models import Sequential\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation,Layer\nfrom keras.layers import Bidirectional,GlobalAveragePooling1D#Concatenate\nfrom keras.models import Model,Sequential\nfrom sklearn import metrics\nfrom keras.preprocessing import sequence, text \nfrom keras import initializers, regularizers, constraints, optimizers, layers # This for the attenition layer\nfrom tensorflow.keras import backend as K #this is also for the attention layer\nimport tensorflow as tf\nimport transformers\nfrom keras.layers.merge import concatenate\nfrom tensorflow.keras import layers","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Data Sets","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Loading train sets\ntrain = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n\n# Loading validation sets\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n\n# Loading test sets\ntest = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DATA Pre-processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# select comment_text for the preprosess (X)\nlist_sentences_train = train[\"comment_text\"] \nlist_sentences_validation = valid['comment_text']\nlist_sentences_test = test[\"content\"]\n\n#select comment_text for the preprosess (y)\ny_train = train.toxic.values \ny_valid = valid.toxic.values ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call the tokenizer with it's paramitera\nmax_features = 20000\ntokenizer = Tokenizer(num_words=max_features,filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',lower=True) \n\n#Fitting tokenizer\ntokenizer.fit_on_texts(list(list_sentences_train) + list(list_sentences_validation) + list(list_sentences_test))\n\n# for bulding the matrix\nword_index = tokenizer.word_index\n\n# Building training set\nlist_tokenized_train = tokenizer.texts_to_sequences(list(list_sentences_train))\ny_train = train['toxic'].values\n\n# Building validation set\nlist_tokenized_validation = tokenizer.texts_to_sequences(list(list_sentences_validation))\ny_valid = valid['toxic'].values\n\n# Building test set\nlist_tokenized_test = tokenizer.texts_to_sequences(list(list_sentences_test))\n\ndel tokenizer # To save RAM space","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maxlen = 200 # length of padding\n\n# Padding sequences for all \nX_t = pad_sequences(list_tokenized_train, maxlen=maxlen)\nX_valid = pad_sequences(list_tokenized_validation, maxlen=maxlen)\nX_te = pad_sequences(list_tokenized_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using Crawl word vector\nwith open('../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl', 'rb') as  infile:\n        crawl_embeddings = pickle.load(infile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using GloVe word vector\nwith open('../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', 'rb') as  infile:\n        glove_embeddings = pickle.load(infile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function for building a matrix\ndef build_matrix(word_index, embeddings_index):\n    ''''\n    Input: word indexing from the tocnizer appove and the pre-trined word vector model\n    \n    output: embedding matrix\n    \n    ''''\n    embedding_matrix = np.zeros((len(word_index) + 1,300 ))\n    for word, i in word_index.items():\n        try:\n            embedding_matrix[i] = embeddings_index[word]\n        except:\n            embedding_matrix[i] = embeddings_index[\"unknown\"]\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Building matrices\nembedding_matrix_1 = build_matrix(word_index, crawl_embeddings)\nembedding_matrix_2 = build_matrix(word_index, glove_embeddings)\n\n# Concatenating embedding matrices \nembedding_matrix = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis=1)\n\ndel embedding_matrix_1, embedding_matrix_2\ndel crawl_embeddings ,glove_embeddings  # for saving RAM Space","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(Layer):\n    \"\"\"\n    Custom Keras attention layer\n    \n    Reference: https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043\n    \"\"\"\n    def __init__(self, step_dim, W_regularizer=None, b_regularizer=None, \n                 W_constraint=None, b_constraint=None, bias=True, **kwargs):\n\n        self.supports_masking = True\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = None\n        super(Attention, self).__init__(**kwargs)\n\n        self.param_W = {\n            'initializer': initializers.get('glorot_uniform'),\n            'name': '{}_W'.format(self.name),\n            'regularizer': regularizers.get(W_regularizer),\n            'constraint': constraints.get(W_constraint)\n        }\n        self.W = None\n\n        self.param_b = {\n            'initializer': 'zero',\n            'name': '{}_b'.format(self.name),\n            'regularizer': regularizers.get(b_regularizer),\n            'constraint': constraints.get(b_constraint)\n        }\n        self.b = None\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.features_dim = input_shape[-1]\n        self.W = self.add_weight(shape=(input_shape[-1],), \n                                 **self.param_W)\n\n        if self.bias:\n            self.b = self.add_weight(shape=(input_shape[1],), \n                                     **self.param_b)\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        step_dim = self.step_dim\n        features_dim = self.features_dim\n\n        eij = K.reshape(\n            K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))),\n            (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define shape of the input \ninp = Input(shape=(maxlen,)) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create embedding layer \nembedding_layer = Embedding(*embedding_matrix.shape,\n                                weights=[embedding_matrix],\n                                trainable=False) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pass input into the embded lyer \nx = embedding_layer(inp) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feed into bidirectional wech it will out but \nx = Bidirectional(LSTM(256, return_sequences=True))(x) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feed into bidirectional wech it will out but\nx = Bidirectional(LSTM(128, return_sequences=True))(x) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call the GlobalAveragePooling1D \navrege = GlobalAveragePooling1D()(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call the Attention \nattention = Attention(maxlen)(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# concate these techniqes to form layer that perform on the output from the Bi-LSTM \nhidden = concatenate([attention,avrege])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using dense with 512 output with relu acttivation function\nx = Dense(512, activation='relu')(hidden)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# perform a dropout with 0.5 to avoid ofer fitting \nx =  Dropout(0.5)(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using dense with 128 output with relu acttivation function \nx = Dense(128, activation=\"relu\")(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using dense output with sigmoid acttivation function \no = Dense(1, activation='sigmoid')(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call the model \nmodel = Model(inputs=inp, outputs=o)\nmodel.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=[tf.keras.metrics.AUC()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model fitting on train data set\nmodel.fit(X_t,y_train,batch_size=32,epochs=2,validation_split=0.1)\n\n# NOTE : THE RUN MAY TAKE SOME TIME ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model fitting on Validation data set\nmodel.fit(X_valid,y_valid,batch_size=32,epochs=2,validation_split=0.1)\n# NOTE : THE RUN MAY TAKE SOME TIME ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predect the toxicity of the test\nval = model.predict(X_te, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save the predections into the submetion file \nsub['toxic'] = val \nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}