{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom transformers import TFBertModel, BertTokenizer\nimport transformers\n\ntf.config.experimental_run_functions_eagerly(False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T04:05:40.079793Z","iopub.execute_input":"2022-08-05T04:05:40.080302Z","iopub.status.idle":"2022-08-05T04:05:49.402568Z","shell.execute_reply.started":"2022-08-05T04:05:40.080192Z","shell.execute_reply":"2022-08-05T04:05:49.401439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def typeModel():\n    model = tf.keras.Sequential()\n    \n    model.add(layers.Bidirectional(layers.LSTM(units=100, return_sequences=True), input_shape=[23,1]))\n    model.add(layers.Bidirectional(layers.LSTM(units=50, dropout=0.2, return_sequences=True)))\n    model.add(layers.Bidirectional(layers.LSTM(units=20, return_sequences=True)))\n    model.add(layers.Bidirectional(layers.LSTM(units=10, return_sequences=True)))  \n    model.add(layers.Dense(1, activation='tanh'))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-05T04:05:49.404304Z","iopub.execute_input":"2022-08-05T04:05:49.405539Z","iopub.status.idle":"2022-08-05T04:05:49.415397Z","shell.execute_reply.started":"2022-08-05T04:05:49.405482Z","shell.execute_reply":"2022-08-05T04:05:49.413988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TYPEMODEL = typeModel()\nBERTMODEL = TFBertModel.from_pretrained(\"bert-base-uncased\")\n\nclass BertLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(BertLayer, self).__init__()\n\n    def call(self, input_ids, token_type_ids, attention_mask):\n        out = []\n        for i in range(8):\n            out.append(BERTMODEL(input_ids[i], token_type_ids=token_type_ids[i], attention_mask=attention_mask[i])[1])\n        out = tf.convert_to_tensor(out)\n        out = tf.reshape(out, [8, 23,768])\n        return out\n\n\n    \ndef build_model(batchSize=8):\n    input_ids = layers.Input(shape=(23, 512), dtype=tf.int32, name=\"input_ids\")\n    token_type_ids = layers.Input(shape=(23, 512), dtype=tf.int32, name=\"token_type_ids\")\n    attention_mask = layers.Input(shape=(23, 512), dtype=tf.int32, name=\"attention_mask\")\n    type_input = layers.Input(shape=(23, 1))\n\n    \n    bertOut = BertLayer()(input_ids, token_type_ids, attention_mask)\n    typeOut = TYPEMODEL(type_input)\n\n    multiply_layer = layers.Multiply()([bertOut, typeOut])\n    \n    finalOut = layers.Bidirectional(layers.LSTM(units=400, return_sequences=True))(multiply_layer)\n    finalOut = layers.Bidirectional(layers.LSTM(units=200, return_sequences=True))(finalOut)\n    finalOut = layers.Bidirectional(layers.LSTM(units=100, return_sequences=True))(finalOut)\n    finalOut = layers.Bidirectional(layers.LSTM(units=50, return_sequences=True))(finalOut)\n    finalOut = layers.Dense(4, activation='softmax')(finalOut)\n\n    model = Model(inputs=[input_ids, token_type_ids, attention_mask, type_input], outputs=finalOut)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-05T04:05:49.419078Z","iopub.execute_input":"2022-08-05T04:05:49.419596Z","iopub.status.idle":"2022-08-05T04:06:24.570332Z","shell.execute_reply.started":"2022-08-05T04:05:49.419550Z","shell.execute_reply":"2022-08-05T04:06:24.569160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T04:06:24.572282Z","iopub.execute_input":"2022-08-05T04:06:24.572758Z","iopub.status.idle":"2022-08-05T04:06:45.967711Z","shell.execute_reply.started":"2022-08-05T04:06:24.572716Z","shell.execute_reply":"2022-08-05T04:06:45.966613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T04:08:15.406818Z","iopub.execute_input":"2022-08-05T04:08:15.407212Z","iopub.status.idle":"2022-08-05T04:08:15.418112Z","shell.execute_reply.started":"2022-08-05T04:08:15.407181Z","shell.execute_reply":"2022-08-05T04:08:15.416996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}