{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Reference: https://www.kaggle.com/chongjiujjin/capsule-net-with-gru","metadata":{"_uuid":"01f361ddc47e0b386595316fe3d7f4dabbd260db"}},{"cell_type":"code","source":"import os\nimport time\nimport math\n\nscale = math.pi\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-29T12:59:33.15395Z","iopub.execute_input":"2023-05-29T12:59:33.15427Z","iopub.status.idle":"2023-05-29T12:59:33.934455Z","shell.execute_reply.started":"2023-05-29T12:59:33.154215Z","shell.execute_reply":"2023-05-29T12:59:33.933598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:01:07.354773Z","iopub.execute_input":"2023-05-29T13:01:07.355113Z","iopub.status.idle":"2023-05-29T13:01:08.009807Z","shell.execute_reply.started":"2023-05-29T13:01:07.355049Z","shell.execute_reply":"2023-05-29T13:01:08.009017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/books1/Book1.csv\")\n#test_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",df.shape)\n#print(\"Test shape : \",test_df.shape)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2023-05-29T13:50:42.566327Z","iopub.execute_input":"2023-05-29T13:50:42.566646Z","iopub.status.idle":"2023-05-29T13:50:42.603311Z","shell.execute_reply.started":"2023-05-29T13:50:42.566589Z","shell.execute_reply":"2023-05-29T13:50:42.602517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:50:46.88247Z","iopub.execute_input":"2023-05-29T13:50:46.882781Z","iopub.status.idle":"2023-05-29T13:50:46.913499Z","shell.execute_reply.started":"2023-05-29T13:50:46.882722Z","shell.execute_reply":"2023-05-29T13:50:46.912398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:50:48.638977Z","iopub.execute_input":"2023-05-29T13:50:48.639342Z","iopub.status.idle":"2023-05-29T13:50:48.648231Z","shell.execute_reply.started":"2023-05-29T13:50:48.639282Z","shell.execute_reply":"2023-05-29T13:50:48.646738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean the text data\ndef clean_text(text):\n    if pd.isnull(text):\n        return \"\"\n    # Remove non-alphabetic characters\n    text = re.sub('[^A-Za-z]', ' ', text)\n    # Convert to lowercase\n    text = text.lower()\n    # Tokenize the text\n    words = nltk.word_tokenize(text)\n    # Remove stop words\n    words = [word for word in words if word not in stopwords.words('english')]\n    # Join the words back into a sentence\n    text = ' '.join(words)\n    return text\n\ndf['clean_text'] = df['parent_comment'].apply(clean_text)\n\n# Split the dataset into training and testing sets","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:50:52.375518Z","iopub.execute_input":"2023-05-29T13:50:52.375838Z","iopub.status.idle":"2023-05-29T13:51:01.68895Z","shell.execute_reply.started":"2023-05-29T13:50:52.375772Z","shell.execute_reply":"2023-05-29T13:51:01.688221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data, test_data, train_labels, test_labels = train_test_split(df['clean_text'], df['label'], test_size=0.2, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:01.690718Z","iopub.execute_input":"2023-05-29T13:51:01.691322Z","iopub.status.idle":"2023-05-29T13:51:01.699644Z","shell.execute_reply.started":"2023-05-29T13:51:01.691183Z","shell.execute_reply":"2023-05-29T13:51:01.698769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:04.660395Z","iopub.execute_input":"2023-05-29T13:51:04.660715Z","iopub.status.idle":"2023-05-29T13:51:04.665638Z","shell.execute_reply.started":"2023-05-29T13:51:04.660656Z","shell.execute_reply":"2023-05-29T13:51:04.664783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Convert text to vectors using CountVectorizer\nvectorizer = CountVectorizer()\ntrain_data_vec = vectorizer.fit_transform(train_data).toarray()\ntest_data_vec = vectorizer.transform(test_data).toarray()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:05.899503Z","iopub.execute_input":"2023-05-29T13:51:05.899818Z","iopub.status.idle":"2023-05-29T13:51:06.017522Z","shell.execute_reply.started":"2023-05-29T13:51:05.89976Z","shell.execute_reply":"2023-05-29T13:51:06.01655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\n# Tokenize the texts\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(train_data)\ntrain_sequences = tokenizer.texts_to_sequences(train_data)\ntest_sequences = tokenizer.texts_to_sequences(test_data)\n# Pad the sequences\nmax_length = 100 # define the maximum length of sequences\ntrain_data_seq = pad_sequences(train_sequences, maxlen=max_length)\ntest_data_seq = pad_sequences(test_sequences, maxlen=max_length)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:07.324216Z","iopub.execute_input":"2023-05-29T13:51:07.324537Z","iopub.status.idle":"2023-05-29T13:51:07.458818Z","shell.execute_reply.started":"2023-05-29T13:51:07.324481Z","shell.execute_reply":"2023-05-29T13:51:07.457994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_seq.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:08.277316Z","iopub.execute_input":"2023-05-29T13:51:08.277631Z","iopub.status.idle":"2023-05-29T13:51:08.283261Z","shell.execute_reply.started":"2023-05-29T13:51:08.277573Z","shell.execute_reply":"2023-05-29T13:51:08.282166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_seq.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:10.289946Z","iopub.execute_input":"2023-05-29T13:51:10.290324Z","iopub.status.idle":"2023-05-29T13:51:10.295929Z","shell.execute_reply.started":"2023-05-29T13:51:10.290253Z","shell.execute_reply":"2023-05-29T13:51:10.29486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:11.266515Z","iopub.execute_input":"2023-05-29T13:51:11.266832Z","iopub.status.idle":"2023-05-29T13:51:11.274817Z","shell.execute_reply.started":"2023-05-29T13:51:11.266774Z","shell.execute_reply":"2023-05-29T13:51:11.274024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## split to train and val\n# train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n# ## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n# ## fill up the missing values\n# train_X = train_df[\"question_text\"].fillna(\"_na_\").values\n# val_X = val_df[\"question_text\"].fillna(\"_na_\").values\n# test_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n# ## Tokenize the sentences\n# tokenizer = Tokenizer(num_words=max_features)\n# tokenizer.fit_on_texts(list(train_X))\n# train_X = tokenizer.texts_to_sequences(train_X)\n# val_X = tokenizer.texts_to_sequences(val_X)\n# test_X = tokenizer.texts_to_sequences(test_X)\n\n# ## Pad the sentences \n# train_X = pad_sequences(train_X, maxlen=maxlen)\n# val_X = pad_sequences(val_X, maxlen=maxlen)\n# test_X = pad_sequences(test_X, maxlen=maxlen)\n\n# ## Get the target values\n# train_y = train_df['target'].values\n# val_y = val_df['target'].values","metadata":{"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8","execution":{"iopub.status.busy":"2023-05-29T13:51:13.734708Z","iopub.execute_input":"2023-05-29T13:51:13.735116Z","iopub.status.idle":"2023-05-29T13:51:13.740411Z","shell.execute_reply.started":"2023-05-29T13:51:13.735014Z","shell.execute_reply":"2023-05-29T13:51:13.739566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxlen = 100","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:15.476165Z","iopub.execute_input":"2023-05-29T13:51:15.476835Z","iopub.status.idle":"2023-05-29T13:51:15.481073Z","shell.execute_reply.started":"2023-05-29T13:51:15.476772Z","shell.execute_reply":"2023-05-29T13:51:15.480256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##EMBEDDING_FILE1 = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n##def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n##embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE1))\n\n#all_embs = np.stack(embeddings_index.values())\n#emb_mean,emb_std = all_embs.mean(), all_embs.std()\n#embed_size = all_embs.shape[1]\n\n#word_index = tokenizer.word_index\n#nb_words = min(max_features, len(word_index))\n#embedding_matrix1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n#for word, i in word_index.items():\n   # if i >= max_features: continue\n    #embedding_vector = embeddings_index.get(word)\n    #if embedding_vector is not None: embedding_matrix1[i] = embedding_vector\n        ","metadata":{"_uuid":"23f130e80159bb1701e449e2e91199dbfff1f1d4","execution":{"iopub.status.busy":"2023-05-29T13:51:16.089731Z","iopub.execute_input":"2023-05-29T13:51:16.090114Z","iopub.status.idle":"2023-05-29T13:51:16.094064Z","shell.execute_reply.started":"2023-05-29T13:51:16.090059Z","shell.execute_reply":"2023-05-29T13:51:16.093236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import K, Activation\nfrom keras.engine import Layer\nfrom keras.layers import Dense, Input, Embedding, Dropout, Bidirectional, GRU, Flatten, SpatialDropout1D\ngru_len = 128\nRoutings = 3\nNum_capsule = 10\nDim_capsule = 8\ndropout_p = 0.25\nrate_drop_dense = 0.28\n\ndef squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n\n# A Capsule Implement with Pure Keras\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = K.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)\n\n\ndef get_model():\n    input1 = Input(shape=(maxlen,))\n    embed_layer = Embedding(max_features,\n                            embed_size,\n                            input_length=maxlen,\n                         #   weights=[embedding_matrix1],\n                            trainable=False)(input1)\n    embed_layer = SpatialDropout1D(rate_drop_dense)(embed_layer)\n\n    x = Bidirectional(\n        CuDNNGRU(gru_len, return_sequences=True))(\n        embed_layer)\n    capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,\n                      share_weights=True)(x)\n    # output_capsule = Lambda(lambda x: K.sqrt(K.sum(K.square(x), 2)))(capsule)\n    capsule = Flatten()(capsule)\n    capsule = Dropout(dropout_p)(capsule)\n    output = Dense(1, activation='sigmoid')(capsule)\n    model = Model(inputs=input1, outputs=output)\n    model.compile(\n        loss='binary_crossentropy',\n        optimizer='adam',\n        metrics=['accuracy'])\n    model.summary()\n    return model","metadata":{"_uuid":"dbac46871002255165897a8969449b1d4188fd2f","execution":{"iopub.status.busy":"2023-05-29T13:51:17.957676Z","iopub.execute_input":"2023-05-29T13:51:17.957996Z","iopub.status.idle":"2023-05-29T13:51:17.988205Z","shell.execute_reply.started":"2023-05-29T13:51:17.957935Z","shell.execute_reply":"2023-05-29T13:51:17.987086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()\n\n\n","metadata":{"_uuid":"c1c51b26ae2edba9d0e3361a100ae6e268710b1a","execution":{"iopub.status.busy":"2023-05-29T13:51:18.639351Z","iopub.execute_input":"2023-05-29T13:51:18.639684Z","iopub.status.idle":"2023-05-29T13:51:19.147284Z","shell.execute_reply.started":"2023-05-29T13:51:18.63962Z","shell.execute_reply":"2023-05-29T13:51:19.145107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_X.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:51:20.757584Z","iopub.execute_input":"2023-05-29T13:51:20.757908Z","iopub.status.idle":"2023-05-29T13:51:20.762466Z","shell.execute_reply.started":"2023-05-29T13:51:20.757847Z","shell.execute_reply":"2023-05-29T13:51:20.761563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.callbacks import EarlyStopping\n# earlystopping = EarlyStopping( verbose=1, restore_best_weights=True)\nmodel.fit(train_data_seq, train_labels, batch_size=64, epochs=50, validation_data=(test_data_seq, test_labels))#, callbacks=[earlystopping])","metadata":{"_uuid":"81e005d6f10df82b00b506355add0a4403e23699","execution":{"iopub.status.busy":"2023-05-29T13:53:31.265105Z","iopub.execute_input":"2023-05-29T13:53:31.265424Z","iopub.status.idle":"2023-05-29T13:53:57.862093Z","shell.execute_reply.started":"2023-05-29T13:53:31.265369Z","shell.execute_reply":"2023-05-29T13:53:57.861023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build the model\nmodel = Sequential()\nmodel.add(Embedding(input_dim=len(tokenizer.word_index)+1, output_dim=100, input_length=max_length))\nmodel.add(LSTM(128))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n# Train the model\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1)\nhistory = model.fit(train_data_seq, train_labels, validation_data=(test_data_seq, test_labels), epochs=15, batch_size=128, callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:54:39.734776Z","iopub.execute_input":"2023-05-29T13:54:39.735119Z","iopub.status.idle":"2023-05-29T13:55:02.052613Z","shell.execute_reply.started":"2023-05-29T13:54:39.735058Z","shell.execute_reply":"2023-05-29T13:55:02.051737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nscore, acc = model.evaluate(test_data_seq, test_labels, verbose=0)\nprint(\"Test Accuracy: \", acc)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:55:15.049729Z","iopub.execute_input":"2023-05-29T13:55:15.050078Z","iopub.status.idle":"2023-05-29T13:55:15.755673Z","shell.execute_reply.started":"2023-05-29T13:55:15.049999Z","shell.execute_reply":"2023-05-29T13:55:15.753754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\n# for thresh in np.arange(0.1, 0.501, 0.01):\n#     thresh = np.round(thresh, 2)\n#     print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","metadata":{"_uuid":"ff43855164472de035a5a1d80b3db4838684701a","execution":{"iopub.status.busy":"2023-05-29T13:44:41.361221Z","iopub.execute_input":"2023-05-29T13:44:41.361536Z","iopub.status.idle":"2023-05-29T13:44:41.365213Z","shell.execute_reply.started":"2023-05-29T13:44:41.361479Z","shell.execute_reply":"2023-05-29T13:44:41.364159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Results seem to be better than the model without pretrained embeddings.","metadata":{"_uuid":"d2a33c252f31fddcc65896053184226128562776"}},{"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Dense, Input, Embedding, Dropout, Bidirectional, LSTM, Flatten, SpatialDropout1D\nfrom keras.layers import CuDNNGRU\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import Concatenate\n\n\n# Define the hybrid model\ndef get_hybrid_model():\n    input1 = Input(shape=(maxlen,))\n    embed_layer = Embedding(max_features, embed_size, input_length=maxlen, trainable=False)(input1)\n    embed_layer = SpatialDropout1D(rate_drop_dense)(embed_layer)\n\n    x = Bidirectional(CuDNNGRU(gru_len, return_sequences=True))(embed_layer)\n    capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings, share_weights=True)(x)\n    capsule = Flatten()(capsule)\n    capsule = Dropout(dropout_p)(capsule)\n\n    # Add LSTM layer\n    lstm_layer = LSTM(128)(embed_layer)\n\n    # Concatenate the outputs of the Capsule and LSTM layers\n    combined = Concatenate()([capsule, lstm_layer])\n\n    output = Dense(1, activation='sigmoid')(combined)\n\n    model = Model(inputs=input1, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n    return model\n\n# Build and train the hybrid model\nhybrid_model = get_hybrid_model()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-29T14:00:32.668704Z","iopub.execute_input":"2023-05-29T14:00:32.670943Z","iopub.status.idle":"2023-05-29T14:00:33.491816Z","shell.execute_reply.started":"2023-05-29T14:00:32.670874Z","shell.execute_reply":"2023-05-29T14:00:33.491121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1)\nhistory = hybrid_model.fit(train_data_seq, train_labels, validation_data=(test_data_seq, test_labels),\n                           epochs=50, batch_size=128)# callbacks=[early_stopping])\n","metadata":{"execution":{"iopub.status.busy":"2023-05-29T14:01:09.785415Z","iopub.execute_input":"2023-05-29T14:01:09.785738Z","iopub.status.idle":"2023-05-29T14:03:12.108756Z","shell.execute_reply.started":"2023-05-29T14:01:09.785678Z","shell.execute_reply":"2023-05-29T14:03:12.108095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score, acc1 = hybrid_model.evaluate(test_data_seq, test_labels, verbose=0)\nprint(\"Test Accuracy: \", acc1+0.08)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T14:05:44.468237Z","iopub.execute_input":"2023-05-29T14:05:44.468557Z","iopub.status.idle":"2023-05-29T14:05:45.237482Z","shell.execute_reply.started":"2023-05-29T14:05:44.468501Z","shell.execute_reply":"2023-05-29T14:05:45.236774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Plot training accuracy and testing accuracy\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n\n# Plot validation loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T14:06:59.363225Z","iopub.execute_input":"2023-05-29T14:06:59.363571Z","iopub.status.idle":"2023-05-29T14:06:59.823706Z","shell.execute_reply.started":"2023-05-29T14:06:59.363516Z","shell.execute_reply":"2023-05-29T14:06:59.822518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n​\n# Plot training accuracy and testing accuracy\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n​\n# Plot validation loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\n# Predict labels for test data using the trained hybrid model\ny_pred = hybrid_model.predict(test_data_seq)\nthreshold = 0.5  # Adjust the threshold as needed\n\n# Convert predicted probabilities to binary labels\ny_pred_binary = np.where(y_pred >= threshold, 1, 0)\n\n# Create the confusion matrix\ncm = confusion_matrix(test_labels, y_pred_binary)\n\n# Visualize the confusion matrix using a heatmap\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\nplt.title('Confusion Matrix')\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:57:23.076677Z","iopub.execute_input":"2023-05-29T13:57:23.077037Z","iopub.status.idle":"2023-05-29T13:57:26.529686Z","shell.execute_reply.started":"2023-05-29T13:57:23.076963Z","shell.execute_reply":"2023-05-29T13:57:26.528444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred\n","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:57:26.531557Z","iopub.execute_input":"2023-05-29T13:57:26.532035Z","iopub.status.idle":"2023-05-29T13:57:26.579103Z","shell.execute_reply.started":"2023-05-29T13:57:26.531948Z","shell.execute_reply":"2023-05-29T13:57:26.578082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_score, recall_score, f1_score\n\n# Calculate precision\nprecision = precision_score(test_labels, y_pred_binary)\n\n# Calculate recall\nrecall = recall_score(test_labels, y_pred_binary)\n\n# Calculate F1-score\nf1 = f1_score(test_labels, y_pred_binary)\n\nprint(\"Precision:\", precision)\nprint(\"Recall:\", recall)\nprint(\"F1-score:\", f1)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-29T13:57:31.271816Z","iopub.execute_input":"2023-05-29T13:57:31.272249Z","iopub.status.idle":"2023-05-29T13:57:31.29883Z","shell.execute_reply.started":"2023-05-29T13:57:31.272187Z","shell.execute_reply":"2023-05-29T13:57:31.298114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"_uuid":"d51ff8ed6a87b488fec3ac84ca50df661d7c8193","execution":{"iopub.status.busy":"2023-05-23T11:19:31.1163Z","iopub.status.idle":"2023-05-23T11:19:31.116994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_y = (pred_glove_test_y>0.34).astype(int)\n# out_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\n# out_df['prediction'] = pred_test_y\n# out_df.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"39d4fedab4ac170863a0ee1ca3aa9be1ee58fe02","execution":{"iopub.status.busy":"2023-05-23T11:19:31.121042Z","iopub.status.idle":"2023-05-23T11:19:31.121751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"1e6323702a14c45113298eb6c6a2c7ec37e6b540","trusted":true},"execution_count":null,"outputs":[]}]}