{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":3043,"databundleVersionId":46668,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Predict Closed Questions on StackOverflow**","metadata":{}},{"cell_type":"markdown","source":"**Group Members:** <br>\nNermeen Soliman - 19104699 <br>\nSeifeldin Sami - 19106471 <br>\nMirna Ahmed - 19104977 <br>","metadata":{}},{"cell_type":"markdown","source":"**Project Description:**\n\nProgrammers ask many questions on Stack Overflow all the time, some are great, others are repetitive, time-wasting, or incomplete. So, in this project, you want to predict whether a new question will be closed or not, along with the reason why.","metadata":{}},{"cell_type":"markdown","source":"# Imports for Cleaning Dataset","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/jbesomi/texthero.git -qq","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:42.360284Z","iopub.execute_input":"2024-05-31T20:27:42.360981Z","iopub.status.idle":"2024-05-31T20:27:48.198169Z","shell.execute_reply.started":"2024-05-31T20:27:42.360944Z","shell.execute_reply":"2024-05-31T20:27:48.196951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport time\nimport os\nimport shutil\nimport pandas as pd\nimport numpy as np\nimport zipfile\nimport texthero as hero\nfrom nltk.corpus import stopwords\nimport nltk\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:48.200231Z","iopub.execute_input":"2024-05-31T20:27:48.200524Z","iopub.status.idle":"2024-05-31T20:27:48.205348Z","shell.execute_reply.started":"2024-05-31T20:27:48.200497Z","shell.execute_reply":"2024-05-31T20:27:48.204761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download the NLTK stopwords\nnltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:48.206180Z","iopub.execute_input":"2024-05-31T20:27:48.206410Z","iopub.status.idle":"2024-05-31T20:27:48.229322Z","shell.execute_reply.started":"2024-05-31T20:27:48.206388Z","shell.execute_reply":"2024-05-31T20:27:48.228681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Uploading Dataset","metadata":{}},{"cell_type":"code","source":"file_path = '/kaggle/input/predict-closed-questions-on-stack-overflow/train-sample.csv'\n# Load the CSV file\ndata = pd.read_csv(file_path, sep=',', on_bad_lines='skip')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:48.230750Z","iopub.execute_input":"2024-05-31T20:27:48.230977Z","iopub.status.idle":"2024-05-31T20:27:51.837817Z","shell.execute_reply.started":"2024-05-31T20:27:48.230954Z","shell.execute_reply":"2024-05-31T20:27:51.836993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.838786Z","iopub.execute_input":"2024-05-31T20:27:51.839067Z","iopub.status.idle":"2024-05-31T20:27:51.843865Z","shell.execute_reply.started":"2024-05-31T20:27:51.839039Z","shell.execute_reply":"2024-05-31T20:27:51.843261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.OpenStatus.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.844724Z","iopub.execute_input":"2024-05-31T20:27:51.844963Z","iopub.status.idle":"2024-05-31T20:27:51.864823Z","shell.execute_reply.started":"2024-05-31T20:27:51.844937Z","shell.execute_reply":"2024-05-31T20:27:51.864263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.865670Z","iopub.execute_input":"2024-05-31T20:27:51.865930Z","iopub.status.idle":"2024-05-31T20:27:51.884004Z","shell.execute_reply.started":"2024-05-31T20:27:51.865880Z","shell.execute_reply":"2024-05-31T20:27:51.883415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's take 'TITLE' & 'BODYMARKDOWN' & OpenStatus Columns\ndata = data[['Title', 'BodyMarkdown', 'OpenStatus']]\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.884839Z","iopub.execute_input":"2024-05-31T20:27:51.885083Z","iopub.status.idle":"2024-05-31T20:27:51.899458Z","shell.execute_reply.started":"2024-05-31T20:27:51.885058Z","shell.execute_reply":"2024-05-31T20:27:51.898765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.900359Z","iopub.execute_input":"2024-05-31T20:27:51.900612Z","iopub.status.idle":"2024-05-31T20:27:51.906116Z","shell.execute_reply.started":"2024-05-31T20:27:51.900584Z","shell.execute_reply":"2024-05-31T20:27:51.905513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Dataset","metadata":{}},{"cell_type":"code","source":"data['Text'] = data['Title'] + ' ' + data['BodyMarkdown']","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:51.908814Z","iopub.execute_input":"2024-05-31T20:27:51.909079Z","iopub.status.idle":"2024-05-31T20:27:52.070584Z","shell.execute_reply.started":"2024-05-31T20:27:51.909052Z","shell.execute_reply":"2024-05-31T20:27:52.069889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.drop(columns = ['Title', 'BodyMarkdown'])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:52.071434Z","iopub.execute_input":"2024-05-31T20:27:52.071664Z","iopub.status.idle":"2024-05-31T20:27:52.081580Z","shell.execute_reply.started":"2024-05-31T20:27:52.071639Z","shell.execute_reply":"2024-05-31T20:27:52.080965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:52.082361Z","iopub.execute_input":"2024-05-31T20:27:52.082576Z","iopub.status.idle":"2024-05-31T20:27:52.097292Z","shell.execute_reply.started":"2024-05-31T20:27:52.082553Z","shell.execute_reply":"2024-05-31T20:27:52.096643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import texthero.preprocessing as preprocessing\n\n# Define a custom pipeline for preprocessing\ncustom_pipeline = [\n    preprocessing.fillna,                   # Fill NaN values\n    preprocessing.lowercase,                # Convert to lowercase\n    preprocessing.remove_whitespace,        # Remove leading/trailing whitespace\n    preprocessing.remove_punctuation,       # Remove punctuation\n    preprocessing.remove_diacritics,        # Remove diacritics\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:52.098196Z","iopub.execute_input":"2024-05-31T20:27:52.098460Z","iopub.status.idle":"2024-05-31T20:27:52.106107Z","shell.execute_reply.started":"2024-05-31T20:27:52.098431Z","shell.execute_reply":"2024-05-31T20:27:52.105542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's pre process the data using library called \"Text_hero\"\ndata['Text'] = hero.clean(data['Text'])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:27:52.106888Z","iopub.execute_input":"2024-05-31T20:27:52.107101Z","iopub.status.idle":"2024-05-31T20:28:32.573657Z","shell.execute_reply.started":"2024-05-31T20:27:52.107079Z","shell.execute_reply":"2024-05-31T20:28:32.572941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:32.574615Z","iopub.execute_input":"2024-05-31T20:28:32.574862Z","iopub.status.idle":"2024-05-31T20:28:32.582079Z","shell.execute_reply.started":"2024-05-31T20:28:32.574837Z","shell.execute_reply":"2024-05-31T20:28:32.581397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.OpenStatus.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:32.583058Z","iopub.execute_input":"2024-05-31T20:28:32.583577Z","iopub.status.idle":"2024-05-31T20:28:32.601156Z","shell.execute_reply.started":"2024-05-31T20:28:32.583549Z","shell.execute_reply":"2024-05-31T20:28:32.600491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['id'] = range(len(data))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:32.602033Z","iopub.execute_input":"2024-05-31T20:28:32.602319Z","iopub.status.idle":"2024-05-31T20:28:32.862616Z","shell.execute_reply.started":"2024-05-31T20:28:32.602290Z","shell.execute_reply":"2024-05-31T20:28:32.861900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization","metadata":{}},{"cell_type":"code","source":"!pip install -qq plotly wordcloud","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:32.863513Z","iopub.execute_input":"2024-05-31T20:28:32.863778Z","iopub.status.idle":"2024-05-31T20:28:36.046101Z","shell.execute_reply.started":"2024-05-31T20:28:32.863752Z","shell.execute_reply":"2024-05-31T20:28:36.045020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfrom plotly.offline import init_notebook_mode\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:36.047524Z","iopub.execute_input":"2024-05-31T20:28:36.047823Z","iopub.status.idle":"2024-05-31T20:28:36.056605Z","shell.execute_reply.started":"2024-05-31T20:28:36.047793Z","shell.execute_reply":"2024-05-31T20:28:36.055928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.pie(data, values='id', names='OpenStatus', height=600, title='Classification of Questions')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:36.057503Z","iopub.execute_input":"2024-05-31T20:28:36.057746Z","iopub.status.idle":"2024-05-31T20:28:36.685266Z","shell.execute_reply.started":"2024-05-31T20:28:36.057721Z","shell.execute_reply":"2024-05-31T20:28:36.684159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['Text_Length'] = data['Text'].apply(lambda x: len(x.split()))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:36.686500Z","iopub.execute_input":"2024-05-31T20:28:36.686855Z","iopub.status.idle":"2024-05-31T20:28:37.375999Z","shell.execute_reply.started":"2024-05-31T20:28:36.686824Z","shell.execute_reply":"2024-05-31T20:28:37.375333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(data, x=\"Text_Length\", height=700, color='OpenStatus', title=\"Text Length Distribution\", marginal=\"box\")","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:37.377019Z","iopub.execute_input":"2024-05-31T20:28:37.377275Z","iopub.status.idle":"2024-05-31T20:28:38.107874Z","shell.execute_reply.started":"2024-05-31T20:28:37.377250Z","shell.execute_reply":"2024-05-31T20:28:38.107255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Text = data['Text'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:38.108820Z","iopub.execute_input":"2024-05-31T20:28:38.109060Z","iopub.status.idle":"2024-05-31T20:28:38.116407Z","shell.execute_reply.started":"2024-05-31T20:28:38.109036Z","shell.execute_reply":"2024-05-31T20:28:38.115824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wordcloud = WordCloud(max_words=1500, width=600, background_color='black').generate(\" \".join(Text))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.title(\"Words from Text\")\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:28:38.117173Z","iopub.execute_input":"2024-05-31T20:28:38.117399Z","iopub.status.idle":"2024-05-31T20:29:33.118314Z","shell.execute_reply.started":"2024-05-31T20:28:38.117375Z","shell.execute_reply":"2024-05-31T20:29:33.117433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del Text","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.119231Z","iopub.execute_input":"2024-05-31T20:29:33.119471Z","iopub.status.idle":"2024-05-31T20:29:33.124789Z","shell.execute_reply.started":"2024-05-31T20:29:33.119445Z","shell.execute_reply":"2024-05-31T20:29:33.124181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Test Split","metadata":{}},{"cell_type":"code","source":"label_id = {'open': 0, 'not a real question': 1, 'off topic': 2, 'not constructive': 3, 'too localized': 4}","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.125551Z","iopub.execute_input":"2024-05-31T20:29:33.125766Z","iopub.status.idle":"2024-05-31T20:29:33.138874Z","shell.execute_reply.started":"2024-05-31T20:29:33.125743Z","shell.execute_reply":"2024-05-31T20:29:33.138271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['OpenStatus']= data['OpenStatus'].map(label_id)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.139606Z","iopub.execute_input":"2024-05-31T20:29:33.139817Z","iopub.status.idle":"2024-05-31T20:29:33.157839Z","shell.execute_reply.started":"2024-05-31T20:29:33.139795Z","shell.execute_reply":"2024-05-31T20:29:33.157134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.161557Z","iopub.execute_input":"2024-05-31T20:29:33.161778Z","iopub.status.idle":"2024-05-31T20:29:33.170746Z","shell.execute_reply.started":"2024-05-31T20:29:33.161755Z","shell.execute_reply":"2024-05-31T20:29:33.170171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_main, remaining_data = train_test_split(data, test_size=0.30, shuffle=True, random_state=203, stratify=data['OpenStatus'])\nval_main, test_main = train_test_split(remaining_data, test_size=0.3333, shuffle=True, random_state=203, stratify=remaining_data['OpenStatus'])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.171478Z","iopub.execute_input":"2024-05-31T20:29:33.171713Z","iopub.status.idle":"2024-05-31T20:29:33.241068Z","shell.execute_reply.started":"2024-05-31T20:29:33.171676Z","shell.execute_reply":"2024-05-31T20:29:33.240430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"!pip -qq install transformers","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:33.241954Z","iopub.execute_input":"2024-05-31T20:29:33.242202Z","iopub.status.idle":"2024-05-31T20:29:36.487702Z","shell.execute_reply.started":"2024-05-31T20:29:33.242178Z","shell.execute_reply":"2024-05-31T20:29:36.486532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom transformers import AutoTokenizer,TFBertModel\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Input,GlobalAveragePooling1D,Dense\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:36.489263Z","iopub.execute_input":"2024-05-31T20:29:36.489557Z","iopub.status.idle":"2024-05-31T20:29:57.591315Z","shell.execute_reply.started":"2024-05-31T20:29:36.489528Z","shell.execute_reply":"2024-05-31T20:29:57.590204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:57.592543Z","iopub.execute_input":"2024-05-31T20:29:57.593362Z","iopub.status.idle":"2024-05-31T20:29:57.597320Z","shell.execute_reply.started":"2024-05-31T20:29:57.593330Z","shell.execute_reply":"2024-05-31T20:29:57.596621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(1)\n    tf.config.experimental_connect_to_cluster(tpu)\n    print(2)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(3)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    print(4)\n    BATCH_SIZE = strategy.num_replicas_in_sync * 4\n    print(\"Running on TPU:\", tpu.master())\n    print(f\"Batch Size: {BATCH_SIZE}\")\n    \nexcept ValueError:\n    strategy = tf.distribute.get_strategy()\n    BATCH_SIZE = 32\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    print(f\"Batch Size: {BATCH_SIZE}\")  \n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:29:57.598525Z","iopub.execute_input":"2024-05-31T20:29:57.598803Z","iopub.status.idle":"2024-05-31T20:30:07.084732Z","shell.execute_reply.started":"2024-05-31T20:29:57.598773Z","shell.execute_reply":"2024-05-31T20:30:07.083793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_check='bert-base-uncased'\ntokenizer = AutoTokenizer.from_pretrained(model_check)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:07.085845Z","iopub.execute_input":"2024-05-31T20:30:07.086111Z","iopub.status.idle":"2024-05-31T20:30:07.886505Z","shell.execute_reply.started":"2024-05-31T20:30:07.086084Z","shell.execute_reply":"2024-05-31T20:30:07.885705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_text(text, tokenizer):\n    encoded = tokenizer.batch_encode_plus(\n        text,\n        add_special_tokens=True,\n        max_length=128,#gui code\n        padding='max_length',\n        truncation=True,\n        return_attention_mask=True,\n        return_tensors=\"tf\",\n    )\n    input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n    attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n\n    return {\n        \"input_ids\": input_ids,\n        \"attention_masks\": attention_masks\n    }","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:07.887426Z","iopub.execute_input":"2024-05-31T20:30:07.887661Z","iopub.status.idle":"2024-05-31T20:30:07.892346Z","shell.execute_reply.started":"2024-05-31T20:30:07.887635Z","shell.execute_reply":"2024-05-31T20:30:07.891693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length = len(data)\n\nx_train = encode_text(train_main['Text'].tolist(), tokenizer)\nx_val = encode_text(val_main['Text'].tolist(), tokenizer)\nx_test = encode_text(test_main['Text'].tolist(), tokenizer)\n\ny_train = train_main['OpenStatus'].values\ny_val = val_main['OpenStatus'].values\ny_test = test_main['OpenStatus'].values","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:07.893092Z","iopub.execute_input":"2024-05-31T20:30:07.893323Z","iopub.status.idle":"2024-05-31T20:30:21.056311Z","shell.execute_reply.started":"2024-05-31T20:30:07.893300Z","shell.execute_reply":"2024-05-31T20:30:21.055447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    transformer_model = TFBertModel.from_pretrained(model_check)\n\n    input_ids = Input(shape=(None,),name='input_ids', dtype='int32')\n    input_masks = Input(shape=(None,), name='attention_mask', dtype='int32')\n\n    embedding_layer = transformer_model(input_ids, attention_mask=input_masks).last_hidden_state\n\n    embedding = GlobalAveragePooling1D()(embedding_layer)\n\n    x = Dense(512, activation='relu')(embedding)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(256, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(128, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    output = Dense(5, activation='softmax')(x)\n\n    model = Model(inputs=[input_ids, input_masks], outputs = output)\n    model.compile(loss='sparse_categorical_crossentropy',optimizer=tf.keras.optimizers.Adam(learning_rate=0.00001),metrics='accuracy')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:21.057281Z","iopub.execute_input":"2024-05-31T20:30:21.057531Z","iopub.status.idle":"2024-05-31T20:30:44.854475Z","shell.execute_reply.started":"2024-05-31T20:30:21.057506Z","shell.execute_reply":"2024-05-31T20:30:44.853245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze BERT Layers\nfor layer in transformer_model.layers:\n    layer.trainable = False","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:44.855659Z","iopub.execute_input":"2024-05-31T20:30:44.855923Z","iopub.status.idle":"2024-05-31T20:30:44.864813Z","shell.execute_reply.started":"2024-05-31T20:30:44.855895Z","shell.execute_reply":"2024-05-31T20:30:44.864017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:44.865769Z","iopub.execute_input":"2024-05-31T20:30:44.866005Z","iopub.status.idle":"2024-05-31T20:30:44.922112Z","shell.execute_reply.started":"2024-05-31T20:30:44.865981Z","shell.execute_reply":"2024-05-31T20:30:44.921114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"earlystopping = EarlyStopping(monitor='val_loss',min_delta = 0, patience = 5, verbose = 1, restore_best_weights=True)\n\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, factor=0.3, min_lr=0.00000001)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:44.923203Z","iopub.execute_input":"2024-05-31T20:30:44.923483Z","iopub.status.idle":"2024-05-31T20:30:44.928083Z","shell.execute_reply.started":"2024-05-31T20:30:44.923454Z","shell.execute_reply":"2024-05-31T20:30:44.927333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit((np.asarray(x_train['input_ids']),np.asarray(x_train['attention_masks'])), \n                    y_train, batch_size=BATCH_SIZE, epochs=20, \n                    validation_data=((np.asarray(x_val['input_ids']),np.asarray(x_val['attention_masks'])), y_val),\n                    callbacks=[earlystopping, learning_rate_reduction])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T20:30:44.929020Z","iopub.execute_input":"2024-05-31T20:30:44.929290Z","iopub.status.idle":"2024-05-31T21:05:33.412590Z","shell.execute_reply.started":"2024-05-31T20:30:44.929264Z","shell.execute_reply":"2024-05-31T21:05:33.411569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Inference","metadata":{}},{"cell_type":"markdown","source":"*   Learning Curves\n*   Confusion Matrix\n*   Classification Report","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve, recall_score","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:33.413975Z","iopub.execute_input":"2024-05-31T21:05:33.414315Z","iopub.status.idle":"2024-05-31T21:05:33.418537Z","shell.execute_reply.started":"2024-05-31T21:05:33.414280Z","shell.execute_reply":"2024-05-31T21:05:33.417825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:33.419367Z","iopub.execute_input":"2024-05-31T21:05:33.419610Z","iopub.status.idle":"2024-05-31T21:05:33.750254Z","shell.execute_reply.started":"2024-05-31T21:05:33.419585Z","shell.execute_reply":"2024-05-31T21:05:33.749198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:33.751285Z","iopub.execute_input":"2024-05-31T21:05:33.751520Z","iopub.status.idle":"2024-05-31T21:05:33.949382Z","shell.execute_reply.started":"2024-05-31T21:05:33.751496Z","shell.execute_reply":"2024-05-31T21:05:33.948436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_probs = model.predict((np.asarray(x_test['input_ids']),np.asarray(x_test['attention_masks'])))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:33.950353Z","iopub.execute_input":"2024-05-31T21:05:33.950608Z","iopub.status.idle":"2024-05-31T21:05:55.192740Z","shell.execute_reply.started":"2024-05-31T21:05:33.950582Z","shell.execute_reply":"2024-05-31T21:05:55.191672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(y_pred_probs, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:55.194368Z","iopub.execute_input":"2024-05-31T21:05:55.194654Z","iopub.status.idle":"2024-05-31T21:05:55.198812Z","shell.execute_reply.started":"2024-05-31T21:05:55.194625Z","shell.execute_reply":"2024-05-31T21:05:55.197950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Confusion Matrix**","metadata":{}},{"cell_type":"code","source":"!pip install seaborn\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:55.199843Z","iopub.execute_input":"2024-05-31T21:05:55.200126Z","iopub.status.idle":"2024-05-31T21:05:59.715453Z","shell.execute_reply.started":"2024-05-31T21:05:55.200096Z","shell.execute_reply":"2024-05-31T21:05:59.714302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(confusion_matrix(y_test, y_pred),cmap='viridis',annot=True,fmt='.5g',\n            xticklabels=['open','not a real question','off topic','not constructive','too localized'],yticklabels=['open','not a real question','off topic','not constructive','too localized'])\nplt.xlabel('Predicted Class')\nplt.ylabel('Actual Class')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:05:59.716835Z","iopub.execute_input":"2024-05-31T21:05:59.717133Z","iopub.status.idle":"2024-05-31T21:06:00.039183Z","shell.execute_reply.started":"2024-05-31T21:05:59.717104Z","shell.execute_reply":"2024-05-31T21:06:00.038421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T21:06:00.040032Z","iopub.execute_input":"2024-05-31T21:06:00.040276Z","iopub.status.idle":"2024-05-31T21:06:00.064649Z","shell.execute_reply.started":"2024-05-31T21:06:00.040252Z","shell.execute_reply":"2024-05-31T21:06:00.063919Z"},"trusted":true},"execution_count":null,"outputs":[]}]}