{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":12863,"databundleVersionId":788719,"sourceType":"competition"},{"sourceId":42887,"sourceType":"datasetVersion","datasetId":32801}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-15T18:27:27.046038Z","iopub.execute_input":"2024-01-15T18:27:27.046742Z","iopub.status.idle":"2024-01-15T18:27:27.055322Z","shell.execute_reply.started":"2024-01-15T18:27:27.046710Z","shell.execute_reply":"2024-01-15T18:27:27.054481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import necessary libraries\nimport numpy as np \nimport pandas as pd\nimport json\nimport csv\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport re\nimport gc\nimport seaborn as sns\nimport random\nfrom itertools import islice\nimport datetime\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, SpatialDropout1D, Dense, Dropout, Input, concatenate, Conv1D, Activation, Flatten, MaxPooling1D\nfrom torch.utils.tensorboard import SummaryWriter\n\nfrom nltk.corpus import stopwords\n\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom sklearn.metrics import precision_recall_fscore_support, recall_score, precision_score, f1_score\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:27.098463Z","iopub.execute_input":"2024-01-15T18:27:27.098997Z","iopub.status.idle":"2024-01-15T18:27:35.451389Z","shell.execute_reply.started":"2024-01-15T18:27:27.098971Z","shell.execute_reply":"2024-01-15T18:27:35.450580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\n%tensorboard --logdir logs","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:35.453210Z","iopub.execute_input":"2024-01-15T18:27:35.454233Z","iopub.status.idle":"2024-01-15T18:27:41.984757Z","shell.execute_reply.started":"2024-01-15T18:27:35.454195Z","shell.execute_reply":"2024-01-15T18:27:41.983867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tensorboard_callback(dir_name, experiment_name):\n  \"\"\"\n  Creates a TensorBoard callback instand to store log files.\n\n  Stores log files with the filepath:\n    \"dir_name/experiment_name/current_datetime/\"\n\n  Args:\n    dir_name: target directory to store TensorBoard log files\n    experiment_name: name of experiment directory (e.g. efficientnet_model_1)\n  \"\"\"\n  log_dir = dir_name + \"/\" + experiment_name + \"/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n  tensorboard_callback = tf.keras.callbacks.TensorBoard(\n      log_dir=log_dir\n  )\n  print(f\"Saving TensorBoard log files to: {log_dir}\")\n  return tensorboard_callback","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:41.986054Z","iopub.execute_input":"2024-01-15T18:27:41.986425Z","iopub.status.idle":"2024-01-15T18:27:41.992664Z","shell.execute_reply.started":"2024-01-15T18:27:41.986391Z","shell.execute_reply":"2024-01-15T18:27:41.991473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#path for data files\ntrain_path = '../input/tensorflow2-question-answering/simplified-nq-train.jsonl'\ntest_path = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl'","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:41.995301Z","iopub.execute_input":"2024-01-15T18:27:41.995908Z","iopub.status.idle":"2024-01-15T18:27:42.008975Z","shell.execute_reply.started":"2024-01-15T18:27:41.995882Z","shell.execute_reply":"2024-01-15T18:27:42.008101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Exploratory data analysis*** ","metadata":{}},{"cell_type":"code","source":"chunksize = 1000","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:42.010090Z","iopub.execute_input":"2024-01-15T18:27:42.010462Z","iopub.status.idle":"2024-01-15T18:27:42.018467Z","shell.execute_reply.started":"2024-01-15T18:27:42.010429Z","shell.execute_reply":"2024-01-15T18:27:42.017635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(path, sample = True, chunksize = chunksize):\n    if sample == True:\n        df = []\n        with open(path, 'r') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('Sample data have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient='records', lines=True, dtype={'example_id': str})\n        print('Sample data have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(train_path, sample = True)\ntest = read_data(test_path, sample = False)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:42.019458Z","iopub.execute_input":"2024-01-15T18:27:42.019806Z","iopub.status.idle":"2024-01-15T18:27:43.782305Z","shell.execute_reply.started":"2024-01-15T18:27:42.019773Z","shell.execute_reply":"2024-01-15T18:27:43.781425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[4].annotations","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.783855Z","iopub.execute_input":"2024-01-15T18:27:43.784223Z","iopub.status.idle":"2024-01-15T18:27:43.790500Z","shell.execute_reply.started":"2024-01-15T18:27:43.784189Z","shell.execute_reply":"2024-01-15T18:27:43.789607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.792025Z","iopub.execute_input":"2024-01-15T18:27:43.792384Z","iopub.status.idle":"2024-01-15T18:27:43.921003Z","shell.execute_reply.started":"2024-01-15T18:27:43.792352Z","shell.execute_reply":"2024-01-15T18:27:43.920159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.921970Z","iopub.execute_input":"2024-01-15T18:27:43.922254Z","iopub.status.idle":"2024-01-15T18:27:43.928170Z","shell.execute_reply.started":"2024-01-15T18:27:43.922230Z","shell.execute_reply":"2024-01-15T18:27:43.927309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.931852Z","iopub.execute_input":"2024-01-15T18:27:43.932647Z","iopub.status.idle":"2024-01-15T18:27:43.942357Z","shell.execute_reply.started":"2024-01-15T18:27:43.932620Z","shell.execute_reply":"2024-01-15T18:27:43.941431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.iloc[0]['long_answer_candidates']","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.943446Z","iopub.execute_input":"2024-01-15T18:27:43.944290Z","iopub.status.idle":"2024-01-15T18:27:43.953381Z","shell.execute_reply.started":"2024-01-15T18:27:43.944255Z","shell.execute_reply":"2024-01-15T18:27:43.952520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Part 1- identifying correct long answer to a question**","metadata":{}},{"cell_type":"code","source":"def get_question_and_document(line):\n    example_id = line['example_id']\n    question = line['question_text']\n    text = line['document_text'].split(' ')\n    annotations = line['annotations'][0]    \n    return question, text, annotations, example_id\n                \ndef get_long_candidate(i, annotations, candidate):\n    # check if this candidate is the correct long answer\n    if i == annotations['long_answer']['candidate_index']:\n        label = True\n    else:\n        label = False\n\n    # get place where long answer starts and ends in the document text\n    long_start = candidate['start_token']\n    long_end = candidate['end_token']    \n    return label, long_start, long_end\n\n\ndef form_data_row(question, label, text, long_start, long_end, example_id, annotations):\n    row = {\n        'example_id': example_id,\n        'annotations': annotations,\n        'question': question,\n        'long_answer': ' '.join(text[long_start:long_end]),\n        'is_long_answer': label,\n        'long_start_token': long_start,\n        'long_end_token': long_end\n    }    \n    return row","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.954518Z","iopub.execute_input":"2024-01-15T18:27:43.954765Z","iopub.status.idle":"2024-01-15T18:27:43.962590Z","shell.execute_reply.started":"2024-01-15T18:27:43.954742Z","shell.execute_reply":"2024-01-15T18:27:43.961745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_test_dataset(test_df):\n    new_test_df = []\n    for idx in test_df.index:\n        question = test_df.iloc[idx]['question_text']\n        text = test_df.iloc[idx]['document_text']\n        long_answer_candidates = test_df.iloc[idx]['long_answer_candidates']\n        for candidate in long_answer_candidates:\n            start_token = candidate['start_token']\n            end_token = candidate['end_token']\n            row = {\n                'example_id': test_df.iloc[idx]['example_id'],\n                'question': question,\n                'long_answer': ''.join(text[start_token:end_token]),\n                'long_start_token': start_token,\n                'long_end_token': end_token\n            }\n            new_test_df.append(row)\n            \n    return pd.DataFrame(new_test_df)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:43.963631Z","iopub.execute_input":"2024-01-15T18:27:43.965498Z","iopub.status.idle":"2024-01-15T18:27:43.976248Z","shell.execute_reply.started":"2024-01-15T18:27:43.965474Z","shell.execute_reply":"2024-01-15T18:27:43.975413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = process_test_dataset(test)\ntest_data.head()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-15T18:27:43.977302Z","iopub.execute_input":"2024-01-15T18:27:43.977624Z","iopub.status.idle":"2024-01-15T18:27:45.833538Z","shell.execute_reply.started":"2024-01-15T18:27:43.977593Z","shell.execute_reply":"2024-01-15T18:27:45.832638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(file_path, questions_start, questions_end):\n    rows = []\n    with open(file_path) as file:\n        for _ in range(questions_start - 1):\n            next(file)\n        \n        for outer_i in tqdm(range(questions_start, questions_end)):\n            line = file.readline()\n            line = json.loads(line)\n            question, text, annotations, example_id = get_question_and_document(line)            \n            candidate_index = annotations['long_answer']['candidate_index']\n            \n            for inner_i, candidate in enumerate(line['long_answer_candidates']):\n                label, long_start, long_end = get_long_candidate(inner_i, annotations, candidate)\n                \n                if label == True or (inner_i % sample_rate == 0):\n                    rows.append(\n                        form_data_row(question, int(label), text, long_start, long_end, example_id, annotations)\n                    )\n                    \n    return pd.DataFrame(rows)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:45.834943Z","iopub.execute_input":"2024-01-15T18:27:45.835389Z","iopub.status.idle":"2024-01-15T18:27:45.843878Z","shell.execute_reply.started":"2024-01-15T18:27:45.835351Z","shell.execute_reply":"2024-01-15T18:27:45.842906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define training parameters \nnum_train_ques = 10000\nnum_val_ques = 500\nsample_rate = 15","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:45.844996Z","iopub.execute_input":"2024-01-15T18:27:45.845307Z","iopub.status.idle":"2024-01-15T18:27:45.856746Z","shell.execute_reply.started":"2024-01-15T18:27:45.845276Z","shell.execute_reply":"2024-01-15T18:27:45.856016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_indices(num_train_ques, num_test_ques):\n    start_train = random.randint(0, 30000)\n    end_train = start_train + num_train_ques\n    \n    start_test = random.randint(end_train, 60000)\n    end_test = start_test + num_test_ques\n    \n    return start_train, end_train, start_test, end_test","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:45.857762Z","iopub.execute_input":"2024-01-15T18:27:45.858043Z","iopub.status.idle":"2024-01-15T18:27:45.866652Z","shell.execute_reply.started":"2024-01-15T18:27:45.858011Z","shell.execute_reply":"2024-01-15T18:27:45.865795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_train, end_train, start_val, end_val = get_random_indices(num_train_ques, num_val_ques)\nstart_train, end_train, start_val, end_val","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:45.867601Z","iopub.execute_input":"2024-01-15T18:27:45.867883Z","iopub.status.idle":"2024-01-15T18:27:45.878174Z","shell.execute_reply.started":"2024-01-15T18:27:45.867859Z","shell.execute_reply":"2024-01-15T18:27:45.877397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = load_data(train_path, start_train, end_train)\nval_data = load_data(train_path, start_val, end_val)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:27:45.879290Z","iopub.execute_input":"2024-01-15T18:27:45.879613Z","iopub.status.idle":"2024-01-15T18:28:31.562486Z","shell.execute_reply.started":"2024-01-15T18:27:45.879581Z","shell.execute_reply":"2024-01-15T18:28:31.561542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.563709Z","iopub.execute_input":"2024-01-15T18:28:31.563995Z","iopub.status.idle":"2024-01-15T18:28:31.581956Z","shell.execute_reply.started":"2024-01-15T18:28:31.563969Z","shell.execute_reply":"2024-01-15T18:28:31.581038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.582993Z","iopub.execute_input":"2024-01-15T18:28:31.583386Z","iopub.status.idle":"2024-01-15T18:28:31.604186Z","shell.execute_reply.started":"2024-01-15T18:28:31.583348Z","shell.execute_reply":"2024-01-15T18:28:31.603292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape, val_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.605357Z","iopub.execute_input":"2024-01-15T18:28:31.605703Z","iopub.status.idle":"2024-01-15T18:28:31.614528Z","shell.execute_reply.started":"2024-01-15T18:28:31.605671Z","shell.execute_reply":"2024-01-15T18:28:31.612824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Pre-processing texts**","metadata":{"execution":{"iopub.status.busy":"2022-01-07T13:00:09.060209Z","iopub.execute_input":"2022-01-07T13:00:09.060503Z","iopub.status.idle":"2022-01-07T13:00:09.065406Z","shell.execute_reply.started":"2022-01-07T13:00:09.060471Z","shell.execute_reply":"2022-01-07T13:00:09.064588Z"}}},{"cell_type":"code","source":"def preprocess_text(text):\n    # Removing HTML tags\n    html_removed = re.sub(r'<.*?>', '', text)\n    \n    # Removing stopwords\n    words = html_removed.split()\n    words = [word for word in words if word.lower() not in stopwords.words('english')]\n    \n    # Joining the words back into a sentence\n    processed_text = ' '.join(words)\n    \n    return processed_text\n\ndef preprocess_df(df):\n    df_copy = df.copy()\n    df_copy['long_answer'] = df_copy['long_answer'].apply(preprocess_text)\n    df_copy['question'] = df_copy['question'].apply(preprocess_text)\n    return df_copy","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.615629Z","iopub.execute_input":"2024-01-15T18:28:31.615966Z","iopub.status.idle":"2024-01-15T18:28:31.623122Z","shell.execute_reply.started":"2024-01-15T18:28:31.615935Z","shell.execute_reply":"2024-01-15T18:28:31.622220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data = preprocess_df(test_data)\n# test_data.to_csv('test_data.csv')\n# test_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.624210Z","iopub.execute_input":"2024-01-15T18:28:31.625341Z","iopub.status.idle":"2024-01-15T18:28:31.634370Z","shell.execute_reply.started":"2024-01-15T18:28:31.625315Z","shell.execute_reply":"2024-01-15T18:28:31.633583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = preprocess_df(train_data)\n# val_data = preprocess_df(val_data)\n# train_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.635558Z","iopub.execute_input":"2024-01-15T18:28:31.635901Z","iopub.status.idle":"2024-01-15T18:28:31.643854Z","shell.execute_reply.started":"2024-01-15T18:28:31.635867Z","shell.execute_reply":"2024-01-15T18:28:31.643027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data.to_csv('train_data.csv')\n# val_data.to_csv('val_data.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.645016Z","iopub.execute_input":"2024-01-15T18:28:31.645406Z","iopub.status.idle":"2024-01-15T18:28:31.652013Z","shell.execute_reply.started":"2024-01-15T18:28:31.645376Z","shell.execute_reply":"2024-01-15T18:28:31.651286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # load .csv for train, val, test data\n\n# train_data = pd.read_csv('/kaggle/input/csv-dataset/train_data.csv')\n# val_data = pd.read_csv('/kaggle/input/csv-dataset/val_data.csv')\n# test_data = pd.read_csv('/kaggle/input/csv-dataset/test_data.csv')\n\n# train_data['long_answer'] = train_data['long_answer'].astype(str)\n# val_data['long_answer'] = val_data['long_answer'].astype(str)\n# test_data['long_answer'] = test_data['long_answer'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.653126Z","iopub.execute_input":"2024-01-15T18:28:31.653405Z","iopub.status.idle":"2024-01-15T18:28:31.660588Z","shell.execute_reply.started":"2024-01-15T18:28:31.653382Z","shell.execute_reply":"2024-01-15T18:28:31.659883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Pre-processing***","metadata":{}},{"cell_type":"code","source":"# Shuffle training dataframe\n\ntrain_data = train_data.sample(frac=1, random_state=42)\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.668849Z","iopub.execute_input":"2024-01-15T18:28:31.669130Z","iopub.status.idle":"2024-01-15T18:28:31.706164Z","shell.execute_reply.started":"2024-01-15T18:28:31.669101Z","shell.execute_reply":"2024-01-15T18:28:31.705354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How many examples of each class?\ntrain_data.is_long_answer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.707107Z","iopub.execute_input":"2024-01-15T18:28:31.707368Z","iopub.status.idle":"2024-01-15T18:28:31.720389Z","shell.execute_reply.started":"2024-01-15T18:28:31.707345Z","shell.execute_reply":"2024-01-15T18:28:31.719545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Data is unbalanced, up-sampling\n\n# from sklearn.utils import resample\n# #create two different dataframe of majority and minority class \n# df_majority = train_df[(train_df['is_long_answer']==0)] \n# df_minority = train_df[(train_df['is_long_answer']==1)] \n# # upsample minority class\n# df_minority_upsampled = resample(df_minority, \n#                                  replace=True,    # sample with replacement\n#                                  n_samples= int((len(train_df) * 50) / 100), # to match majority class\n#                                  random_state=42)  # reproducible results\n# # Combine majority class with upsampled minority class\n# train_df = pd.concat([df_minority_upsampled, df_majority])\n# train_df.is_long_answer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.721750Z","iopub.execute_input":"2024-01-15T18:28:31.722368Z","iopub.status.idle":"2024-01-15T18:28:31.728805Z","shell.execute_reply.started":"2024-01-15T18:28:31.722332Z","shell.execute_reply":"2024-01-15T18:28:31.728053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Total training samples: {len(train_data)}\")\nprint(f\"Total validation samples: {len(val_data)}\")\nprint(f\"Total samples: {len(train_data) + len(val_data)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.729959Z","iopub.execute_input":"2024-01-15T18:28:31.730340Z","iopub.status.idle":"2024-01-15T18:28:31.741787Z","shell.execute_reply.started":"2024-01-15T18:28:31.730304Z","shell.execute_reply":"2024-01-15T18:28:31.740947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Visualize random training examples\n\n# import random\n# random_index = random.randint(0, len(train_data)-5) # create random indexes not higher than the total number of samples\n# for row in train_data[[\"question\", \"long_answer\", \"is_long_answer\"]][random_index:random_index+5].itertuples():\n#   _, q, la, ila = row\n#   print(f\"Target: {ila}\", \"(true label)\" if ila > 0 else \"(not true label)\")\n#   print(f\"Question:\\n{q}\\n\")\n#   print(f\"Long answer:\\n{la}\\n\")\n#   print(\"---\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.742795Z","iopub.execute_input":"2024-01-15T18:28:31.743473Z","iopub.status.idle":"2024-01-15T18:28:31.750594Z","shell.execute_reply.started":"2024-01-15T18:28:31.743442Z","shell.execute_reply":"2024-01-15T18:28:31.749874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Splitting into training and validation sets\n\n# from sklearn.model_selection import train_test_split\n\n# # Use train_test_split to split training data into training and validation sets\n# train_data, val_data, train_labels, val_labels = train_test_split(\n#     train_df[[\"question\", \"long_answer\"]],  # Select relevant columns\n#     train_df[\"is_long_answer\"],\n#     test_size=0.3,\n#     random_state=42\n# )\n\n# len(train_data), len(val_data), len(train_labels), len(val_labels)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.751831Z","iopub.execute_input":"2024-01-15T18:28:31.752512Z","iopub.status.idle":"2024-01-15T18:28:31.759356Z","shell.execute_reply.started":"2024-01-15T18:28:31.752480Z","shell.execute_reply":"2024-01-15T18:28:31.758496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Converting text into numbers***","metadata":{}},{"cell_type":"code","source":"# Text vectorization (tokenization)\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import TextVectorization\n\n# Use the default TextVectorization variables\ntext_vectorizer = TextVectorization(max_tokens=None, # how many words in the vocabulary (all of the different words in your text)\n                                    standardize=\"lower_and_strip_punctuation\", # how to process text\n                                    split=\"whitespace\", # how to split tokens\n                                    ngrams=None, # create groups of n-words?\n                                    output_mode=\"int\", # how to map tokens to numbers\n                                    output_sequence_length=None) # how long should the output sequence of tokens be?\n                                    # pad_to_max_tokens=True) # Not valid if using max_tokens=None","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:31.760408Z","iopub.execute_input":"2024-01-15T18:28:31.760891Z","iopub.status.idle":"2024-01-15T18:28:32.297026Z","shell.execute_reply.started":"2024-01-15T18:28:31.760859Z","shell.execute_reply":"2024-01-15T18:28:32.296065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find average number of tokens (words) in long answers\nround(sum([len(i.split()) for i in train_data.long_answer])/len(train_data.long_answer))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:32.298259Z","iopub.execute_input":"2024-01-15T18:28:32.298522Z","iopub.status.idle":"2024-01-15T18:28:32.884003Z","shell.execute_reply.started":"2024-01-15T18:28:32.298499Z","shell.execute_reply":"2024-01-15T18:28:32.883022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup text vectorization with custom variables\nmax_vocab_length = 40000 # max number of words to have in our vocabulary; true value ~ 67k\nmax_length = round(sum([len(i.split()) for i in train_data.long_answer])/len(train_data.long_answer)) # max length our sequences will be (e.g. how many words from a sentence does our model see?)\n\ntext_vectorizer = TextVectorization(max_tokens=max_vocab_length,\n                                    output_mode=\"int\",\n                                    output_sequence_length=max_length)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:32.885090Z","iopub.execute_input":"2024-01-15T18:28:32.885382Z","iopub.status.idle":"2024-01-15T18:28:33.447800Z","shell.execute_reply.started":"2024-01-15T18:28:32.885356Z","shell.execute_reply":"2024-01-15T18:28:33.447050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the text vectorizer to the training text\n# Combine 'question' and 'long_answer' into a single column\ntrain_sentences = train_data['question'] + ' ' + train_data['long_answer']\n\n# Adapt the vectorizer on the training data\ntext_vectorizer.adapt(train_sentences)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:33.448908Z","iopub.execute_input":"2024-01-15T18:28:33.449198Z","iopub.status.idle":"2024-01-15T18:28:41.811756Z","shell.execute_reply.started":"2024-01-15T18:28:33.449173Z","shell.execute_reply":"2024-01-15T18:28:41.810719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the unique words in the vocabulary\nwords_in_vocab = text_vectorizer.get_vocabulary()\ntop_5_words = words_in_vocab[:5] # most common tokens (notice the [UNK] token for \"unknown\" words)\nbottom_5_words = words_in_vocab[-5:] # least common tokens\nprint(f\"Number of words in vocab: {len(words_in_vocab)}\")\nprint(f\"Top 5 most common words: {top_5_words}\") \nprint(f\"Bottom 5 least common words: {bottom_5_words}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:41.813086Z","iopub.execute_input":"2024-01-15T18:28:41.813403Z","iopub.status.idle":"2024-01-15T18:28:41.919490Z","shell.execute_reply.started":"2024-01-15T18:28:41.813377Z","shell.execute_reply":"2024-01-15T18:28:41.918555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Embedding layer***","metadata":{}},{"cell_type":"code","source":"# pretraining\n\nembed_size = 200\nembedding_dict = {}\n\nwith open('../input/glove6b/glove.6B.' + str(embed_size) + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors        \nf.close()\n\nnum_words = len(words_in_vocab) + 1","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:28:41.920668Z","iopub.execute_input":"2024-01-15T18:28:41.920974Z","iopub.status.idle":"2024-01-15T18:29:04.684852Z","shell.execute_reply.started":"2024-01-15T18:28:41.920948Z","shell.execute_reply":"2024-01-15T18:29:04.683799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initializing embedding matrix\nembedding_matrix = np.zeros((num_words, embed_size))\nprint(\"num_words:\", num_words)\nprint(\"embedding_matrix:\", embedding_matrix)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:04.686195Z","iopub.execute_input":"2024-01-15T18:29:04.686552Z","iopub.status.idle":"2024-01-15T18:29:04.692860Z","shell.execute_reply.started":"2024-01-15T18:29:04.686517Z","shell.execute_reply":"2024-01-15T18:29:04.691966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, word in enumerate(words_in_vocab):\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:04.693988Z","iopub.execute_input":"2024-01-15T18:29:04.694335Z","iopub.status.idle":"2024-01-15T18:29:04.832937Z","shell.execute_reply.started":"2024-01-15T18:29:04.694303Z","shell.execute_reply":"2024-01-15T18:29:04.832110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(42)\nfrom tensorflow.keras import layers\n\n# embedding_layer = layers.Embedding(input_dim=max_vocab_length, # set input shape\n#                              output_dim=128, # set size of embedding vector\n#                              embeddings_initializer=\"uniform\", # default, intialize randomly\n#                              input_length=max_length, # how long is each input\n#                              name=\"embedding_1\") \n\n#initialize embedding layer\nembedding_layer = tf.keras.layers.Embedding(\n    len(words_in_vocab) + 1,\n    embed_size,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)\n\nembedding_layer","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:04.834196Z","iopub.execute_input":"2024-01-15T18:29:04.834552Z","iopub.status.idle":"2024-01-15T18:29:04.849052Z","shell.execute_reply.started":"2024-01-15T18:29:04.834519Z","shell.execute_reply":"2024-01-15T18:29:04.848123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a random sentence from training set\nrandom_sentence = random.choice(train_data.long_answer)\nprint(f\"Original text:\\n{random_sentence}\\\n      \\n\\nEmbedded version:\")\n\n# Embed the random sentence (turn it into numerical representation)\nsample_embed = embedding_layer(text_vectorizer([random_sentence]))\nsample_embed","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:04.850274Z","iopub.execute_input":"2024-01-15T18:29:04.850550Z","iopub.status.idle":"2024-01-15T18:29:05.031186Z","shell.execute_reply.started":"2024-01-15T18:29:04.850527Z","shell.execute_reply":"2024-01-15T18:29:05.030250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check out a single token's embedding\nsample_embed[0][0]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.032473Z","iopub.execute_input":"2024-01-15T18:29:05.032864Z","iopub.status.idle":"2024-01-15T18:29:05.045109Z","shell.execute_reply.started":"2024-01-15T18:29:05.032825Z","shell.execute_reply":"2024-01-15T18:29:05.044133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Model definition***","metadata":{}},{"cell_type":"code","source":"# question encoding-encodes the question\nquestion_input = layers.Input(shape=(1,), dtype=\"string\")\nquestion_x = text_vectorizer(question_input) # turn the input text into numbers\nquestion_x = embedding_layer(question_x) # create an embedding of the numerized numbers\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Conv1D(32, kernel_size=3, activation='relu')(question_x)\nquestion_x = MaxPooling1D(pool_size=2)(question_x)\n\n# answer encoding-encodes the answer\nanswer_input = Input(shape=(1,), dtype=\"string\")\nanswer_x = text_vectorizer(answer_input)\nanswer_x = embedding_layer(answer_x)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Conv1D(64, kernel_size=3, activation='relu')(answer_x)\nanswer_x = MaxPooling1D(pool_size=2)(answer_x)\n\n# classification\ncombined_x = concatenate([question_x, answer_x])\ncombined_x = Flatten()(combined_x)  # Flatten before passing to Dense layers\ncombined_x = Dropout(0.3)(combined_x)\nmodel_output = Dense(1, activation='sigmoid')(combined_x) # probability how close the potential answer is to the true answer to the question.\n\n# defining model by combining above three parts\nmodel = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=model_output)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.046329Z","iopub.execute_input":"2024-01-15T18:29:05.046932Z","iopub.status.idle":"2024-01-15T18:29:05.216872Z","shell.execute_reply.started":"2024-01-15T18:29:05.046898Z","shell.execute_reply":"2024-01-15T18:29:05.215930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.218060Z","iopub.execute_input":"2024-01-15T18:29:05.218371Z","iopub.status.idle":"2024-01-15T18:29:05.258538Z","shell.execute_reply.started":"2024-01-15T18:29:05.218346Z","shell.execute_reply":"2024-01-15T18:29:05.257589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    loss='binary_crossentropy', \n    optimizer='sgd',\n    metrics=['BinaryAccuracy', 'Recall', 'Precision'])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.259862Z","iopub.execute_input":"2024-01-15T18:29:05.260205Z","iopub.status.idle":"2024-01-15T18:29:05.278969Z","shell.execute_reply.started":"2024-01-15T18:29:05.260172Z","shell.execute_reply":"2024-01-15T18:29:05.278144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensorboard_callback = create_tensorboard_callback(dir_name=\"model_long\", experiment_name=\"glove\")\n\n#define callbacks - to avoid plateauing & achieve early stopping\ncallbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=4, verbose=1),\n    tensorboard_callback]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.280203Z","iopub.execute_input":"2024-01-15T18:29:05.280835Z","iopub.status.idle":"2024-01-15T18:29:05.293001Z","shell.execute_reply.started":"2024-01-15T18:29:05.280801Z","shell.execute_reply":"2024-01-15T18:29:05.292140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define model parameters\n\nfrom sklearn.utils.class_weight import compute_class_weight\n\nepochs = 30\nbatch_size = 256\n\n# dealing with unbalanced data for the classification problem\nclass_weights = compute_class_weight('balanced', classes=np.unique(train_data['is_long_answer']), y=train_data['is_long_answer'])\nclass_weights_dict = {0: class_weights[0], 1: class_weights[1]}\nclass_weights_dict","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.294144Z","iopub.execute_input":"2024-01-15T18:29:05.294464Z","iopub.status.idle":"2024-01-15T18:29:05.326920Z","shell.execute_reply.started":"2024-01-15T18:29:05.294432Z","shell.execute_reply":"2024-01-15T18:29:05.326122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    x=[train_data['long_answer'], train_data['question']],\n    y=train_data['is_long_answer'],\n    validation_data=([val_data['long_answer'], val_data['question']], val_data['is_long_answer']),\n    epochs=epochs,\n    batch_size=batch_size,\n    callbacks=callbacks,\n    class_weight=class_weights_dict,\n    shuffle=True\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:29:05.327961Z","iopub.execute_input":"2024-01-15T18:29:05.328215Z","iopub.status.idle":"2024-01-15T18:30:05.227576Z","shell.execute_reply.started":"2024-01-15T18:29:05.328193Z","shell.execute_reply":"2024-01-15T18:30:05.226652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #save model\n# model.save('long_model_no_pretraining')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:05.228984Z","iopub.execute_input":"2024-01-15T18:30:05.229781Z","iopub.status.idle":"2024-01-15T18:30:05.233644Z","shell.execute_reply.started":"2024-01-15T18:30:05.229737Z","shell.execute_reply":"2024-01-15T18:30:05.232753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Model evaluation***","metadata":{}},{"cell_type":"code","source":"model.evaluate([val_data['long_answer'], val_data['question']], val_data['is_long_answer'])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:05.234817Z","iopub.execute_input":"2024-01-15T18:30:05.235225Z","iopub.status.idle":"2024-01-15T18:30:06.573105Z","shell.execute_reply.started":"2024-01-15T18:30:05.235188Z","shell.execute_reply":"2024-01-15T18:30:06.572130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_prediction_probs = model.predict([val_data['long_answer'], val_data['question']])\nmodel_prediction_probs[:10]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:06.574374Z","iopub.execute_input":"2024-01-15T18:30:06.574740Z","iopub.status.idle":"2024-01-15T18:30:07.371210Z","shell.execute_reply.started":"2024-01-15T18:30:06.574695Z","shell.execute_reply":"2024-01-15T18:30:07.370312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_prediction_probs = tf.squeeze(tf.round(model_prediction_probs)) # squeeze removes single dimensions\nmodel_prediction_probs[:20]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:07.372404Z","iopub.execute_input":"2024-01-15T18:30:07.372679Z","iopub.status.idle":"2024-01-15T18:30:07.383834Z","shell.execute_reply.started":"2024-01-15T18:30:07.372655Z","shell.execute_reply":"2024-01-15T18:30:07.382983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n\ndef calculate_results(y_true, y_pred):\n  \"\"\"\n  Calculates model accuracy, precision, recall and f1 score of a binary classification model.\n\n  Args:\n  -----\n  y_true = true labels in the form of a 1D array\n  y_pred = predicted labels in the form of a 1D array\n\n  Returns a dictionary of accuracy, precision, recall, f1-score.\n  \"\"\"\n  # Calculate model accuracy\n  model_accuracy = accuracy_score(y_true, y_pred) * 100\n  # Calculate model precision, recall and f1 score using \"weighted\" average\n  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\n  model_results = {\"accuracy\": model_accuracy,\n                  \"precision\": model_precision,\n                  \"recall\": model_recall,\n                  \"f1\": model_f1}\n  return model_results","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-15T18:30:07.385246Z","iopub.execute_input":"2024-01-15T18:30:07.385811Z","iopub.status.idle":"2024-01-15T18:30:07.391743Z","shell.execute_reply.started":"2024-01-15T18:30:07.385776Z","shell.execute_reply":"2024-01-15T18:30:07.390810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true=val_data['is_long_answer']\ny_pred=model_prediction_probs\n\naccuracy = accuracy_score(y_true, y_pred)\nrecall = recall_score(y_true, y_pred)\nprecision = precision_score(y_true, y_pred)\nf1 = f1_score(y_true, y_pred)\n\n# model_1_results = calculate_results(y_true=val_data['is_long_answer'], \n#                                     y_pred=model_prediction_probs)\n# model_1_results\n\naccuracy, precision, recall, f1","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:07.392849Z","iopub.execute_input":"2024-01-15T18:30:07.393104Z","iopub.status.idle":"2024-01-15T18:30:07.422745Z","shell.execute_reply.started":"2024-01-15T18:30:07.393062Z","shell.execute_reply":"2024-01-15T18:30:07.422005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_prediction_probs_test = model.predict([test_data['long_answer'], test_data['question']])\nmodel_prediction_probs_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:07.423716Z","iopub.execute_input":"2024-01-15T18:30:07.424012Z","iopub.status.idle":"2024-01-15T18:30:11.871297Z","shell.execute_reply.started":"2024-01-15T18:30:07.423989Z","shell.execute_reply":"2024-01-15T18:30:11.870322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values, counts = np.unique(model_prediction_probs_test, return_counts=True)\nunique_values, counts","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:11.872872Z","iopub.execute_input":"2024-01-15T18:30:11.873254Z","iopub.status.idle":"2024-01-15T18:30:11.880991Z","shell.execute_reply.started":"2024-01-15T18:30:11.873211Z","shell.execute_reply":"2024-01-15T18:30:11.880031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# long_writer = SummaryWriter()\n# train_loss = history.history['loss']\n# train_accuracy = history.history['binary_accuracy']\n# train_precision = history.history['precision']\n# train_recall = history.history['recall']\n\n# # Log values for each epoch\n# for epoch, (loss, accuracy, precision, recall) in enumerate(zip(train_loss, train_accuracy, train_precision, train_recall), 1):\n#     long_writer.add_scalar(\"Train/Epoch/Loss\", loss, epoch)\n#     long_writer.add_scalar(\"Train/Epoch/Accuracy\", accuracy, epoch)\n#     long_writer.add_scalar(\"Train/Epoch/Precision\", precision, epoch)\n#     long_writer.add_scalar(\"Train/Epoch/Recall\", recall, epoch)\n\n# long_writer.add_scalar(\"Validation/F1\", f1, 1)\n# long_writer.close()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:11.882057Z","iopub.execute_input":"2024-01-15T18:30:11.882325Z","iopub.status.idle":"2024-01-15T18:30:11.889020Z","shell.execute_reply.started":"2024-01-15T18:30:11.882302Z","shell.execute_reply":"2024-01-15T18:30:11.888086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r long-model.zip /kaggle/working/model_long","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:11.890105Z","iopub.execute_input":"2024-01-15T18:30:11.890378Z","iopub.status.idle":"2024-01-15T18:30:29.073159Z","shell.execute_reply.started":"2024-01-15T18:30:11.890354Z","shell.execute_reply":"2024-01-15T18:30:29.072036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_data(dataframe, prediction_probabilities, col):\n    # Create a DataFrame from the prediction probabilities\n    dataframe['probability'] = tf.squeeze(prediction_probabilities)\n\n    # Sort the DataFrame by the probability in descending order\n    df_sorted = dataframe.sort_values(by='probability', ascending=False)\n\n    # Keep only the rows with the highest probability for each example_id\n    df_filtered = df_sorted.groupby(col).first().reset_index()\n    df_filtered['probability'] = 1\n    return df_filtered\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.074607Z","iopub.execute_input":"2024-01-15T18:30:29.074936Z","iopub.status.idle":"2024-01-15T18:30:29.081391Z","shell.execute_reply.started":"2024-01-15T18:30:29.074907Z","shell.execute_reply":"2024-01-15T18:30:29.080341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values, counts = np.unique(model_prediction_probs, return_counts=True)\nunique_values, counts","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.082578Z","iopub.execute_input":"2024-01-15T18:30:29.082858Z","iopub.status.idle":"2024-01-15T18:30:29.094062Z","shell.execute_reply.started":"2024-01-15T18:30:29.082835Z","shell.execute_reply":"2024-01-15T18:30:29.093143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_data = filter_data(test_data, model_prediction_probs_test, 'example_id')\nnew_test_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.095223Z","iopub.execute_input":"2024-01-15T18:30:29.095718Z","iopub.status.idle":"2024-01-15T18:30:29.147226Z","shell.execute_reply.started":"2024-01-15T18:30:29.095694Z","shell.execute_reply":"2024-01-15T18:30:29.146285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_val_data = filter_data(val_data, model_prediction_probs, 'question')\nnew_val_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.148518Z","iopub.execute_input":"2024-01-15T18:30:29.148791Z","iopub.status.idle":"2024-01-15T18:30:29.164064Z","shell.execute_reply.started":"2024-01-15T18:30:29.148767Z","shell.execute_reply":"2024-01-15T18:30:29.163273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values, counts = np.unique(new_val_data['is_long_answer'], return_counts=True)\nunique_values, counts","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.165064Z","iopub.execute_input":"2024-01-15T18:30:29.165357Z","iopub.status.idle":"2024-01-15T18:30:29.172049Z","shell.execute_reply.started":"2024-01-15T18:30:29.165333Z","shell.execute_reply":"2024-01-15T18:30:29.171199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_data","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:30:29.173176Z","iopub.execute_input":"2024-01-15T18:30:29.173480Z","iopub.status.idle":"2024-01-15T18:30:29.189109Z","shell.execute_reply.started":"2024-01-15T18:30:29.173456Z","shell.execute_reply":"2024-01-15T18:30:29.188230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Part 2- extracting short answer from a long answer**","metadata":{}},{"cell_type":"code","source":"#part 2\n\n#filter records where short answers exist\ndef get_short_answer(annotations, long_start, long_end):\n    if len(annotations['short_answers']) > 0:\n        short_start = annotations['short_answers'][0]['start_token']\n        short_end = annotations['short_answers'][0]['end_token']        \n        short_start = short_start - long_start\n        short_end = short_end - long_start        \n        return short_start, short_end\n    else:\n        return 0, 0\n    \ndef form_short_data_row(question, text, long_start, long_end, short_start, short_end):\n    long_answer = ' '.join(text[long_start:long_end])\n    short_answer = ' '.join(long_answer.split(' ')[short_start:short_end])\n    row = {\n        'question': question,\n        'long_answer': long_answer,\n        'short_answer': short_answer,\n        'short_start': short_start,\n        'short_end': short_end\n    }    \n    return row","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:49.845751Z","iopub.execute_input":"2024-01-15T18:46:49.846141Z","iopub.status.idle":"2024-01-15T18:46:49.853968Z","shell.execute_reply.started":"2024-01-15T18:46:49.846110Z","shell.execute_reply":"2024-01-15T18:46:49.853008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading short answers\n# def load_short_data(file_path, questions_start, questions_end):\n#     rows = []    \n#     with open(file_path) as file:\n#         for _ in range(questions_start - 1):\n#             next(file)\n\n#         for i in tqdm(range(questions_start, questions_end)):\n#             line = file.readline()\n#             line = json.loads(line)\n#             question, text, annotations = get_question_and_document(line)\n#             for i, candidate in enumerate(line['long_answer_candidates']):\n#                 label, long_start, long_end = get_long_candidate(i, annotations, candidate)\n\n#                 if label == True:\n#                     short_start, short_end = get_short_answer(annotations, long_start, long_end)\n                    \n#                     rows.append(\n#                         form_short_data_row(question, text, long_start, long_end, short_start, short_end)\n#                     )\n#     return pd.DataFrame(rows)\n\n\ndef load_short_data(dataframe, check_col):\n    new_df = []\n\n    for index, row in dataframe.iterrows():\n        if row[check_col] == 1:\n            annotations = row['annotations']\n            short_start, short_end = get_short_answer(annotations, row['long_start_token'], row['long_end_token'])\n            short_answer = ' '.join(row['long_answer'].split(' ')[short_start:short_end])\n            # Create a new row with existing values and short start/end\n            new_row = row.to_dict()\n            new_row['short_start'] = short_start\n            new_row['short_end'] = short_end\n            new_row['short_answer'] = short_answer\n\n            # Append the new row to the new_df list\n            new_df.append(new_row)\n\n    # Convert the list of dictionaries to a DataFrame\n    new_df = pd.DataFrame(new_df)\n\n    return new_df","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:49.871164Z","iopub.execute_input":"2024-01-15T18:46:49.871461Z","iopub.status.idle":"2024-01-15T18:46:49.879065Z","shell.execute_reply.started":"2024-01-15T18:46:49.871436Z","shell.execute_reply":"2024-01-15T18:46:49.878059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_short_data = load_short_data(train_path, start_train, end_train)\n# val_short_data = load_short_data(train_path, start_val, end_val)\n\ntrain_short_data = load_short_data(train_data, 'is_long_answer')\nval_short_data = load_short_data(val_data, 'is_long_answer')\n# val_short_data = load_short_data(new_val_data, 'probability') # for full model","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:49.889355Z","iopub.execute_input":"2024-01-15T18:46:49.889620Z","iopub.status.idle":"2024-01-15T18:46:55.768111Z","shell.execute_reply.started":"2024-01-15T18:46:49.889596Z","shell.execute_reply":"2024-01-15T18:46:55.767127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_short_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:55.769659Z","iopub.execute_input":"2024-01-15T18:46:55.769961Z","iopub.status.idle":"2024-01-15T18:46:55.795950Z","shell.execute_reply.started":"2024-01-15T18:46:55.769934Z","shell.execute_reply":"2024-01-15T18:46:55.794966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_short_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:55.797640Z","iopub.execute_input":"2024-01-15T18:46:55.797981Z","iopub.status.idle":"2024-01-15T18:46:55.832206Z","shell.execute_reply.started":"2024-01-15T18:46:55.797948Z","shell.execute_reply":"2024-01-15T18:46:55.831228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check count values in each column\ndef count_values_in_column(data,feature):\n    total=data.loc[:,feature].value_counts(dropna=False)\n    percentage=round(data.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)\n    return pd.concat([total,percentage],axis=1,keys=['Total','Percentage'])\n\ncount_values_in_column(train_short_data, 'short_answer')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:55.834689Z","iopub.execute_input":"2024-01-15T18:46:55.835419Z","iopub.status.idle":"2024-01-15T18:46:55.858240Z","shell.execute_reply.started":"2024-01-15T18:46:55.835393Z","shell.execute_reply":"2024-01-15T18:46:55.857291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tokenizing\n#tokenization params\nfilters = '!\"''#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\nlower_case = True\nmax_len = 500 #max lenght of a sentence input in to the model\n\n#assigning numeric index to each unique work\ndef define_tokenizer(series):\n    sentences = pd.concat(series)    \n    tokenizer = tf.keras.preprocessing.text.Tokenizer(lower=lower_case,filters=filters)\n    tokenizer.fit_on_texts(sentences)\n    return tokenizer\n\n#encoding\ndef encode(sentences, tokenizer):\n    encoded_sentences = tokenizer.texts_to_sequences(sentences)\n    encoded_sentences = tf.keras.preprocessing.sequence.pad_sequences(encoded_sentences,\n                                                                      maxlen=max_len, padding='post')\n    return encoded_sentences\n\ntokenizer = define_tokenizer([train_data.long_answer, train_data.question, val_data.long_answer, val_data.question])\n\ntrain_long_ans = encode(train_short_data['long_answer'].values, tokenizer)\ntrain_questions = encode(train_short_data['question'].values, tokenizer)\n\nval_long_ans = encode(val_short_data['long_answer'].values, tokenizer)\nval_questions = encode(val_short_data['question'].values, tokenizer)\n\ntest_long_ans = encode(new_test_data['long_answer'].values, tokenizer)\ntest_questions = encode(new_test_data['question'].values, tokenizer)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:46:55.859299Z","iopub.execute_input":"2024-01-15T18:46:55.859558Z","iopub.status.idle":"2024-01-15T18:47:08.032188Z","shell.execute_reply.started":"2024-01-15T18:46:55.859535Z","shell.execute_reply":"2024-01-15T18:47:08.031378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define 2 arrays for the start index and another for the end index\ndef form_short_labels(df, sentence_length):\n    start_labels = np.zeros((len(df), sentence_length))\n    end_labels = np.zeros((len(df), sentence_length))\n\n    #get the token indexes from short_start and short_end columns and assign it to new arrays. encoding with 1\n    for i in range(len(df)):\n        start = df.loc[i].short_start\n        end = df.loc[i].short_end\n        \n        if 0 <= start < sentence_length and 0 <= end < sentence_length:\n            start_labels[i, start] = 1\n            end_labels[i, end] = 1\n        else:\n            continue\n    return start_labels, end_labels\n\ntrain_start_labels, train_end_labels = form_short_labels(train_short_data, max_len)\nval_start_labels, val_end_labels = form_short_labels(val_short_data, max_len)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.033385Z","iopub.execute_input":"2024-01-15T18:47:08.033704Z","iopub.status.idle":"2024-01-15T18:47:08.841221Z","shell.execute_reply.started":"2024-01-15T18:47:08.033678Z","shell.execute_reply":"2024-01-15T18:47:08.840413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_start_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.842342Z","iopub.execute_input":"2024-01-15T18:47:08.842698Z","iopub.status.idle":"2024-01-15T18:47:08.849375Z","shell.execute_reply.started":"2024-01-15T18:47:08.842664Z","shell.execute_reply":"2024-01-15T18:47:08.848488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_short_data.loc[10].question)\n\nprint(train_short_data.loc[10].long_answer)\nprint(train_short_data.loc[10].short_answer)\n\nprint('Start index: {0}'.format(train_start_labels[10]))\nprint('End index: {0}'.format(train_end_labels[10]))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.850382Z","iopub.execute_input":"2024-01-15T18:47:08.850624Z","iopub.status.idle":"2024-01-15T18:47:08.867773Z","shell.execute_reply.started":"2024-01-15T18:47:08.850603Z","shell.execute_reply":"2024-01-15T18:47:08.866880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# short answer model parameters\nshort_epochs = 10\nshort_batch_size = 32\nembed_size_short= 200","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.868832Z","iopub.execute_input":"2024-01-15T18:47:08.869163Z","iopub.status.idle":"2024-01-15T18:47:08.878618Z","shell.execute_reply.started":"2024-01-15T18:47:08.869125Z","shell.execute_reply":"2024-01-15T18:47:08.877785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Evaluation**","metadata":{}},{"cell_type":"code","source":"# # write to matrix after loading from file\n\n# vocab = train_data.long_answer + train_data.question + val_data.long_answer + val_data.question\n# embedding_dict = {word: i for i, word in enumerate(vocab)}\n\n\n# num_words = len(tokenizer.word_index) + 1\n# embedding_matrix = np.zeros((num_words, embed_size_short))\n\n# for word, i in tokenizer.word_index.items():\n#     if i > num_words:\n#         continue\n    \n#     emb_vec = embedding_dict.get(word)\n    \n#     if emb_vec is not None:\n#         embedding_matrix[i] = emb_vec\n        \n        \n# # load as tensorflow embedding\n# #define embedding layer for the short model\n# embedding_layer2 = tf.keras.layers.Embedding(\n#     len(tokenizer.word_index) + 1,\n#     embed_size_short,\n#     embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n#     trainable = True\n# )","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.882434Z","iopub.execute_input":"2024-01-15T18:47:08.882701Z","iopub.status.idle":"2024-01-15T18:47:08.890303Z","shell.execute_reply.started":"2024-01-15T18:47:08.882678Z","shell.execute_reply":"2024-01-15T18:47:08.889533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load from file\nembedding_dict = {}\n\nwith open('../input/glove6b/glove.6B.200' + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors\n        \nf.close()\n\n# write to matrix\nnum_words = len(tokenizer.word_index) + 1\nembedding_matrix = np.zeros((num_words, embed_size_short))\n\nfor word, i in tokenizer.word_index.items():\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec\n        \n# load as tensorflow embedding\n# define embedding layer for the short model\nembedding_layer2 = tf.keras.layers.Embedding(\n    len(tokenizer.word_index) + 1,\n    embed_size_short,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:08.891489Z","iopub.execute_input":"2024-01-15T18:47:08.891745Z","iopub.status.idle":"2024-01-15T18:47:30.739316Z","shell.execute_reply.started":"2024-01-15T18:47:08.891722Z","shell.execute_reply":"2024-01-15T18:47:30.738505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#encode the question input\nquestion_input = Input(shape=(None,))\nquestion_x = embedding_layer2(question_input)\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Bidirectional(LSTM(200, return_sequences=True))(question_x)\nquestion_x = Bidirectional(LSTM(100, return_sequences=True))(question_x)\n\n#encode the answer input\nanswer_input = Input(shape=(None,))\nanswer_x = embedding_layer2(answer_input)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Bidirectional(LSTM(250, return_sequences=True))(answer_x)\nanswer_x = Bidirectional(LSTM(150, return_sequences=True))(answer_x)\n\ncombined_x = concatenate([question_x, answer_x])\n\n#predict start idx of the short answer\nstart_x = Dropout(0.1)(combined_x) \nstart_x = Conv1D(1,1)(start_x)\nstart_x = Flatten()(start_x)\nstart_x = Activation('softmax', name='start_token')(start_x)\n\n#predict end idx\nend_x = Dropout(0.1)(combined_x) \nend_x = Conv1D(1,1)(end_x)\nend_x = Flatten()(end_x)\nend_x = Activation('softmax', name='end_token')(end_x)\n\nshort_model = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=[start_x, end_x])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:30.740545Z","iopub.execute_input":"2024-01-15T18:47:30.741194Z","iopub.status.idle":"2024-01-15T18:47:33.010259Z","shell.execute_reply.started":"2024-01-15T18:47:30.741156Z","shell.execute_reply":"2024-01-15T18:47:33.009467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.compile(\n    loss='categorical_crossentropy', \n    optimizer='adam',\n    metrics=['categorical_accuracy', 'Recall', 'Precision'])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:33.011320Z","iopub.execute_input":"2024-01-15T18:47:33.011600Z","iopub.status.idle":"2024-01-15T18:47:33.024367Z","shell.execute_reply.started":"2024-01-15T18:47:33.011574Z","shell.execute_reply":"2024-01-15T18:47:33.023524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:33.025504Z","iopub.execute_input":"2024-01-15T18:47:33.025819Z","iopub.status.idle":"2024-01-15T18:47:33.083850Z","shell.execute_reply.started":"2024-01-15T18:47:33.025787Z","shell.execute_reply":"2024-01-15T18:47:33.082973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define callbacks for the short model\n#inreased patience or number of epochs with no improvement after which training will be stopped.\ntensorboard_callback_short = create_tensorboard_callback(dir_name=\"model_short\", experiment_name=\"glove\")\ncallbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=4, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1),\n    tensorboard_callback_short\n    ]","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:33.084909Z","iopub.execute_input":"2024-01-15T18:47:33.085196Z","iopub.status.idle":"2024-01-15T18:47:33.090958Z","shell.execute_reply.started":"2024-01-15T18:47:33.085171Z","shell.execute_reply":"2024-01-15T18:47:33.090098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = short_model.fit(\n    x = [train_long_ans, train_questions], \n    y = [train_start_labels, train_end_labels],\n    epochs = short_epochs, \n    callbacks = callbacks,\n    validation_data = ([val_long_ans, val_questions], [val_start_labels, val_end_labels]),\n    batch_size = short_batch_size,  \n    shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:47:33.092247Z","iopub.execute_input":"2024-01-15T18:47:33.092799Z","iopub.status.idle":"2024-01-15T18:55:53.035376Z","shell.execute_reply.started":"2024-01-15T18:47:33.092773Z","shell.execute_reply":"2024-01-15T18:55:53.034565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# short_model.save('short_model_no_pretraining')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:55:53.036613Z","iopub.execute_input":"2024-01-15T18:55:53.036879Z","iopub.status.idle":"2024-01-15T18:55:53.040999Z","shell.execute_reply.started":"2024-01-15T18:55:53.036856Z","shell.execute_reply":"2024-01-15T18:55:53.040084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Epoch: {0}'.format(len(history.history['loss'])))\nprint('Loss: {0}'.format(history.history['loss'][-1]))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:55:53.042219Z","iopub.execute_input":"2024-01-15T18:55:53.042573Z","iopub.status.idle":"2024-01-15T18:55:53.051410Z","shell.execute_reply.started":"2024-01-15T18:55:53.042538Z","shell.execute_reply":"2024-01-15T18:55:53.050467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training final results')\n\naccuracy = history.history['start_token_categorical_accuracy'][-1]\nrecall = history.history['start_token_recall'][-1]\nprecision = history.history['start_token_precision'][-1]\n\nprint('--------------------------------------------------')\nprint('Start token accuracy: {0}'.format(accuracy))\nprint('Start token recall: {0}'.format(recall))\nprint('Start token precision: {0}'.format(precision))\nprint('Start token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))\n\nprint('--------------------------------------------------')\n\naccuracy = history.history['end_token_categorical_accuracy'][-1]\nrecall = history.history['end_token_recall_1'][-1]\nprecision = history.history['end_token_precision_1'][-1]\n\nprint('End token accuracy: {0}'.format(accuracy))\nprint('End token recall: {0}'.format(recall))\nprint('End token precision: {0}'.format(precision))\nprint('End token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:55:53.052407Z","iopub.execute_input":"2024-01-15T18:55:53.052700Z","iopub.status.idle":"2024-01-15T18:55:53.063667Z","shell.execute_reply.started":"2024-01-15T18:55:53.052675Z","shell.execute_reply":"2024-01-15T18:55:53.062779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation final results')\nprint('--------------------------------------------------')\n\naccuracy = history.history['val_start_token_categorical_accuracy'][-1]\nrecall = history.history['val_start_token_recall'][-1]\nprecision = history.history['val_start_token_precision'][-1]\n\nprint('Start token accuracy: {0}'.format(accuracy))\nprint('Start token recall: {0}'.format(recall))\nprint('Start token precision: {0}'.format(precision))\nprint('Start token F1 score: {0:.4f}'.format( 2 * (precision * recall) / (precision + recall)))\n\nprint('--------------------------------------------------')\n\naccuracy = history.history['val_end_token_categorical_accuracy'][-1]\nrecall = history.history['val_end_token_recall_1'][-1]\nprecision = history.history['val_end_token_precision_1'][-1]\n\nprint('End token accuracy: {0}'.format(accuracy))\nprint('End token recall: {0}'.format(recall))\nprint('End token precision: {0}'.format(precision))\nprint('End token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:55:53.064753Z","iopub.execute_input":"2024-01-15T18:55:53.065131Z","iopub.status.idle":"2024-01-15T18:55:53.077416Z","shell.execute_reply.started":"2024-01-15T18:55:53.065098Z","shell.execute_reply":"2024-01-15T18:55:53.076710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.evaluate([val_long_ans, val_questions], [val_start_labels, val_end_labels])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:55:53.078422Z","iopub.execute_input":"2024-01-15T18:55:53.078704Z","iopub.status.idle":"2024-01-15T18:55:54.090799Z","shell.execute_reply.started":"2024-01-15T18:55:53.078680Z","shell.execute_reply":"2024-01-15T18:55:54.089895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model_prediction_probs = short_model.predict([test_long_ans, test_questions])\n# short_model_prediction_probs = short_model.predict([val_long_ans, val_questions])","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:00.864052Z","iopub.execute_input":"2024-01-15T18:58:00.864437Z","iopub.status.idle":"2024-01-15T18:58:02.135851Z","shell.execute_reply.started":"2024-01-15T18:58:00.864406Z","shell.execute_reply":"2024-01-15T18:58:02.135102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_start_labels, predicted_end_labels = short_model_prediction_probs","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.137392Z","iopub.execute_input":"2024-01-15T18:58:02.137681Z","iopub.status.idle":"2024-01-15T18:58:02.141965Z","shell.execute_reply.started":"2024-01-15T18:58:02.137655Z","shell.execute_reply":"2024-01-15T18:58:02.141084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_start_labels = tf.squeeze(tf.round(predicted_start_labels)) \npredicted_end_labels = tf.squeeze(tf.round(predicted_end_labels)) \n\npredicted_start_labels, predicted_end_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.143201Z","iopub.execute_input":"2024-01-15T18:58:02.143636Z","iopub.status.idle":"2024-01-15T18:58:02.159599Z","shell.execute_reply.started":"2024-01-15T18:58:02.143601Z","shell.execute_reply":"2024-01-15T18:58:02.158676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_counts = np.unique(predicted_start_labels[120], return_counts=True)\nunique_counts","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.161718Z","iopub.execute_input":"2024-01-15T18:58:02.161978Z","iopub.status.idle":"2024-01-15T18:58:02.168854Z","shell.execute_reply.started":"2024-01-15T18:58:02.161954Z","shell.execute_reply":"2024-01-15T18:58:02.168033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_labels(start_labels, end_labels):\n    # Decode start labels\n    decoded_start_labels = np.argmax(start_labels, axis=1)\n    \n    # Decode end labels\n    decoded_end_labels = np.argmax(end_labels, axis=1)\n\n    return decoded_start_labels, decoded_end_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.169843Z","iopub.execute_input":"2024-01-15T18:58:02.170109Z","iopub.status.idle":"2024-01-15T18:58:02.176910Z","shell.execute_reply.started":"2024-01-15T18:58:02.170063Z","shell.execute_reply":"2024-01-15T18:58:02.176117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded_start_labels, decoded_end_labels = decode_labels(predicted_start_labels, predicted_end_labels)\ndecoded_start_labels, decoded_end_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.177931Z","iopub.execute_input":"2024-01-15T18:58:02.178282Z","iopub.status.idle":"2024-01-15T18:58:02.190568Z","shell.execute_reply.started":"2024-01-15T18:58:02.178257Z","shell.execute_reply":"2024-01-15T18:58:02.189620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def append_decoded_labels(dataframe, decoded_start_labels, decoded_end_labels):\n    dataframe['short_start_token'] = decoded_start_labels\n    dataframe['short_end_token'] = decoded_end_labels\n    mask1 = dataframe['short_start_token'] != 0\n    mask2 = dataframe['short_end_token'] != 0\n    dataframe.loc[mask1,'short_start_token'] = decoded_start_labels + dataframe['long_start_token']\n    dataframe.loc[mask2,'short_end_token'] = decoded_end_labels + dataframe['long_start_token']\n\n    return dataframe","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.191608Z","iopub.execute_input":"2024-01-15T18:58:02.191921Z","iopub.status.idle":"2024-01-15T18:58:02.198746Z","shell.execute_reply.started":"2024-01-15T18:58:02.191894Z","shell.execute_reply":"2024-01-15T18:58:02.198035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # evaluate model \n    \n# flat_val_start_labels = np.array(val_short_data['short_start'])\n# flat_val_end_labels = np.array(val_short_data['short_end'])\n# flat_decoded_start_labels = decoded_start_labels.flatten()\n# flat_decoded_end_labels = decoded_end_labels.flatten()\n\n# precision, recall, f1, _ = precision_recall_fscore_support(\n#     np.concatenate([flat_val_start_labels, flat_val_end_labels]),\n#     np.concatenate([flat_decoded_start_labels, flat_decoded_end_labels]),\n#     average='weighted',\n#     zero_division=0.0\n# )\n    \n# print(\"Precision:\", precision)\n# print(\"Recall:\", recall)\n# print(\"F1:\", f1)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:11.086496Z","iopub.execute_input":"2024-01-15T18:58:11.087245Z","iopub.status.idle":"2024-01-15T18:58:11.091427Z","shell.execute_reply.started":"2024-01-15T18:58:11.087210Z","shell.execute_reply":"2024-01-15T18:58:11.090485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r short-model.zip /kaggle/working/model_short","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:02.542880Z","iopub.status.idle":"2024-01-15T18:58:02.543239Z","shell.execute_reply.started":"2024-01-15T18:58:02.543043Z","shell.execute_reply":"2024-01-15T18:58:02.543058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_data = append_decoded_labels(new_test_data, decoded_start_labels, decoded_end_labels)\nmask = new_test_data['short_start_token'] != 0\nprint(new_test_data.loc[mask])\ncount_short_answers = np.unique(new_test_data['short_start_token'], return_counts=True)\ncount_short_answers","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:13.657670Z","iopub.execute_input":"2024-01-15T18:58:13.658042Z","iopub.status.idle":"2024-01-15T18:58:13.675537Z","shell.execute_reply.started":"2024-01-15T18:58:13.658011Z","shell.execute_reply":"2024-01-15T18:58:13.674470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_data","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:15.946434Z","iopub.execute_input":"2024-01-15T18:58:15.947189Z","iopub.status.idle":"2024-01-15T18:58:15.962769Z","shell.execute_reply.started":"2024-01-15T18:58:15.947153Z","shell.execute_reply":"2024-01-15T18:58:15.961790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_csv(sample_csv_path, dataframe):\n    dtypes = {'example_id': str, 'PredictionString': str}\n    existing_df = pd.read_csv(sample_csv_path, dtype=dtypes, quoting=csv.QUOTE_NONNUMERIC)\n\n    # Iterate through the existing CSV\n    for index, row in existing_df.iterrows():\n        example_id_with_suffix = row['example_id']\n\n        # Extract the ID from the example_id_with_suffix\n        example_id, suffix = example_id_with_suffix.split('_')\n        corresponding_row = dataframe[dataframe['example_id'] == example_id]\n\n        # Complete the values for \"id_long\" and \"id_short\"\n        if not corresponding_row.empty:\n            if suffix == 'long':\n                long_start = corresponding_row['long_start_token'].values[0]\n                long_end = corresponding_row['long_end_token'].values[0]\n                # Construct the new PredictionString value for long\n                new_long_prediction_string = f\"{long_start}:{long_end}\" if pd.notna(long_start) and pd.notna(long_end) else ''\n                # Update the 'PredictionString' column for long\n                existing_df.at[index, 'PredictionString'] = new_long_prediction_string\n\n            elif suffix == 'short':\n                short_start = corresponding_row['short_start_token'].values[0]\n                short_end = corresponding_row['short_end_token'].values[0]\n                # Construct the new PredictionString value for short\n                new_short_prediction_string = f\"{short_start}:{short_end}\" if pd.notna(short_start) and pd.notna(short_end) and short_end != 0 else ''\n                existing_df.at[index, 'PredictionString'] = new_short_prediction_string\n                \n    # Save the updated DataFrame to a new CSV file\n    existing_df.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:23.069140Z","iopub.execute_input":"2024-01-15T18:58:23.069990Z","iopub.status.idle":"2024-01-15T18:58:23.079182Z","shell.execute_reply.started":"2024-01-15T18:58:23.069956Z","shell.execute_reply":"2024-01-15T18:58:23.078297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_csv(\"../input/tensorflow2-question-answering/sample_submission.csv\", new_test_data)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T18:58:28.960772Z","iopub.execute_input":"2024-01-15T18:58:28.961573Z","iopub.status.idle":"2024-01-15T18:58:29.413829Z","shell.execute_reply.started":"2024-01-15T18:58:28.961541Z","shell.execute_reply":"2024-01-15T18:58:29.412964Z"},"trusted":true},"execution_count":null,"outputs":[]}]}