{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wc -l /kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def select_random_long_answer(long_answer_candidates, seed=None):\n    assert isinstance(long_answer_candidates, list)\n    for long_answer in long_answer_candidates:\n        assert isinstance(long_answer, dict)\n        assert 'start_token' in long_answer\n        assert 'end_token' in long_answer\n    if seed is not None:\n        np.random.seed(seed)\n    index = np.random.randint(len(long_answer_candidates))\n    return long_answer_candidates[index]\n\ndef select_random_short_answer(long_answer, seed=None):\n    assert isinstance(long_answer, dict)\n    assert 'start_token' in long_answer\n    assert 'end_token' in long_answer\n    if seed is not None:\n        np.random.seed(seed)\n    start_token = np.random.randint(low=long_answer['start_token'], high=long_answer['end_token'])\n    end_token = np.random.randint(low=start_token, high=min(start_token + 10, long_answer['end_token'] + 1))\n    return {'start_token': start_token, 'end_token': end_token}\n\ndef get_prediction_string(answer):\n    assert isinstance(answer, dict)\n    assert 'start_token' in answer\n    assert 'end_token' in answer\n    return '{}:{}'.format(answer['start_token'], answer['end_token'])\n\ndef get_answer_text(answer, document_text_tokens):\n    assert isinstance(answer, dict)\n    assert 'start_token' in answer\n    assert 'end_token' in answer\n    answer_tokens = document_text_tokens[answer['start_token']:answer['end_token'] + 1]\n    answer_text = ' '.join(answer_tokens)\n    return answer_text\n\ndef predict_on_chunk_dataframe(df, seed=None):\n    assert isinstance(df, pd.DataFrame)\n    if seed is not None:\n        np.random.seed(seed)\n    \n    df['document_text_tokens'] = df['document_text'].apply(lambda s: s.split())\n    df['long_answer'] = df['long_answer_candidates'].apply(lambda v: select_random_long_answer(v))\n    df['short_answer'] = df['long_answer'].apply(lambda d: select_random_short_answer(d))\n    df['long_answer_text'] = df.apply(lambda row: get_answer_text(row['long_answer'], row['document_text_tokens']), axis=1)\n    df['short_answer_text'] = df.apply(lambda row: get_answer_text(row['short_answer'], row['document_text_tokens']), axis=1)\n    df['long_answer_prediction_string'] = df['long_answer'].apply(get_prediction_string)\n    df['short_answer_prediction_string'] = df['short_answer'].apply(get_prediction_string)\n\n    # Order the columns\n    assert len(set(df.columns)) == len(df.columns)\n    ordered_columns = ['question_text', 'long_answer_text', 'short_answer_text', 'document_text']\n    rest_columns = list(set(df.columns).difference(set(ordered_columns)))\n    df = df[ordered_columns + rest_columns]\n    return df\n\ndef generate_submission(df, seed=None):\n    assert isinstance(df, pd.DataFrame)\n    if seed is not None:\n        np.random.seed(seed)\n    \n    df = predict_on_chunk_dataframe(df)\n    long_predictions = (df[['example_id', 'long_answer_prediction_string']].copy()\n                        .rename({'long_answer_prediction_string': 'PredictionString'}, axis=1))\n    long_predictions['example_id'] = long_predictions['example_id'].apply(lambda s: '{}_long'.format(s))\n    short_predictions = (df[['example_id', 'short_answer_prediction_string']].copy()\n                         .rename({'short_answer_prediction_string': 'PredictionString'}, axis=1))\n    short_predictions['example_id'] = short_predictions['example_id'].apply(lambda s: '{}_short'.format(s))\n    \n    submission_df = (pd.concat([long_predictions, short_predictions], axis=0, ignore_index=True)\n                     .sort_values(by=['example_id'], ascending=True)\n                     .reset_index(drop=True))\n    return submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(42)\ntest_df_reader = pd.read_json('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', lines=True,\n                              chunksize=100)\nsubmission_chunks = [generate_submission(chunk_test_df) for chunk_test_df in test_df_reader]\nsubmission_df = (pd.concat(submission_chunks)\n                 .sort_values(by=['example_id'], ascending=True)\n                 .reset_index(drop=True))\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lh submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head -10 submission.csv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# test_df = pd.read_json('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', lines=True)\n# pd.set_option('display.max_colwidth', 50)\n# test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def select_random_long_answer(long_answer_candidates, seed=None):\n#     assert isinstance(long_answer_candidates, list)\n#     for long_answer in long_answer_candidates:\n#         assert isinstance(long_answer, dict)\n#         assert 'start_token' in long_answer\n#         assert 'end_token' in long_answer\n#     if seed is not None:\n#         np.random.seed(seed)\n#     index = np.random.randint(len(long_answer_candidates))\n#     return long_answer_candidates[index]\n\n# def select_random_short_answer(long_answer, seed=None):\n#     assert isinstance(long_answer, dict)\n#     assert 'start_token' in long_answer\n#     assert 'end_token' in long_answer\n#     if seed is not None:\n#         np.random.seed(seed)\n#     start_token = np.random.randint(low=long_answer['start_token'], high=long_answer['end_token'])\n#     end_token = np.random.randint(low=start_token, high=min(start_token + 10, long_answer['end_token'] + 1))\n#     return {'start_token': start_token, 'end_token': end_token}\n\n# def get_prediction_string(answer):\n#     assert isinstance(answer, dict)\n#     assert 'start_token' in answer\n#     assert 'end_token' in answer\n#     return '{}:{}'.format(answer['start_token'], answer['end_token'])\n\n# def get_answer_text(answer, document_text_tokens):\n#     assert isinstance(answer, dict)\n#     assert 'start_token' in answer\n#     assert 'end_token' in answer\n#     answer_tokens = document_text_tokens[answer['start_token']:answer['end_token'] + 1]\n#     answer_text = ' '.join(answer_tokens)\n#     return answer_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.random.seed(42)\n# test_df['document_text_tokens'] = test_df['document_text'].apply(lambda s: s.split())\n# test_df['long_answer'] = test_df['long_answer_candidates'].apply(lambda v: select_random_long_answer(v))\n# test_df['short_answer'] = test_df['long_answer'].apply(lambda d: select_random_short_answer(d))\n# test_df['long_answer_text'] = test_df.apply(lambda row: get_answer_text(row['long_answer'], row['document_text_tokens']), axis=1)\n# test_df['short_answer_text'] = test_df.apply(lambda row: get_answer_text(row['short_answer'], row['document_text_tokens']), axis=1)\n# test_df['long_answer_prediction_string'] = test_df['long_answer'].apply(get_prediction_string)\n# test_df['short_answer_prediction_string'] = test_df['short_answer'].apply(get_prediction_string)\n\n# # Order the columns\n# assert len(set(test_df.columns)) == len(test_df.columns)\n# ordered_columns = ['question_text', 'long_answer_text', 'short_answer_text', 'document_text']\n# rest_columns = list(set(test_df.columns).difference(set(ordered_columns)))\n# test_df = test_df[ordered_columns + rest_columns]\n# pd.set_option('display.max_colwidth', 50)\n# test_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_max_colwidth = pd.get_option('display.max_colwidth')\n# pd.set_option('display.max_colwidth', 0)\n# test_df[['question_text', 'long_answer_text', 'short_answer_text']]\n# # pd.set_option('display.max_colwidth', display_max_colwidth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Long predictions\n# long_predictions = test_df[['example_id', 'long_answer_prediction_string']].copy().rename({'long_answer_prediction_string': 'PredictionString'}, axis=1)\n# long_predictions['example_id'] = long_predictions['example_id'].apply(lambda s: '{}_long'.format(s))\n# long_predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Short predictions\n# short_predictions = test_df[['example_id', 'short_answer_prediction_string']].copy().rename({'short_answer_prediction_string': 'PredictionString'}, axis=1)\n# short_predictions['example_id'] = short_predictions['example_id'].apply(lambda s: '{}_short'.format(s))\n# short_predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Combine long and short predictions\n# submission_df = (pd.concat([long_predictions, short_predictions], axis=0, ignore_index=True)\n#                  .sort_values(by=['example_id'], ascending=True).reset_index(drop=True))\n# submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ls -lh submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !head -10 submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !head -10 /kaggle/input/tensorflow2-question-answering/sample_submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}