{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%reset -sf","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.155502,"end_time":"2021-07-27T03:06:31.292025","exception":false,"start_time":"2021-07-27T03:06:31.136523","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:55.856242Z","iopub.execute_input":"2021-08-11T15:42:55.856635Z","iopub.status.idle":"2021-08-11T15:42:55.954427Z","shell.execute_reply.started":"2021-08-11T15:42:55.856605Z","shell.execute_reply":"2021-08-11T15:42:55.953354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook covers\n- Dataset Preparation (train-test split)\n- The 7 methods mentioned in our report, and the variants we experimented\n- Algorithm for evaluation metrics\n- Evaluation procedure with the test set\n- Preparation of the dataset for hand evaluation\n- Evaluting NDCG with the hand-labelled dataset\n- Query and update indexes with unseen questions\n\nThe following process is done on another notebook\n- Spellcheck and SpaCy tokenisation for the training set \n- SentenceTransformer computation of vectors for the training set \n- Downloading of the SpaCy and GenSim models\n- Generating the training data for the supervised model\n\n","metadata":{"papermill":{"duration":0.069631,"end_time":"2021-07-27T03:06:31.43175","exception":false,"start_time":"2021-07-27T03:06:31.362119","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## How to run the GUI\n\nTo run the GUI to query and index unseen questions, please\n- Run the entire notebook (takes around 5 minutes to install and index)\n- Go to the last section on GUI to see the sample to query and index\n\n(If you want to see evaulation results on the full test set instead, see Version 38)","metadata":{}},{"cell_type":"code","source":"# notebook hyperparameters\nTEST_SET_SIZE = 1000\nRANKED_LIST_SIZE = 100\nRANDOM_STATE = 42\nEVALUATING = False  # setting to False will only evaulate 10 queries","metadata":{"papermill":{"duration":0.076966,"end_time":"2021-07-27T03:06:31.579561","exception":false,"start_time":"2021-07-27T03:06:31.502595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:55.955968Z","iopub.execute_input":"2021-08-11T15:42:55.956264Z","iopub.status.idle":"2021-08-11T15:42:55.972752Z","shell.execute_reply.started":"2021-08-11T15:42:55.956236Z","shell.execute_reply":"2021-08-11T15:42:55.971546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, collections, random, itertools, functools, time, json\n\nfrom collections import defaultdict, Counter\nfrom math import log\nfrom copy import deepcopy\n\nimport tqdm.notebook as tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nrandom.seed(RANDOM_STATE)\nnp.random.seed(RANDOM_STATE)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.07936,"end_time":"2021-07-27T03:06:31.728421","exception":false,"start_time":"2021-07-27T03:06:31.649061","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:55.974817Z","iopub.execute_input":"2021-08-11T15:42:55.975134Z","iopub.status.idle":"2021-08-11T15:42:55.993395Z","shell.execute_reply.started":"2021-08-11T15:42:55.975106Z","shell.execute_reply":"2021-08-11T15:42:55.992425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\ndf = pd.read_csv(\"/kaggle/input/quora-question-pairs/train.csv.zip\")\ndf[\"question1\"] = df[\"question1\"].astype(str)  # resolve nan\ndf[\"question2\"] = df[\"question2\"].astype(str)\ndf[\"qid1\"] -= 1  #  start index from zero\ndf[\"qid2\"] -= 1","metadata":{"papermill":{"duration":2.046959,"end_time":"2021-07-27T03:06:33.8437","exception":false,"start_time":"2021-07-27T03:06:31.796741","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:55.994814Z","iopub.execute_input":"2021-08-11T15:42:55.995185Z","iopub.status.idle":"2021-08-11T15:42:58.891039Z","shell.execute_reply.started":"2021-08-11T15:42:55.995154Z","shell.execute_reply":"2021-08-11T15:42:58.890041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(10)","metadata":{"papermill":{"duration":0.114277,"end_time":"2021-07-27T03:06:34.02675","exception":false,"start_time":"2021-07-27T03:06:33.912473","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:58.892221Z","iopub.execute_input":"2021-08-11T15:42:58.892509Z","iopub.status.idle":"2021-08-11T15:42:58.926610Z","shell.execute_reply.started":"2021-08-11T15:42:58.892482Z","shell.execute_reply":"2021-08-11T15:42:58.925875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Dataset","metadata":{"papermill":{"duration":0.105896,"end_time":"2021-07-27T03:06:34.238959","exception":false,"start_time":"2021-07-27T03:06:34.133063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# all questions are identified with its qid\nqid_to_question = {}\nfor qid1, qid2, question1, question2 in zip(df[\"qid1\"], df[\"qid2\"], df[\"question1\"], df[\"question2\"]):\n    qid_to_question[qid1] = question1\n    qid_to_question[qid2] = question2","metadata":{"papermill":{"duration":0.518009,"end_time":"2021-07-27T03:06:34.864355","exception":false,"start_time":"2021-07-27T03:06:34.346346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:58.927591Z","iopub.execute_input":"2021-08-11T15:42:58.927992Z","iopub.status.idle":"2021-08-11T15:42:59.519631Z","shell.execute_reply.started":"2021-08-11T15:42:58.927962Z","shell.execute_reply":"2021-08-11T15:42:59.518584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract 1000 questions for testing\ntest_query_qids = set()\n\ndf_duplicate = df[df[\"is_duplicate\"] == 1].sample(frac=1, random_state=RANDOM_STATE)\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n    if is_duplicate and qid1 not in test_query_qids and len(test_query_qids) < TEST_SET_SIZE:\n        test_query_qids.add(qid2)\n    if qid1 in test_query_qids and qid2 in test_query_qids:\n        # to guarantee that there is a duplicate question in the training set\n        test_query_qids.remove(qid1)\n        test_query_qids.remove(qid2)\nassert len(test_query_qids) == TEST_SET_SIZE  # if fail, change random_state\n\ntest_query_qids_list = sorted(test_query_qids)\ntrain_query_qids_list = sorted(set(qid_to_question.keys()) - test_query_qids)\nassert test_query_qids_list[:3] == [331, 489, 501]   # to check random state fixed","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.305019,"end_time":"2021-07-27T03:06:35.281377","exception":false,"start_time":"2021-07-27T03:06:34.976358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:59.520820Z","iopub.execute_input":"2021-08-11T15:42:59.521236Z","iopub.status.idle":"2021-08-11T15:42:59.794773Z","shell.execute_reply.started":"2021-08-11T15:42:59.521204Z","shell.execute_reply":"2021-08-11T15:42:59.794014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # uncomment this to test only limited queries\nif not EVALUATING:\n    test_query_qids_list = test_query_qids_list[:10]\n    TEST_SET_SIZE = 10","metadata":{"papermill":{"duration":0.07612,"end_time":"2021-07-27T03:06:35.42967","exception":false,"start_time":"2021-07-27T03:06:35.35355","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:59.796883Z","iopub.execute_input":"2021-08-11T15:42:59.797337Z","iopub.status.idle":"2021-08-11T15:42:59.800610Z","shell.execute_reply.started":"2021-08-11T15:42:59.797292Z","shell.execute_reply":"2021-08-11T15:42:59.799896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of training set\n\nqid_to_duplicate_qids = defaultdict(set)\nqid_to_nonduplicate_qids = defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    if not (qid1 in test_query_qids or qid2 in test_query_qids):\n        if is_duplicate:\n            qid_to_duplicate_qids[qid1].add(qid2)\n            qid_to_duplicate_qids[qid2].add(qid1)\n        else:\n            qid_to_nonduplicate_qids[qid1].add(qid2)\n            qid_to_nonduplicate_qids[qid2].add(qid1)","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.225533,"end_time":"2021-07-27T03:06:36.726745","exception":false,"start_time":"2021-07-27T03:06:35.501212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:42:59.802705Z","iopub.execute_input":"2021-08-11T15:42:59.803305Z","iopub.status.idle":"2021-08-11T15:43:01.466294Z","shell.execute_reply.started":"2021-08-11T15:42:59.803249Z","shell.execute_reply":"2021-08-11T15:43:01.465521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# complete graph of duplicate relationships\n\nqid_to_duplicate_qids_complete = defaultdict(set)\nqid_to_qid_group_leader = {}\nqid_group_leader_to_duplicate_qid_group = defaultdict(set)\n\nvisited_qids = set()\nfor train_qid in train_query_qids_list:\n    if train_qid in visited_qids:\n        continue\n    current_qids_group = set([train_qid])\n    qid_to_qid_group_leader[train_qid] = train_qid\n    stack = [train_qid]\n    \n    while stack:\n        cur_qid = stack.pop()\n        for nex_qid in qid_to_duplicate_qids[cur_qid]:\n            if nex_qid in current_qids_group:\n                continue\n            qid_to_qid_group_leader[nex_qid] = train_qid\n            stack.append(nex_qid)\n            current_qids_group.add(nex_qid)\n\n    # complete the graph\n    for qid1, qid2 in itertools.combinations(current_qids_group, r=2):\n        qid_to_duplicate_qids_complete[qid1].add(qid2)\n        qid_to_duplicate_qids_complete[qid2].add(qid1)\n    qid_group_leader_to_duplicate_qid_group[train_qid] = current_qids_group\n    visited_qids.update(current_qids_group)","metadata":{"_kg_hide-input":true,"papermill":{"duration":2.449053,"end_time":"2021-07-27T03:06:39.247603","exception":false,"start_time":"2021-07-27T03:06:36.79855","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:01.467299Z","iopub.execute_input":"2021-08-11T15:43:01.467730Z","iopub.status.idle":"2021-08-11T15:43:05.027203Z","shell.execute_reply.started":"2021-08-11T15:43:01.467699Z","shell.execute_reply":"2021-08-11T15:43:05.026163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of the test set\n\ntest_qid_to_duplicate_qids = defaultdict(set)\ntest_qid_to_duplicate_qids_complete = defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n    if qid2 in test_query_qids:\n        qid1, qid2 = qid2, qid1\n    if qid1 in test_query_qids:\n        if qid2 in test_query_qids:\n            continue\n        test_qid_to_duplicate_qids[qid1].add(qid2)\n        test_qid_to_duplicate_qids_complete[qid1].add(qid2)\n        for train_qid in qid_group_leader_to_duplicate_qid_group[qid_to_qid_group_leader[qid2]]:\n            test_qid_to_duplicate_qids_complete[qid1].add(train_qid)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.161612,"end_time":"2021-07-27T03:06:39.480808","exception":false,"start_time":"2021-07-27T03:06:39.319196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.028453Z","iopub.execute_input":"2021-08-11T15:43:05.028760Z","iopub.status.idle":"2021-08-11T15:43:05.190174Z","shell.execute_reply.started":"2021-08-11T15:43:05.028730Z","shell.execute_reply":"2021-08-11T15:43:05.189003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count inconsistencies in dataset\n\ncnt = 0\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    if not is_duplicate and qid1 not in test_query_qids and qid2 not in test_query_qids:\n        if qid_to_qid_group_leader[qid1] == qid_to_qid_group_leader[qid2]:\n            cnt += 1\nprint(\"Number of inconsistencies: \", cnt)  # slightly smaller than 96 because some edges are associated with the test set","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.549375,"end_time":"2021-07-27T03:06:40.100121","exception":false,"start_time":"2021-07-27T03:06:39.550746","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.191581Z","iopub.execute_input":"2021-08-11T15:43:05.192013Z","iopub.status.idle":"2021-08-11T15:43:05.579542Z","shell.execute_reply.started":"2021-08-11T15:43:05.191969Z","shell.execute_reply":"2021-08-11T15:43:05.578571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_mask = (df[\"qid1\"].isin(test_query_qids)) | (df[\"qid2\"].isin(test_query_qids))\ntrain_df = df[~test_mask].copy()\ntest_df = df[test_mask].copy()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.138302,"end_time":"2021-07-27T03:06:40.308504","exception":false,"start_time":"2021-07-27T03:06:40.170202","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.580876Z","iopub.execute_input":"2021-08-11T15:43:05.581177Z","iopub.status.idle":"2021-08-11T15:43:05.669378Z","shell.execute_reply.started":"2021-08-11T15:43:05.581148Z","shell.execute_reply":"2021-08-11T15:43:05.668561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean up\ndel qid_to_qid_group_leader, qid_group_leader_to_duplicate_qid_group\ndel cnt\ndel test_query_qids   # not sorted, use test_query_qids_list\ndel df                # all data you can train on is in train_df\n\n# enable use of complete graphs\ntest_qid_to_duplicate_qids = test_qid_to_duplicate_qids_complete\nqid_to_duplicate_qids = qid_to_duplicate_qids_complete","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.200144,"end_time":"2021-07-27T03:06:40.580253","exception":false,"start_time":"2021-07-27T03:06:40.380109","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.670587Z","iopub.execute_input":"2021-08-11T15:43:05.671116Z","iopub.status.idle":"2021-08-11T15:43:05.785447Z","shell.execute_reply.started":"2021-08-11T15:43:05.671083Z","shell.execute_reply":"2021-08-11T15:43:05.784289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Metrics","metadata":{"execution":{"iopub.execute_input":"2021-06-08T11:21:27.301993Z","iopub.status.busy":"2021-06-08T11:21:27.301617Z","iopub.status.idle":"2021-06-08T11:21:27.305697Z","shell.execute_reply":"2021-06-08T11:21:27.304715Z","shell.execute_reply.started":"2021-06-08T11:21:27.301962Z"},"papermill":{"duration":0.070542,"end_time":"2021-07-27T03:06:40.721489","exception":false,"start_time":"2021-07-27T03:06:40.650947","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def method_random_guess(test_qid):\n    # returns ranklist and scores of each size RANKED_LIST_SIZE\n    return random.choices(train_query_qids_list, k=RANKED_LIST_SIZE), [0]*RANKED_LIST_SIZE\n\n# 1000 x 100 (the ranked list of similar qn for each of the 1000 test qns)\nranklists_method_random_guess = [method_random_guess(test_qid)[0] for test_qid in test_query_qids_list]","metadata":{"papermill":{"duration":0.115762,"end_time":"2021-07-27T03:06:40.906597","exception":false,"start_time":"2021-07-27T03:06:40.790835","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.786790Z","iopub.execute_input":"2021-08-11T15:43:05.787149Z","iopub.status.idle":"2021-08-11T15:43:05.798720Z","shell.execute_reply.started":"2021-08-11T15:43:05.787117Z","shell.execute_reply":"2021-08-11T15:43:05.797754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_sample_query_results(test_qid, method_ranklist, method_scores=[0]*RANKED_LIST_SIZE, num_to_show=10):\n    # not a metric, just print a few examples and its scores\n    print(\"Query: {}\".format(qid_to_question[test_qid]))\n    for rank, (score, result_qid) in enumerate(zip(method_scores, method_ranklist[:num_to_show]), start=1):\n        relevance = \"Registered\" if result_qid in test_qid_to_duplicate_qids[test_qid] else \"Unregistered\"\n        print(\"Rank {} - Score {:.4f} - {}:  \\t{}\".format(rank, score, relevance, qid_to_question[result_qid]))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.078743,"end_time":"2021-07-27T03:06:41.054473","exception":false,"start_time":"2021-07-27T03:06:40.97573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.800055Z","iopub.execute_input":"2021-08-11T15:43:05.800367Z","iopub.status.idle":"2021-08-11T15:43:05.811609Z","shell.execute_reply.started":"2021-08-11T15:43:05.800339Z","shell.execute_reply":"2021-08-11T15:43:05.810468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_random_guess(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.080391,"end_time":"2021-07-27T03:06:41.205365","exception":false,"start_time":"2021-07-27T03:06:41.124974","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.813196Z","iopub.execute_input":"2021-08-11T15:43:05.813547Z","iopub.status.idle":"2021-08-11T15:43:05.828953Z","shell.execute_reply.started":"2021-08-11T15:43:05.813518Z","shell.execute_reply":"2021-08-11T15:43:05.827910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_first_relevant_rank(method_ranklists, considered=1, eps=10**-6, debug=True, **kwargs):\n    # calculation of the statistics of the rank of the first c=considered duplicates\n    # if the duplicate does not appear in the ranklist, it has a default rank of RANKED_LIST_SIZE\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n    reciprocal_ranks = []\n    ranks = []\n    for test_qid, ranklist in zip(test_query_qids_list, method_ranklists):\n        test_qid_to_rank = {result_qid:rank for rank, result_qid in enumerate(ranklist, start=1)}\n        rank = []  # may be shorter than `considered` because of lack of duplicates\n        for expected_qid in test_qid_to_duplicate_qids[test_qid]:\n            if expected_qid in test_qid_to_rank:\n                rank.append(test_qid_to_rank[expected_qid])\n            else:\n                rank.append(RANKED_LIST_SIZE+1)\n        rank.sort()\n        ranks.extend(rank[:considered])\n        if rank[0] > RANKED_LIST_SIZE:\n            reciprocal_ranks.append(0)\n        else:\n            reciprocal_ranks.append(1/rank[0])\n    \n    plt.figure(figsize=(14,4))\n    plt.title(\"Highest rank of duplicate question\")\n    plt.hist(ranks, bins=np.arange(RANKED_LIST_SIZE+2))\n    plt.xlabel(\"Rank\")\n    plt.ylabel(\"Frequency\")\n    plt.show()\n    \n    mrr = sum(reciprocal_ranks)/len(reciprocal_ranks)\n    har = 1/(mrr+eps)\n    print(f\"Mean Reciprocal Rank (MRR) is {mrr:.2f}\")\n    print(f\"Harmonic Average Rank (HAR) is {har:.2f}\")    \n    \n    p50 = np.median(ranks)\n    proportion_out_of_result = ranks.count(RANKED_LIST_SIZE+1)/len(ranks)\n    if debug:\n        print(\"Median rank: {:.2f}\".format(p50))\n        print(\"Proportion out of result: {:.3f}\".format(proportion_out_of_result))\n    \n    return mrr, har, p50, proportion_out_of_result","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.082823,"end_time":"2021-07-27T03:06:41.360563","exception":false,"start_time":"2021-07-27T03:06:41.27774","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.830689Z","iopub.execute_input":"2021-08-11T15:43:05.831415Z","iopub.status.idle":"2021-08-11T15:43:05.846417Z","shell.execute_reply.started":"2021-08-11T15:43:05.831350Z","shell.execute_reply":"2021-08-11T15:43:05.845189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_first_relevant_rank(ranklists_method_random_guess)","metadata":{"papermill":{"duration":0.395156,"end_time":"2021-07-27T03:06:41.825405","exception":false,"start_time":"2021-07-27T03:06:41.430249","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:05.847787Z","iopub.execute_input":"2021-08-11T15:43:05.848670Z","iopub.status.idle":"2021-08-11T15:43:06.231204Z","shell.execute_reply.started":"2021-08-11T15:43:05.848525Z","shell.execute_reply":"2021-08-11T15:43:06.229760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_auc(method_ranklists, k=10, weights=None, debug=True, **kwargs):\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n    \n    counts = np.array([0.]*k)\n    ## Identify duplicates among top K ranks for each test\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n        topk = ranklist[:k]\n        is_duplicate = np.array([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in topk])\n        counts += is_duplicate \n    \n    ## Calculate AUC\n    if weights:\n        counts *= np.array(weights)/sum(weights)\n    else:\n        counts /= k\n    \n    auc = sum(counts)/(TEST_SET_SIZE)\n    \n    if debug:\n        print(f\"{auc:.2%} of top {k} results are duplicates\")\n\n    return auc # between [0,1], 1 is perfect","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.080597,"end_time":"2021-07-27T03:06:41.977728","exception":false,"start_time":"2021-07-27T03:06:41.897131","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.232607Z","iopub.execute_input":"2021-08-11T15:43:06.233031Z","iopub.status.idle":"2021-08-11T15:43:06.241638Z","shell.execute_reply.started":"2021-08-11T15:43:06.232988Z","shell.execute_reply":"2021-08-11T15:43:06.240563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_auc(ranklists_method_random_guess)\n_ = evaluation_with_auc(ranklists_method_random_guess, weights = [10,9,8,7,6,5,4,3,2,1])","metadata":{"papermill":{"duration":0.161031,"end_time":"2021-07-27T03:06:42.247939","exception":false,"start_time":"2021-07-27T03:06:42.086908","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.243093Z","iopub.execute_input":"2021-08-11T15:43:06.243529Z","iopub.status.idle":"2021-08-11T15:43:06.261995Z","shell.execute_reply.started":"2021-08-11T15:43:06.243487Z","shell.execute_reply":"2021-08-11T15:43:06.260803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def single_r_precision(test_qid, ranklist):\n    # use this to check a single test query\n    num_duplicate = len(test_qid_to_duplicate_qids[test_qid]) # this dict needs to be updated when train:test set separation is updated\n    if num_duplicate == 0:\n        return 0, 0, 0\n    top_r = ranklist[:num_duplicate]\n    num_duplicates_in_top_r = sum([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in top_r])\n    r_precision = num_duplicates_in_top_r/num_duplicate\n    return num_duplicate, num_duplicates_in_top_r, r_precision\n\n\ndef evaluation_with_r_precision(method_ranklists, k=10, report_k=0, debug=True, **kwargs):\n    print(np.array(method_ranklists).shape)\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE) # method_ranklists size is (1000,100)\n    \n    total_num_duplicates = np.array([0 for i in range(TEST_SET_SIZE)])\n    r_precision = np.array([0 for i in range(TEST_SET_SIZE)])\n    \n    ## Iter over 1->1000 tests\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)): # iter over 1->1000 tests\n        total_num_duplicates[i], num_duplicates_in_top_r, r_precision[i] = single_r_precision(test_qid, ranklist)\n    \n    # note: if want do error analysis, intervene here to find test cases with low r precision\n    if report_k > 0:\n        k_lowest_r_precision_idx = np.argpartition(r_precision, k)[:k]\n        k_lowest_r_precision_test_qids = np.array(test_query_qids_list)[k_lowest_r_precision_idx]\n\n    ## Calculate metrics\n    avg_r_precision = r_precision.mean()\n    weighted_avg_r_precision = np.multiply(r_precision, total_num_duplicates).sum() / total_num_duplicates.sum()\n    \n    if debug:\n        print(f\"Average R-Precision = {avg_r_precision:.2%}\")\n        print(f\"Weighted Average R-Precision by proportion of duplicates = {weighted_avg_r_precision:.2%}\") \n        if avg_r_precision > weighted_avg_r_precision:\n            print(\"A higher average R-Precisions suggests that there are many test queries with high R-Precision but there are some test queries with high number of duplicates that model is not effective with.\")\n    \n    if not report_k: return avg_r_precision, weighted_avg_r_precision\n    else:\n        return avg_r_precision, weighted_avg_r_precision, k_lowest_r_precision_test_qids","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.081175,"end_time":"2021-07-27T03:06:42.396107","exception":false,"start_time":"2021-07-27T03:06:42.314932","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.263800Z","iopub.execute_input":"2021-08-11T15:43:06.264226Z","iopub.status.idle":"2021-08-11T15:43:06.277987Z","shell.execute_reply.started":"2021-08-11T15:43:06.264182Z","shell.execute_reply":"2021-08-11T15:43:06.277051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_r_precision(ranklists_method_random_guess, k=10)","metadata":{"papermill":{"duration":0.150502,"end_time":"2021-07-27T03:06:42.64824","exception":false,"start_time":"2021-07-27T03:06:42.497738","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.282763Z","iopub.execute_input":"2021-08-11T15:43:06.283085Z","iopub.status.idle":"2021-08-11T15:43:06.294679Z","shell.execute_reply.started":"2021-08-11T15:43:06.283057Z","shell.execute_reply":"2021-08-11T15:43:06.293729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_precision_recall_at_k(method_ranklists, k=10, exclude_precision=False, exclude_recall=False, debug=True, **kwargs):\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n    ## Evaluation returns the macro average P@K and R@Kfor test set\n    ## Interpretation P@K: what % of top k retrieved is relevant?\n    ## Interpretation R@K: what % of all duplicates for query is retrieved within top k?\n    \n    ## Iter thru each test\n    precisions_at_k = []\n    recalls_at_k = []\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n        ## 1. Set rank threshold K, ignore all docs after K\n        ## 2. Count num_relevant in top-K\n        ## 3. Count total_num_duplicates_for_query\n        ## 4. P@K = num_relevant/k\n        ## 5. R@K = num_relevant/total_num_duplicates_for_query\n        topk = ranklist[:k]\n        num_relevant = sum([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in topk])\n        \n        precision_at_k = num_relevant/k\n        precisions_at_k.append(precision_at_k)\n        \n        total_num_duplicates_for_query = len(test_qid_to_duplicate_qids[test_qid])\n        recall_at_k = num_relevant/total_num_duplicates_for_query\n        recalls_at_k.append(recall_at_k)\n    \n    mean_precision_at_k = sum(precisions_at_k)/len(precisions_at_k) # macro average\n    mean_recall_at_k = sum(recalls_at_k)/len(recalls_at_k) # macro average\n    print(f\"Macro Average Precision@k={k} is {mean_precision_at_k:.2%}\")\n    print(f\"Macro Average Recall@k={k} is {mean_recall_at_k:.2%}\")\n    return (mean_precision_at_k, mean_recall_at_k)\n\n_ = evaluation_with_precision_recall_at_k(ranklists_method_random_guess, k=10)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.101741,"end_time":"2021-07-27T03:06:42.82164","exception":false,"start_time":"2021-07-27T03:06:42.719899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.297065Z","iopub.execute_input":"2021-08-11T15:43:06.297362Z","iopub.status.idle":"2021-08-11T15:43:06.308865Z","shell.execute_reply.started":"2021-08-11T15:43:06.297333Z","shell.execute_reply":"2021-08-11T15:43:06.307820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_map(method_ranklists, debug=True, **kwargs):\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n    ## Interpretation: what is the average precision for all relevant docs across all queries?\n\n    ## Iter thru each test\n    average_precisions = []\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n        ## 1. Find the rank positions of each of the R relevant docs: K1, K2, ... KR and sort \n        ## 2. Compute P@K for each K1, K2, ... If K >=RANKED_LIST_SIZE, assume never retrieved\n        ## 3. AP = average of P@K for query\n        ## 4. MAP = macro average of AP across queries\n\n        ## 1. Find the rank positions of each of the R relevant docs: K1, K2, ... and sort \n        dup_qids_in_train_set = [dup_qid for dup_qid in test_qid_to_duplicate_qids[test_qid] if dup_qid in train_query_qids_list] # find all the dup_qid that can be found in the train set so you know total dup qn that could be found\n        total_num_dup_qid = len(dup_qids_in_train_set) # how many dup qn to expect\n\n        dup_ranks = []\n        for dup_qid in dup_qids_in_train_set:\n            if dup_qid not in ranklist: # not found\n                dup_ranks.append(RANKED_LIST_SIZE) # give \"out of range\" rank which would be checked later during calculation\n                continue\n            dup_ranks.append(list(ranklist).index(dup_qid)+1) # append the rank of the retrieved dup qn\n        \n        dup_ranks, dup_qids_in_train_set = (list(t) for t in zip(*sorted(zip(dup_ranks, dup_qids_in_train_set)))) # sort by rank\n        ## 2. Compute P@K for each K1, K2, ... If K >=RANKED_LIST_SIZE, assume never retrieved\n        precisions_at_k = []\n        for j, rank in enumerate(dup_ranks, start=1): # dup_ranks is sorted\n            if rank >= RANKED_LIST_SIZE: # handle \"unretrieved\" duplicates\n                precisions_at_k.append(0)\n            else: \n                precision_at_k = j / rank # = num_dup_so_far / rank_of_latest_dup_found\n                precisions_at_k.append(precision_at_k)\n        \n        ## 3. AP = average of P@K for query\n        average_precisions.append(sum(precisions_at_k)/len(precisions_at_k))\n    \n    ## Out of test query loop\n    ## 4. MAP = macro average of AP across queries\n    MAP = sum(average_precisions)/len(average_precisions)\n    print(f\"Mean Average Precision (MAP) is {MAP:.2%}\")\n    return MAP\n\n_ = evaluation_with_map(ranklists_method_random_guess)","metadata":{"_kg_hide-input":true,"papermill":{"duration":8.564513,"end_time":"2021-07-27T03:06:51.459009","exception":false,"start_time":"2021-07-27T03:06:42.894496","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.310780Z","iopub.execute_input":"2021-08-11T15:43:06.311291Z","iopub.status.idle":"2021-08-11T15:43:06.507314Z","shell.execute_reply.started":"2021-08-11T15:43:06.311247Z","shell.execute_reply":"2021-08-11T15:43:06.505823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_process(method, test_query_qids_list=test_query_qids_list, \n                       calculate_metrics=True, use_tqdm=True, **kwargs):\n    # executes the method and runs the evaluation functions \n    ranklists, scorelists = [], []\n    \n    iterator = tqdm.tqdm if use_tqdm else iter\n        \n    for test_qid in iterator(test_query_qids_list):\n        ranklist, scores = method(test_qid)\n        ranklists.append(ranklist)\n        scorelists.append(scores)\n    \n    if calculate_metrics:\n        evaluation_with_first_relevant_rank(ranklists, **kwargs)\n        # evaluation_with_auc(ranklists, **kwargs)\n        evaluation_with_r_precision(ranklists, **kwargs)\n\n        evaluation_with_precision_recall_at_k(ranklists, k=10, **kwargs)\n        evaluation_with_map(ranklists, **kwargs)\n\n    return ranklists, scorelists","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.079025,"end_time":"2021-07-27T03:06:51.646845","exception":false,"start_time":"2021-07-27T03:06:51.56782","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.508973Z","iopub.execute_input":"2021-08-11T15:43:06.509463Z","iopub.status.idle":"2021-08-11T15:43:06.516980Z","shell.execute_reply.started":"2021-08-11T15:43:06.509416Z","shell.execute_reply":"2021-08-11T15:43:06.515715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_random_guess = evaluation_process(method_random_guess)","metadata":{"papermill":{"duration":8.987885,"end_time":"2021-07-27T03:07:00.708941","exception":false,"start_time":"2021-07-27T03:06:51.721056","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:06.518500Z","iopub.execute_input":"2021-08-11T15:43:06.518789Z","iopub.status.idle":"2021-08-11T15:43:07.071484Z","shell.execute_reply.started":"2021-08-11T15:43:06.518761Z","shell.execute_reply":"2021-08-11T15:43:07.070169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing the Text","metadata":{"papermill":{"duration":0.07408,"end_time":"2021-07-27T03:07:00.857093","exception":false,"start_time":"2021-07-27T03:07:00.783013","status":"completed"},"tags":[]}},{"cell_type":"code","source":"## This entire cell is important to enable tokeniser pipeline \n## Use this to replace tokenise function if using Tokenise then Spellcheck (TSC) pipeline\n\n######### spacy basic tokenizer\nimport spacy\nprint(\"Spacy version: \", spacy.__version__)\nfrom spacy.tokenizer import Tokenizer  # https://spacy.io/api/tokenizer\n\n# !python3 -m spacy download en_core_web_sm\nprint(\"Loading Spacy en_core_web_sm loaded\")\nnlp = spacy.load(\"en_core_web_sm\")\ntokenizer = Tokenizer(nlp.vocab)\ntokenizer.add_special_case(\"[math]\", [{\"ORTH\": \"[math]\"}]) # see qid=7: '[math]23^{24}[/math]' becomes one token\n# add more special cases here if found","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.278553,"end_time":"2021-07-27T03:07:02.209456","exception":false,"start_time":"2021-07-27T03:07:00.930903","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:07.072976Z","iopub.execute_input":"2021-08-11T15:43:07.073405Z","iopub.status.idle":"2021-08-11T15:43:09.104738Z","shell.execute_reply.started":"2021-08-11T15:43:07.073343Z","shell.execute_reply":"2021-08-11T15:43:09.103879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def spacy_tokenise(text, lower=False, split_last_punc=True):\n    \"\"\"\n    returns a list of tokens given a question text\n    note: each punctuation is also considered a token\n    note: \"\\n\" is a token\n    note: \"'s\" is a token\n    note: '(Koh-i-Noor)' is a token\n    see tokenizer instantiation code for special cases or to add\n    lowercase text only after spell check\n    \"\"\"\n    if lower: text = text.lower()\n    tokens = tokenizer(text)\n    token_list = [token.text for token in tokens]\n\n    # further split tokens that end with certain punct e.g. \"me?\" => \"me\", \"?\"\n    if split_last_punc: \n        split_lists = [[token[:-1], token[-1]] if (token[-1] in [\"!\",\"?\",\",\",\":\"]) else [token] for token in token_list]\n        token_list = [token for sublist in split_lists for token in sublist]\n    return token_list\n\n######### symspell spellchecker\nprint(\"Loading symspell\")\n!pip install symspellpy\nfrom symspellpy.symspellpy import SymSpell, Verbosity  # https://github.com/mammothb/symspellpy\nimport pkg_resources\n\n# instantiate spellchecker\nsym = SymSpell(max_dictionary_edit_distance=2, prefix_length=7, count_threshold=1)\n# https://symspellpy.readthedocs.io/en/latest/api/symspellpy.html\ndictionary_path = pkg_resources.resource_filename(\"symspellpy\", \"frequency_dictionary_en_82_765.txt\")\nsym.load_dictionary(dictionary_path, 0, 1) # might take a short while\n\ndef spellcheck_single(word):\n    # returns top correct spelling or the same word if no correction found within max_edit_distance\n    if not word.isascii(): return word # do not spellcheck non ascii words e.g. シ\n\n    # obtain list of suggestions\n    suggestions = sym.lookup(word, Verbosity.CLOSEST, max_edit_distance=2,\n        include_unknown=True, # a mispelled word with no found corrections is returned as is\n        ignore_token=r\"[:,.!?\\\\-]\" # use if want to avoid correcting certain phrases\n        )\n    # get the term from the suggestItem object\n    suggested_words = [suggestion._term for suggestion in suggestions]\n    \n    # check if the input word is legit and return if so else return corrected word\n    word_lower = word.lower()\n    if word_lower in suggested_words: return word_lower # do not correct if input is a legit word\n    else: return suggested_words[0] # top suggestion\n\ndef spellcheck_compound(sent):\n    # spellchecks a sentence\n    suggestions = sym.lookup_compound(sent, max_edit_distance=2)\n    return suggestions[0]._term # returns the top suggestion\n\n######### tokenise pipeline\ndef tokenise_then_spellcheck(sent):\n    # 8 times faster than spellcheck_then_tokenise\n    tokens = spacy_tokenise(sent) # NOTE: replace tokenise with spacy_tokenise\n    checked_tokens = [spellcheck_single(token).lower() for token in tokens] # lower after spell check\n    return checked_tokens\n\ndef spellcheck_then_tokenise(sent):\n    checked_sent = spellcheck_compound(sent)\n    tokens = spacy_tokenise(checked_sent, lower=True) # lower after spell check\n    return tokens","metadata":{"_kg_hide-input":true,"papermill":{"duration":11.545898,"end_time":"2021-07-27T03:07:13.868595","exception":false,"start_time":"2021-07-27T03:07:02.322697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:09.106367Z","iopub.execute_input":"2021-08-11T15:43:09.106922Z","iopub.status.idle":"2021-08-11T15:43:23.366248Z","shell.execute_reply.started":"2021-08-11T15:43:09.106887Z","shell.execute_reply":"2021-08-11T15:43:23.365460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define tokenisation process\n\nimport pickle\nqid_to_tokens_preprocessed_filename = \"../input/quora-question-pairs-tokenise-pipeline/qid_to_processed_token_list_tokenise_then_spellcheck.pkl\"\nwith open(qid_to_tokens_preprocessed_filename, \"rb\") as f:\n    qid_to_tokens_preprocessed = pickle.load(f)\n\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstopword_set = set(stopwords.words())\nstopword_set.update([\"?\", \",\"])\n\ndef nltk_tokenize(sentence):\n    return word_tokenize(sentence.lower())\n\ndef tokenise_qid(qid, qid_to_tokens_preprocessed=qid_to_tokens_preprocessed, \n                 tokenise_method=tokenise_then_spellcheck):\n    # return a list of tokens, does not remove stopwords or duplicates\n    if qid_to_tokens_preprocessed and qid in qid_to_tokens_preprocessed:\n        return qid_to_tokens_preprocessed[qid]\n    return tokenise_method(qid_to_question[qid])","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.922682,"end_time":"2021-07-27T03:07:17.909271","exception":false,"start_time":"2021-07-27T03:07:13.986589","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:23.367317Z","iopub.execute_input":"2021-08-11T15:43:23.367716Z","iopub.status.idle":"2021-08-11T15:43:28.270894Z","shell.execute_reply.started":"2021-08-11T15:43:23.367687Z","shell.execute_reply":"2021-08-11T15:43:28.269710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_vsm(train_query_qids_list=train_query_qids_list, stopword_set=stopword_set, exclude_stopwords=True):\n    '''\n    Input:\n        qid_to_question = {qid: question string}\n            Note: only use the test subset of qids\n    \n    Outputs:\n        qid_to_tokens = {qid: set(tokens)}\n        token_to_qids = {token: set(qids)}\n        tf = {token: {qid: TF as int}}\n        df = {token: DF as int}\n        L = {qid: question length as int}\n    '''\n    qid_to_tokens = defaultdict(set)\n    token_to_qids = defaultdict(set)\n    tf = defaultdict(Counter)\n    df = defaultdict(int)\n    L = defaultdict(int)\n\n    qid_processed = set()\n    for qid in tqdm.tqdm(train_query_qids_list):\n        qid_tokenised = tokenise_qid(qid)\n\n        for token in set(qid_tokenised):\n            if token not in stopword_set or not exclude_stopwords:\n                # store qid-to-token mapping\n                qid_to_tokens[qid].add(token)\n                token_to_qids[token].add(qid)\n\n                # compute and store term frequency\n                tf[token][qid] += 1 \n\n                # store doc frequency in df\n                df[token] += 1\n\n        # store doc length in L (double-count repeated tokens)\n        L[qid] = len(qid_tokenised)\n        \n    # output\n    return qid_to_tokens, token_to_qids, tf, df, L","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.089859,"end_time":"2021-07-27T03:07:18.080114","exception":false,"start_time":"2021-07-27T03:07:17.990255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:28.272450Z","iopub.execute_input":"2021-08-11T15:43:28.272885Z","iopub.status.idle":"2021-08-11T15:43:28.283322Z","shell.execute_reply.started":"2021-08-11T15:43:28.272842Z","shell.execute_reply":"2021-08-11T15:43:28.281871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qid_to_tokens, token_to_qids, tf, df, L = preprocess_vsm()\n\n# save a copy of the original to allow reset later\nqid_to_tokens_original, token_to_qids_original = deepcopy(qid_to_tokens), deepcopy(token_to_qids)\ntf_original, df_original, L_original = deepcopy(tf), deepcopy(df), deepcopy(L)","metadata":{"papermill":{"duration":10.072982,"end_time":"2021-07-27T03:07:28.231276","exception":false,"start_time":"2021-07-27T03:07:18.158294","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:43:28.285171Z","iopub.execute_input":"2021-08-11T15:43:28.285757Z","iopub.status.idle":"2021-08-11T15:44:10.218732Z","shell.execute_reply.started":"2021-08-11T15:43:28.285710Z","shell.execute_reply":"2021-08-11T15:44:10.217610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 0 - Baseline\nOrder by the number of overlapping non-stopword words. Random if tie.","metadata":{"papermill":{"duration":0.078734,"end_time":"2021-07-27T03:07:28.391118","exception":false,"start_time":"2021-07-27T03:07:28.312384","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def method_overlapping_root_word_count(query_qid, ignore_stopwords=True):\n    query_tokens = set(tokenise_qid(query_qid))\n    if ignore_stopwords:\n        query_tokens = [token for token in query_tokens if token not in stopword_set]\n    counter = collections.Counter()\n    \n    for dummy_qid in random.choices(train_query_qids_list, k=RANKED_LIST_SIZE):\n        # prefill with random results to address the possibility of no matches\n        counter[dummy_qid] = 0.01\n    \n    for query_token in query_tokens:\n        counter += collections.Counter(token_to_qids[query_token])\n    \n    query_results = list(counter.items())\n    random.shuffle(query_results)  # so that qids are not ordered\n    query_results = sorted(query_results, key=lambda x:x[1], reverse=True)[:RANKED_LIST_SIZE]\n\n    return [x[0] for x in query_results], [x[1] for x in query_results]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.13074,"end_time":"2021-07-27T03:07:28.601757","exception":false,"start_time":"2021-07-27T03:07:28.471017","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:10.220232Z","iopub.execute_input":"2021-08-11T15:44:10.220558Z","iopub.status.idle":"2021-08-11T15:44:10.229166Z","shell.execute_reply.started":"2021-08-11T15:44:10.220530Z","shell.execute_reply":"2021-08-11T15:44:10.228063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_overlapping_root_word_count(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.211462,"end_time":"2021-07-27T03:07:28.892444","exception":false,"start_time":"2021-07-27T03:07:28.680982","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:10.231055Z","iopub.execute_input":"2021-08-11T15:44:10.231368Z","iopub.status.idle":"2021-08-11T15:44:10.374424Z","shell.execute_reply.started":"2021-08-11T15:44:10.231339Z","shell.execute_reply":"2021-08-11T15:44:10.373431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_overlapping_root_word_count = evaluation_process(method_overlapping_root_word_count)","metadata":{"papermill":{"duration":41.328264,"end_time":"2021-07-27T03:08:10.302016","exception":false,"start_time":"2021-07-27T03:07:28.973752","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:10.375779Z","iopub.execute_input":"2021-08-11T15:44:10.376069Z","iopub.status.idle":"2021-08-11T15:44:11.625571Z","shell.execute_reply.started":"2021-08-11T15:44:10.376041Z","shell.execute_reply":"2021-08-11T15:44:11.624417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1 - TFIDF","metadata":{"papermill":{"duration":0.122654,"end_time":"2021-07-27T03:08:10.547825","exception":false,"start_time":"2021-07-27T03:08:10.425171","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def compute_idf(doc_freq, N):\n    '''\n    Inputs:\n        doc_freq = document frequency of some token\n        N = corpus size including query\n    \n    Output:\n        idf = IDF as float\n    '''\n    return log(N/doc_freq)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.129287,"end_time":"2021-07-27T03:08:10.799104","exception":false,"start_time":"2021-07-27T03:08:10.669817","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:11.627044Z","iopub.execute_input":"2021-08-11T15:44:11.627358Z","iopub.status.idle":"2021-08-11T15:44:11.632592Z","shell.execute_reply.started":"2021-08-11T15:44:11.627326Z","shell.execute_reply":"2021-08-11T15:44:11.631141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import reduce\nimport operator\n\ndef prod(iterable):\n    return reduce(operator.mul, iterable, 1)\n\n\ndef use_vsm(qid_query, \\\n    # qid_to_tokens=qid_to_tokens, tf=tf, df=df, L=L,\n    method='tf-idf', compute_idf=compute_idf,\n    k1=1.5, k3=1.5, b=0.75,\n    smoothing='add-one', alpha=0.75, eps=10**(-6),\n    exclude_stopwords=True,\n    return_top=RANKED_LIST_SIZE):\n    \n    '''\n    Inputs:\n        qid_query = qid of question match   # this comes from \"test\" set\n        qid_to_tokens = {qid: set(tokens)}  # this is the \"training\" corpus\n        tf = {token: term freq}             # required for all methods\n        df = {token: doc freq}              # required for method='tf-idf','bm25'\n        L = {qid: doc length}               # required for method='bm25','unigram'\n\n        method = model to apply\n        k1, k3, b = tuning params           # required for method='bm25'\n        smoothing = type of smoothing       # required for method='unigram'\n        return_top = num of docs to return\n    \n    Procedure:\n        0. Corpus is already tokenised, tf, df, L already computed\n        1. Tokenise query, expand tf, df, L with query information\n        \n        if method='boolean':\n            Remove idf calculation, then use method='tf-idf'\n\n        if method='tf-idf':\n            2. Compute tf-idf weights only for relevant (t,d) pairs\n            3. Compute cosine similarity only for docs containing query terms\n        \n        if method='bm25':\n            2. Compute RSV summation terms only for relevant (t,d) pairs\n            3. Compute RSV only for docs containing query terms\n        \n        if method='unigram':\n            2. Compute probabilities only for relevant (t,d) pairs\n            3. Compute query probability only for docs containing query terms\n        \n        4. Return docs in ranked order\n\n    Output:\n        ranking = [qids in decreasing order of match]\n        scoring = [corresponding scores]\n    '''\n    \n    assert method in ['boolean','tf-idf','bm25','unigram'], \"Supported methods: 'boolean', 'tf-idf', 'bm25', 'unigram'\"\n    assert len(L.keys()) > 0 if method=='bm25' else True, \"Please include L for bm25\"\n    assert len(L.keys()) > 0 if method=='unigram' else True, \"Please include L for unigram\"\n    assert smoothing in ['add-one','linear-interpolation'] if method=='unigram' else True\n    assert alpha >= 0 and alpha <= 1 if smoothing=='linear-interpolation' else True\n\n    qid_tmp = time.time()\n\n    ''' STEP 1: PROCESS QUERY '''\n    query_tokenised = tokenise_qid(qid_query)\n    \n    for token in set(query_tokenised):\n        if token not in stopword_set or not exclude_stopwords:\n            # store qid-to-token mapping\n            # store query as qid=0 (corpus starts from qid=1)\n            qid_to_tokens[qid_tmp].add(token)\n\n            # compute and store term frequency\n            tf[token][qid_tmp] = sum([1 if t==token else 0 for t in query_tokenised])\n            \n            # update doc frequency in df\n            df[token] += 1\n    \n    # store query length\n    L[qid_tmp] = len(query_tokenised)\n\n    if method=='boolean':\n        def compute_idf(doc_freq, N):\n            return 1\n        method = 'tf-idf'\n        \n    if method=='tf-idf':\n        \n        ''' STEP 2: COMPUTE TF-IDF WEIGHTS '''\n        weights = defaultdict(lambda: defaultdict(float))\n        N = len(qid_to_tokens) # original corpus + query\n\n        # only bother computing for tokens in the query\n        for token in set(query_tokenised):\n            if token not in stopword_set or exclude_stopwords==False:\n                weights[qid_tmp][token] = tf[token][qid_tmp] * compute_idf(df[token], N)\n                \n                for qid in tf[token].keys():\n                    weights[qid][token] = tf[token][qid] * compute_idf(df[token], N)\n                    \n                    # also compute weight for other tokens contained by these qids\n                    # needed for computing qid vector length\n                    for other_token in qid_to_tokens[qid]:\n                        weights[qid][other_token] = tf[other_token][qid] * compute_idf(df[other_token], N)\n\n                        \n        ''' STEP 3: COMPUTE COSINE SIMILARITY TO QUERY '''\n        cosine_similarities = defaultdict(float)\n        # compute denominator (part 1), i.e., |q| * |d|\n        query_vector_length = (sum([w**2 for w in weights[qid_tmp].values()]))**0.5\n        \n        for qid in weights.keys():\n            \n            # compute numerator, i.e., dot product of q and d\n            cosine_numerator = 0\n            \n            for token in weights[qid].keys():\n                if token in weights[qid_tmp]:\n                    cosine_numerator += weights[qid][token] * weights[qid_tmp][token]\n            \n            # compute denominator (part 2), i.e., |q| * |d|\n            qid_vector_length = (sum([w**2 for w in weights[qid].values()]))**0.5\n\n            # compute and store cosine similarity between q and d\n            cosine_similarities[qid] = cosine_numerator / (query_vector_length+eps) / (qid_vector_length+eps)\n        \n        scores = cosine_similarities\n\n    if method=='bm25':\n\n        ''' STEP 2: COMPUTE RSV TERMS '''\n        rsv_terms = defaultdict(lambda: defaultdict(float))\n        N = len(qid_to_tokens) # original corpus + query\n        L_avg = sum(L.values())/len(L.values())\n\n        # only bother computing for tokens in the query\n        for token in set(query_tokenised):\n            for qid in tf[token].keys():\n                rsv_terms[qid][token] = compute_idf(df[token], N) \\\n                    * (k1+1)*tf[token][qid] / (k1*((1-b)+b*L[qid]*L_avg) + tf[token][qid]) \\\n                        * (k3+1)*tf[token][qid_tmp] / (k3 + tf[token][qid_tmp])\n\n        ''' STEP 3: COMPUTE RSV '''\n        rsv = {qid: sum(rsv_terms[qid].values()) for qid in rsv_terms.keys()}\n        scores = rsv\n    \n    if method=='unigram':\n        \n        ''' STEP 2: COMPUTE PROBABILITIES '''\n        probabilities = defaultdict(lambda: defaultdict(float))\n        corpus_model = defaultdict(float)\n        \n        # only bother computing for tokens in the query\n        for token in set(query_tokenised):\n            for qid in tf[token].keys():\n\n                if smoothing=='add-one':\n                    probabilities[qid][token] = (tf[token][qid]+1) / (L[qid]+len(query_tokenised))\n                else:\n                    probabilities[qid][token] = (tf[token][qid]) / (L[qid])\n\n                # for linear-interpolation smoothing, build corpus language model\n                if smoothing=='linear-interpolation':\n                    corpus_model[token] += tf[token][qid]\n\n        # remaining operations for linear-interpolation smoothing        \n        if smoothing=='linear-interpolation':\n            # finish building corpus language model by dividing corpus tf by corpus L\n            total_corpus_length = sum(L.values())\n            for token in corpus_model.keys():\n                corpus_model[token] = corpus_model[token] / total_corpus_length\n            \n            # then update the probabilities\n            for qid in probabilities.keys():\n                for token in probabilities[qid].keys():\n                    probabilities[qid][token] = alpha*probabilities[qid][token] + (1-alpha)*corpus_model[token]\n\n        ''' STEP 3: COMPUTE QUERY PROBABILITY '''\n        query_prob = {qid: -log(prod(probabilities[qid].values())) for qid in probabilities.keys()}\n        scores = query_prob\n\n    ''' STEP 4: RANK DOCUMENTS AND RETURN RESULT '''\n    # cleanup\n    if qid_tmp in qid_to_tokens:\n        del qid_to_tokens[qid_tmp]\n    for token in set(query_tokenised):\n        if token not in stopword_set or not exclude_stopwords:\n            del tf[token][qid_tmp]\n            df[token] -= 1\n    \n    if qid_tmp in scores:\n        del scores[qid_tmp] # remove query from result\n    ranking = sorted(scores, key=scores.get, reverse=True)\n    scoring = sorted(scores.values(), reverse=True)\n\n    # if too few documents match the query, add dummy documents\n    if len(ranking) < return_top:\n        ranking.extend([0]*(return_top-len(ranking)))\n        scoring.extend([0]*(return_top-len(ranking)))\n\n    # return top k results\n    return ranking[:return_top], scoring[:return_top]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.151547,"end_time":"2021-07-27T03:08:11.072416","exception":false,"start_time":"2021-07-27T03:08:10.920869","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:11.634143Z","iopub.execute_input":"2021-08-11T15:44:11.634461Z","iopub.status.idle":"2021-08-11T15:44:11.670493Z","shell.execute_reply.started":"2021-08-11T15:44:11.634432Z","shell.execute_reply":"2021-08-11T15:44:11.669470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_boolean(qid):\n    return use_vsm(qid, method='boolean')","metadata":{"papermill":{"duration":0.088761,"end_time":"2021-07-27T03:08:11.242566","exception":false,"start_time":"2021-07-27T03:08:11.153805","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:11.671902Z","iopub.execute_input":"2021-08-11T15:44:11.672181Z","iopub.status.idle":"2021-08-11T15:44:11.690525Z","shell.execute_reply.started":"2021-08-11T15:44:11.672154Z","shell.execute_reply":"2021-08-11T15:44:11.689444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_boolean(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.569085,"end_time":"2021-07-27T03:08:11.933683","exception":false,"start_time":"2021-07-27T03:08:11.364598","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:11.691561Z","iopub.execute_input":"2021-08-11T15:44:11.691868Z","iopub.status.idle":"2021-08-11T15:44:12.303000Z","shell.execute_reply.started":"2021-08-11T15:44:11.691841Z","shell.execute_reply":"2021-08-11T15:44:12.301808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_boolean = evaluation_process(method_boolean)","metadata":{"papermill":{"duration":229.032717,"end_time":"2021-07-27T03:12:01.089996","exception":false,"start_time":"2021-07-27T03:08:12.057279","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:12.304548Z","iopub.execute_input":"2021-08-11T15:44:12.304831Z","iopub.status.idle":"2021-08-11T15:44:16.820643Z","shell.execute_reply.started":"2021-08-11T15:44:12.304803Z","shell.execute_reply":"2021-08-11T15:44:16.819311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_tf_idf(qid):\n    return use_vsm(qid, method='tf-idf')","metadata":{"papermill":{"duration":0.089012,"end_time":"2021-07-27T03:12:01.262758","exception":false,"start_time":"2021-07-27T03:12:01.173746","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:16.822169Z","iopub.execute_input":"2021-08-11T15:44:16.822740Z","iopub.status.idle":"2021-08-11T15:44:16.827725Z","shell.execute_reply.started":"2021-08-11T15:44:16.822692Z","shell.execute_reply":"2021-08-11T15:44:16.826580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_tf_idf(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.566464,"end_time":"2021-07-27T03:12:01.911336","exception":false,"start_time":"2021-07-27T03:12:01.344872","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:16.829208Z","iopub.execute_input":"2021-08-11T15:44:16.829659Z","iopub.status.idle":"2021-08-11T15:44:17.536501Z","shell.execute_reply.started":"2021-08-11T15:44:16.829616Z","shell.execute_reply":"2021-08-11T15:44:17.535299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_tf_idf = evaluation_process(method_tf_idf)","metadata":{"papermill":{"duration":270.338967,"end_time":"2021-07-27T03:16:32.332994","exception":false,"start_time":"2021-07-27T03:12:01.994027","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:17.538408Z","iopub.execute_input":"2021-08-11T15:44:17.538840Z","iopub.status.idle":"2021-08-11T15:44:22.772150Z","shell.execute_reply.started":"2021-08-11T15:44:17.538793Z","shell.execute_reply":"2021-08-11T15:44:22.771129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2 - BM25","metadata":{"papermill":{"duration":0.126043,"end_time":"2021-07-27T03:16:32.586026","exception":false,"start_time":"2021-07-27T03:16:32.459983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def method_bm25(qid):\n    return use_vsm(qid, method='bm25')","metadata":{"papermill":{"duration":0.134424,"end_time":"2021-07-27T03:16:32.848962","exception":false,"start_time":"2021-07-27T03:16:32.714538","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:22.773643Z","iopub.execute_input":"2021-08-11T15:44:22.774047Z","iopub.status.idle":"2021-08-11T15:44:22.779190Z","shell.execute_reply.started":"2021-08-11T15:44:22.774002Z","shell.execute_reply":"2021-08-11T15:44:22.777676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_bm25(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.272279,"end_time":"2021-07-27T03:16:33.248359","exception":false,"start_time":"2021-07-27T03:16:32.97608","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:22.780645Z","iopub.execute_input":"2021-08-11T15:44:22.780966Z","iopub.status.idle":"2021-08-11T15:44:23.010963Z","shell.execute_reply.started":"2021-08-11T15:44:22.780936Z","shell.execute_reply":"2021-08-11T15:44:23.009695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_bm25 = evaluation_process(method_bm25)","metadata":{"papermill":{"duration":93.119547,"end_time":"2021-07-27T03:18:06.49716","exception":false,"start_time":"2021-07-27T03:16:33.377613","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:23.012160Z","iopub.execute_input":"2021-08-11T15:44:23.012472Z","iopub.status.idle":"2021-08-11T15:44:27.063711Z","shell.execute_reply.started":"2021-08-11T15:44:23.012439Z","shell.execute_reply":"2021-08-11T15:44:27.062607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3 - Unigram Language Model","metadata":{}},{"cell_type":"code","source":"def method_unigram(qid):\n    # see previous versions for results for other alpha values and smoothing='add-one'\n    return use_vsm(qid, method='unigram', smoothing='linear-interpolation', alpha=0)","metadata":{"papermill":{"duration":0.093103,"end_time":"2021-07-27T03:18:06.676069","exception":false,"start_time":"2021-07-27T03:18:06.582966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:27.065433Z","iopub.execute_input":"2021-08-11T15:44:27.065872Z","iopub.status.idle":"2021-08-11T15:44:27.070866Z","shell.execute_reply.started":"2021-08-11T15:44:27.065825Z","shell.execute_reply":"2021-08-11T15:44:27.070035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_unigram(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.285686,"end_time":"2021-07-27T03:18:07.050747","exception":false,"start_time":"2021-07-27T03:18:06.765061","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:27.071891Z","iopub.execute_input":"2021-08-11T15:44:27.072358Z","iopub.status.idle":"2021-08-11T15:44:27.281367Z","shell.execute_reply.started":"2021-08-11T15:44:27.072325Z","shell.execute_reply":"2021-08-11T15:44:27.280306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_unigram = evaluation_process(method_unigram)","metadata":{"papermill":{"duration":81.456535,"end_time":"2021-07-27T03:19:28.63621","exception":false,"start_time":"2021-07-27T03:18:07.179675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:27.283548Z","iopub.execute_input":"2021-08-11T15:44:27.283987Z","iopub.status.idle":"2021-08-11T15:44:29.037695Z","shell.execute_reply.started":"2021-08-11T15:44:27.283938Z","shell.execute_reply":"2021-08-11T15:44:29.036530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 4 - Word Embeddings","metadata":{"papermill":{"duration":0.143012,"end_time":"2021-07-27T03:27:40.342224","exception":false,"start_time":"2021-07-27T03:27:40.199212","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### Using spaCy models","metadata":{}},{"cell_type":"code","source":"def to_vec(token_or_list):\n    # converts a token string or a list of tokens into a word or doc vec respectively\n    if type(token_or_list) == list:\n        # token list needs to be joined into a sentence first\n        token_or_list = ' '.join(token_or_list)\n    return nlp(token_or_list).vector\n\nnlp2 = spacy.load(\"en_core_web_lg\")\ndef to_vec2(token_or_list):\n    # converts a token string or a list of tokens into a word or doc vec respectively\n    if type(token_or_list) == list:\n        # token list needs to be joined into a sentence first\n        token_or_list = ' '.join(token_or_list)\n    return nlp2(token_or_list).vector","metadata":{"papermill":{"duration":7.71044,"end_time":"2021-07-27T03:27:48.196842","exception":false,"start_time":"2021-07-27T03:27:40.486402","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:29.041136Z","iopub.execute_input":"2021-08-11T15:44:29.041474Z","iopub.status.idle":"2021-08-11T15:44:38.786153Z","shell.execute_reply.started":"2021-08-11T15:44:29.041443Z","shell.execute_reply":"2021-08-11T15:44:38.785238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pre-processed dict\nwith open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_vec.pkl\", \"rb\") as f:\n    qid_to_vec = pickle.load(f)\n\nprint(\"Pre-processed question vector is of shape {}\".format(qid_to_vec[0].shape))","metadata":{"papermill":{"duration":3.830079,"end_time":"2021-07-27T03:27:52.18346","exception":false,"start_time":"2021-07-27T03:27:48.353381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:38.787487Z","iopub.execute_input":"2021-08-11T15:44:38.787978Z","iopub.status.idle":"2021-08-11T15:44:43.400833Z","shell.execute_reply.started":"2021-08-11T15:44:38.787945Z","shell.execute_reply":"2021-08-11T15:44:43.399638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import dot\nfrom numpy.linalg import norm\n\ndef method_spacy_embedding_similarity(test_qid):\n    tokens = tokenise_then_spellcheck(qid_to_question[test_qid])\n    test_vec = to_vec(tokens)\n    \n    ## Run baseline model as a filter\n    qid_list, scores = method_overlapping_root_word_count(test_qid)\n    \n    cos_sims = [] # bigger better\n    for train_qid in qid_list:# train_query_qids_list:\n        train_vec = qid_to_vec[train_qid]\n        cos_sim = dot(test_vec, train_vec)/(norm(test_vec)*norm(train_vec))\n        cos_sims.append(cos_sim)\n\n    cos_sims = np.array(cos_sims)\n    qid_list = np.array(qid_list) # train_query_qids_list\n    inds = cos_sims.argsort()[::-1] # reverse so biggest come first\n    cos_sims = cos_sims[inds]\n    ranklist = qid_list[inds] \n\n    return ranklist[:RANKED_LIST_SIZE], cos_sims[:RANKED_LIST_SIZE]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.104052,"end_time":"2021-07-27T03:27:52.383802","exception":false,"start_time":"2021-07-27T03:27:52.27975","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:43.404241Z","iopub.execute_input":"2021-08-11T15:44:43.404630Z","iopub.status.idle":"2021-08-11T15:44:43.412554Z","shell.execute_reply.started":"2021-08-11T15:44:43.404599Z","shell.execute_reply":"2021-08-11T15:44:43.411463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_spacy_embedding_similarity(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.213195,"end_time":"2021-07-27T03:27:52.692233","exception":false,"start_time":"2021-07-27T03:27:52.479038","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:43.413768Z","iopub.execute_input":"2021-08-11T15:44:43.414185Z","iopub.status.idle":"2021-08-11T15:44:43.580835Z","shell.execute_reply.started":"2021-08-11T15:44:43.414155Z","shell.execute_reply":"2021-08-11T15:44:43.579870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_spacy_embedding_similarity = evaluation_process(method_spacy_embedding_similarity)","metadata":{"papermill":{"duration":56.957678,"end_time":"2021-07-27T03:28:49.745604","exception":false,"start_time":"2021-07-27T03:27:52.787926","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:43.584322Z","iopub.execute_input":"2021-08-11T15:44:43.584696Z","iopub.status.idle":"2021-08-11T15:44:44.960284Z","shell.execute_reply.started":"2021-08-11T15:44:43.584660Z","shell.execute_reply":"2021-08-11T15:44:44.959161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_vec_trf.pkl\", \"rb\") as f: # note, actually lg not trf\n    qid_to_vec2 = pickle.load(f)\n\nprint(\"Pre-processed question vector is of shape {}\".format(qid_to_vec2[0].shape)) # 300 dim vec","metadata":{"papermill":{"duration":8.707086,"end_time":"2021-07-27T03:28:58.550388","exception":false,"start_time":"2021-07-27T03:28:49.843302","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:44.961550Z","iopub.execute_input":"2021-08-11T15:44:44.961853Z","iopub.status.idle":"2021-08-11T15:44:52.554250Z","shell.execute_reply.started":"2021-08-11T15:44:44.961823Z","shell.execute_reply":"2021-08-11T15:44:52.552964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_spacy_embedding_similarity_lg(test_qid):\n    tokens = tokenise_then_spellcheck(qid_to_question[test_qid])\n    test_vec = to_vec2(tokens)\n    \n    ## Run baseline model as a filter\n    qid_list, scores = method_overlapping_root_word_count(test_qid)\n    \n    cos_sims = [] # bigger better\n    for train_qid in qid_list:# train_query_qids_list:\n        train_vec = qid_to_vec2[train_qid]\n        cos_sim = dot(test_vec, train_vec)/(norm(test_vec)*norm(train_vec))\n        cos_sims.append(cos_sim)\n\n    cos_sims = np.array(cos_sims)\n    qid_list = np.array(qid_list) # train_query_qids_list)\n    inds = cos_sims.argsort()[::-1] # reverse so biggest come first\n    cos_sims = cos_sims[inds]\n    ranklist = qid_list[inds] \n\n    return ranklist[:RANKED_LIST_SIZE], cos_sims[:RANKED_LIST_SIZE]","metadata":{"papermill":{"duration":0.108147,"end_time":"2021-07-27T03:28:58.755706","exception":false,"start_time":"2021-07-27T03:28:58.647559","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:52.555720Z","iopub.execute_input":"2021-08-11T15:44:52.556040Z","iopub.status.idle":"2021-08-11T15:44:52.564588Z","shell.execute_reply.started":"2021-08-11T15:44:52.556007Z","shell.execute_reply":"2021-08-11T15:44:52.563356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_spacy_embedding_similarity_lg(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.203005,"end_time":"2021-07-27T03:28:59.057568","exception":false,"start_time":"2021-07-27T03:28:58.854563","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:52.566613Z","iopub.execute_input":"2021-08-11T15:44:52.567518Z","iopub.status.idle":"2021-08-11T15:44:52.723701Z","shell.execute_reply.started":"2021-08-11T15:44:52.567437Z","shell.execute_reply":"2021-08-11T15:44:52.722411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_spacy_embedding_similarity_lg = evaluation_process(method_spacy_embedding_similarity_lg)","metadata":{"papermill":{"duration":66.276217,"end_time":"2021-07-27T03:30:05.48221","exception":false,"start_time":"2021-07-27T03:28:59.205993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:52.725129Z","iopub.execute_input":"2021-08-11T15:44:52.725579Z","iopub.status.idle":"2021-08-11T15:44:54.192921Z","shell.execute_reply.started":"2021-08-11T15:44:52.725533Z","shell.execute_reply":"2021-08-11T15:44:54.191862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gensim WordMover Distance on Boolean Retrieval\n* Applies further sorting by wordmover distance on the output ranklist of Boolean Retrieval  \n* Current pre-trained model: `glove-wiki-gigaword-50`\n","metadata":{"papermill":{"duration":0.102012,"end_time":"2021-07-27T03:30:05.685815","exception":false,"start_time":"2021-07-27T03:30:05.583803","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gensim\nimport gensim.downloader\n# gensim.downloader.info() # find more models to download\n\nfrom gensim.models import KeyedVectors\n\ntry: model = KeyedVectors.load(\"../input/ir-project-download-keyed-vectors/glove-wiki-gigaword-50.keyedvectors\")\nexcept: # gs_model not downloaded\n    model = gensim.downloader.load('glove-wiki-gigaword-50')\n    # model.save(\"/kaggle/working/glove-wiki-gigaword-50.keyedvectors\") # if not already saved","metadata":{"papermill":{"duration":2.795339,"end_time":"2021-07-27T03:30:08.581535","exception":false,"start_time":"2021-07-27T03:30:05.786196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:54.194273Z","iopub.execute_input":"2021-08-11T15:44:54.194602Z","iopub.status.idle":"2021-08-11T15:44:55.609417Z","shell.execute_reply.started":"2021-08-11T15:44:54.194570Z","shell.execute_reply":"2021-08-11T15:44:55.608260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_wordmover_distance(test_qid, model):\n    # out of box duplicate finder does not work!\n    # returns ranklist and scores of each size RANKED_LIST_SIZE\n    \n    ## Run baseline model as a filter\n    ranklist, scores = method_overlapping_root_word_count(test_qid)\n    \n    ## Process test question\n    test_qn = tokenise_qid(test_qid)\n    \n    ## Get wordmover distance from every candidate\n    distances = []\n    qid_list = ranklist\n    for candidate_qid in qid_list:\n        candidate_qn = tokenise_qid(candidate_qid)\n        distances.append(1-model.wmdistance(test_qn, candidate_qn))\n    \n    ## Sort by distance\n    sorted_dist_and_candidate_qid = sorted(zip(distances,qid_list))[::-1]\n    sorted_candidate_qid = [qid for _,qid in sorted_dist_and_candidate_qid]\n    sorted_dist = [dist for dist,_ in sorted_dist_and_candidate_qid]\n    return sorted_candidate_qid[:RANKED_LIST_SIZE], sorted_dist[:RANKED_LIST_SIZE]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.164531,"end_time":"2021-07-27T03:30:08.895849","exception":false,"start_time":"2021-07-27T03:30:08.731318","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:55.611110Z","iopub.execute_input":"2021-08-11T15:44:55.611564Z","iopub.status.idle":"2021-08-11T15:44:55.627566Z","shell.execute_reply.started":"2021-08-11T15:44:55.611517Z","shell.execute_reply":"2021-08-11T15:44:55.626531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_wordmover_distance_glovewiki50(test_qid):\n    return method_wordmover_distance(test_qid, model)","metadata":{"papermill":{"duration":0.104814,"end_time":"2021-07-27T03:30:09.100109","exception":false,"start_time":"2021-07-27T03:30:08.995295","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:55.628949Z","iopub.execute_input":"2021-08-11T15:44:55.629269Z","iopub.status.idle":"2021-08-11T15:44:55.650341Z","shell.execute_reply.started":"2021-08-11T15:44:55.629238Z","shell.execute_reply":"2021-08-11T15:44:55.649166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_wordmover_distance_glovewiki50(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.529375,"end_time":"2021-07-27T03:30:09.729731","exception":false,"start_time":"2021-07-27T03:30:09.200356","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:55.651889Z","iopub.execute_input":"2021-08-11T15:44:55.652257Z","iopub.status.idle":"2021-08-11T15:44:56.147232Z","shell.execute_reply.started":"2021-08-11T15:44:55.652222Z","shell.execute_reply":"2021-08-11T15:44:56.145867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_wordmover_distance_glovewiki50 = evaluation_process(method_wordmover_distance_glovewiki50)","metadata":{"papermill":{"duration":407.490172,"end_time":"2021-07-27T03:36:57.320453","exception":false,"start_time":"2021-07-27T03:30:09.830281","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:44:56.148606Z","iopub.execute_input":"2021-08-11T15:44:56.148923Z","iopub.status.idle":"2021-08-11T15:45:03.289047Z","shell.execute_reply.started":"2021-08-11T15:44:56.148892Z","shell.execute_reply":"2021-08-11T15:45:03.287458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_to_try = ['glove-wiki-gigaword-300', 'glove-twitter-50','word2vec-google-news-300','fasttext-wiki-news-subwords-300']\n\nif not EVALUATING:\n    models_to_try = []\n\nfor m in models_to_try:\n    print(\"Model: \",m)\n    try:\n        model = KeyedVectors.load(f\"../input/ir-project-download-keyed-vectors/{m}.keyedvectors\")\n    except:\n        model = gensim.downloader.load(m)\n\n    def method_wordmover_distance_new_model(test_qid):\n        return method_wordmover_distance(test_qid, model)\n\n    show_sample_query_results(test_query_qids_list[0], *method_wordmover_distance_new_model(test_query_qids_list[0]))\n\n    _ = evaluation_process(method_wordmover_distance_new_model)","metadata":{"papermill":{"duration":1590.724927,"end_time":"2021-07-27T04:03:28.198666","exception":false,"start_time":"2021-07-27T03:36:57.473739","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:45:03.290753Z","iopub.execute_input":"2021-08-11T15:45:03.291211Z","iopub.status.idle":"2021-08-11T15:45:03.298376Z","shell.execute_reply.started":"2021-08-11T15:45:03.291162Z","shell.execute_reply":"2021-08-11T15:45:03.297178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 5 - Sentence Embeddings\n\nEach sentence can be embedded as a vector with SentenceTransformer","metadata":{"papermill":{"duration":0.109411,"end_time":"2021-07-27T04:03:28.417517","exception":false,"start_time":"2021-07-27T04:03:28.308106","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install sentence-transformers > /dev/null","metadata":{"papermill":{"duration":13.01968,"end_time":"2021-07-27T04:03:41.547066","exception":false,"start_time":"2021-07-27T04:03:28.527386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:45:03.300067Z","iopub.execute_input":"2021-08-11T15:45:03.300602Z","iopub.status.idle":"2021-08-11T15:45:17.096015Z","shell.execute_reply.started":"2021-08-11T15:45:03.300545Z","shell.execute_reply":"2021-08-11T15:45:17.094996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\nmodel_name = 'bert-base-nli-stsb-mean-tokens'\nmodel_tf = SentenceTransformer(model_name)","metadata":{"papermill":{"duration":30.338695,"end_time":"2021-07-27T04:04:11.993728","exception":false,"start_time":"2021-07-27T04:03:41.655033","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:45:17.098640Z","iopub.execute_input":"2021-08-11T15:45:17.098966Z","iopub.status.idle":"2021-08-11T15:45:41.143904Z","shell.execute_reply.started":"2021-08-11T15:45:17.098932Z","shell.execute_reply":"2021-08-11T15:45:41.142774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"bert-base-nli-stsb-mean-tokens\"\nsentence_vectors = np.load(f\"../input/quora-question-pairs-bert-sentence-vectors/sentence_vectors_{model_name}.npy\")\nsentence_vectors = {i:vec for i,vec in enumerate(sentence_vectors)}","metadata":{"_kg_hide-input":true,"papermill":{"duration":17.52396,"end_time":"2021-07-27T04:04:29.625343","exception":false,"start_time":"2021-07-27T04:04:12.101383","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:45:41.145457Z","iopub.execute_input":"2021-08-11T15:45:41.145754Z","iopub.status.idle":"2021-08-11T15:46:03.496837Z","shell.execute_reply.started":"2021-08-11T15:45:41.145725Z","shell.execute_reply":"2021-08-11T15:46:03.494951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.spatial.distance import cosine\n\ndef method_sentence_vector(query_qid, method_preliminary=method_overlapping_root_word_count, preliminary_factor=1):\n    # method_preliminary can be either of the previous methods\n    # recommended method_overlapping_root_word_count, method_boolean, method_tf_idf\n    sentence_vectors[query_qid] = model_tf.encode(qid_to_question[query_qid], show_progress_bar=False)\n\n    qid_list, preliminary_scores = method_preliminary(query_qid)\n    \n    # sort by cosine similarity\n    query_sentence_vector = sentence_vectors[query_qid]\n    query_results = [(qid, preliminary_factor*preliminary_score+1-abs(cosine(query_sentence_vector, sentence_vectors[qid])))\n                     for qid,preliminary_score in zip(qid_list,preliminary_scores)]\n    query_results = sorted(query_results, key=lambda x:x[1], reverse=True)[:RANKED_LIST_SIZE]\n    \n    return [x[0] for x in query_results], [x[1] for x in query_results]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.124698,"end_time":"2021-07-27T04:04:29.870859","exception":false,"start_time":"2021-07-27T04:04:29.746161","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:03.507365Z","iopub.execute_input":"2021-08-11T15:46:03.507811Z","iopub.status.idle":"2021-08-11T15:46:03.516759Z","shell.execute_reply.started":"2021-08-11T15:46:03.507763Z","shell.execute_reply":"2021-08-11T15:46:03.515976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_sentence_vector(test_query_qids_list[0], preliminary_factor=0))","metadata":{"papermill":{"duration":0.223248,"end_time":"2021-07-27T04:04:30.21088","exception":false,"start_time":"2021-07-27T04:04:29.987632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:03.518877Z","iopub.execute_input":"2021-08-11T15:46:03.519649Z","iopub.status.idle":"2021-08-11T15:46:03.892271Z","shell.execute_reply.started":"2021-08-11T15:46:03.519605Z","shell.execute_reply":"2021-08-11T15:46:03.890995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_sentence_vector(test_query_qids_list[0], preliminary_factor=1))","metadata":{"papermill":{"duration":0.208374,"end_time":"2021-07-27T04:04:30.533463","exception":false,"start_time":"2021-07-27T04:04:30.325089","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:03.893857Z","iopub.execute_input":"2021-08-11T15:46:03.894258Z","iopub.status.idle":"2021-08-11T15:46:04.096705Z","shell.execute_reply.started":"2021-08-11T15:46:03.894222Z","shell.execute_reply":"2021-08-11T15:46:04.095073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_sentence_vector = evaluation_process(method_sentence_vector)","metadata":{"papermill":{"duration":49.999948,"end_time":"2021-07-27T04:05:20.650273","exception":false,"start_time":"2021-07-27T04:04:30.650325","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:04.098270Z","iopub.execute_input":"2021-08-11T15:46:04.098652Z","iopub.status.idle":"2021-08-11T15:46:06.224624Z","shell.execute_reply.started":"2021-08-11T15:46:04.098612Z","shell.execute_reply":"2021-08-11T15:46:06.223289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 6 - Supervised Model","metadata":{"papermill":{"duration":0.176126,"end_time":"2021-07-27T04:05:21.002006","exception":false,"start_time":"2021-07-27T04:05:20.82588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SUPERVISED_MODEL_TRAINING_SET_SIZE = 10000\nLOAD_DATA_FOR_SUPERVISED = True\nDIR_DATA_FOR_SUPERVISED = \"../input/ir-project-supervised-model-data-preparation/\"\n\nsupervised_query_qids = random.sample(set(qid_to_duplicate_qids.keys()) - set(test_query_qids_list), \n                                      SUPERVISED_MODEL_TRAINING_SET_SIZE)\n\ndef create_supervised_features(qids, testing=True):\n    \n    kwargs = {\"test_query_qids_list\": qids, \"calculate_metrics\": False, \"use_tqdm\": False}\n    method_to_ranklists_scorelists_supervised = {\n        \"overlapping_root_word_count\": evaluation_process(method_overlapping_root_word_count, **kwargs),\n        \"boolean\": evaluation_process(method_boolean, **kwargs),\n        \"tf_idf\": evaluation_process(method_tf_idf, **kwargs),\n        \"bm25\": evaluation_process(method_bm25, **kwargs),\n        \"unigram\": evaluation_process(method_unigram, **kwargs),\n        \"spacy_embedding_similarity\": evaluation_process(method_spacy_embedding_similarity, **kwargs),\n        \"spacy_embedding_similarity_lg\": evaluation_process(method_spacy_embedding_similarity_lg, **kwargs),\n        \"wordmover_distance_glovewiki50\": evaluation_process(method_wordmover_distance_glovewiki50, **kwargs),\n        \"sentence_vector\": evaluation_process(method_sentence_vector, **kwargs),\n    }\n    return method_to_ranklists_scorelists_supervised\n\ndef parse_ndarray(obj):  # https://stackoverflow.com/a/52604722/5894029\n    if isinstance(obj, np.ndarray):\n        return obj.tolist()\n\nif not LOAD_DATA_FOR_SUPERVISED:\n    method_to_ranklists_scorelists_supervised = create_supervised_features(supervised_query_qids)\n\n    with open(DIR_DATA_FOR_SUPERVISED+'supervised_query_qids.json', 'w') as f:\n        json.dump(supervised_query_qids, f, indent=4, default=parse_ndarray)    \n\n    with open(DIR_DATA_FOR_SUPERVISED+'method_to_ranklists_scorelists_supervised.json', 'w') as f:\n        json.dump(method_to_ranklists_scorelists_supervised, f, indent=4, default=parse_ndarray)\n\nwith open(DIR_DATA_FOR_SUPERVISED+'supervised_query_qids.json') as f:\n    supervised_query_qids = json.load(f)\n\nwith open(DIR_DATA_FOR_SUPERVISED+'method_to_ranklists_scorelists_supervised.json') as f:\n    method_to_ranklists_scorelists_supervised = json.load(f)","metadata":{"papermill":{"duration":13037.519327,"end_time":"2021-07-27T07:42:38.69573","exception":false,"start_time":"2021-07-27T04:05:21.176403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:06.226325Z","iopub.execute_input":"2021-08-11T15:46:06.226831Z","iopub.status.idle":"2021-08-11T15:46:18.229853Z","shell.execute_reply.started":"2021-08-11T15:46:06.226784Z","shell.execute_reply":"2021-08-11T15:46:18.228689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_supervised_features_into_df(method_to_ranklists_scorelists_supervised, training=False, \n                                      supervised_query_qids_set=set(supervised_query_qids)):\n    supervised_scores = defaultdict(dict)\n    for method, (ranklists, scorelists) in method_to_ranklists_scorelists_supervised.items():\n        for supervised_query_qid, ranklist, scorelist in zip(supervised_query_qids, ranklists, scorelists):\n            for candidate_qid, score in zip(ranklist, scorelist):\n                if training and candidate_qid in supervised_query_qids_set:\n                    continue\n                supervised_scores[supervised_query_qid, candidate_qid][method] = score\n                \n    df_supervised = pd.DataFrame.from_dict(supervised_scores, orient='index')\n    return df_supervised\n\ndef extract_supervised_labels_from_df(df_supervised):\n    supervised_labels = [int(candidate_qid in qid_to_duplicate_qids[supervised_query_qid]) \n                         for supervised_query_qid, candidate_qid in df_supervised.index]\n    return supervised_labels\n\ndf_supervised = parse_supervised_features_into_df(method_to_ranklists_scorelists_supervised, training=True)\nsupervised_labels = extract_supervised_labels_from_df(df_supervised)\n\n# extracted and total number of positive labels\nsum(supervised_labels), sum(len(qid_to_duplicate_qids[supervised_query_qid]) for supervised_query_qid in supervised_query_qids)","metadata":{"papermill":{"duration":55.560475,"end_time":"2021-07-27T07:43:34.375093","exception":false,"start_time":"2021-07-27T07:42:38.814618","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:46:18.231239Z","iopub.execute_input":"2021-08-11T15:46:18.231634Z","iopub.status.idle":"2021-08-11T15:47:25.392522Z","shell.execute_reply.started":"2021-08-11T15:46:18.231588Z","shell.execute_reply":"2021-08-11T15:47:25.391456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Logistic Regression","metadata":{"papermill":{"duration":0.145501,"end_time":"2021-07-27T07:43:34.640319","exception":false,"start_time":"2021-07-27T07:43:34.494818","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression(random_state=0, class_weight='balanced').fit(np.nan_to_num(df_supervised.values), supervised_labels)\nfor coef, feature in zip(clf.coef_[0], df_supervised.columns):\n    print(\"{:.4f}\".format(coef), feature)","metadata":{"papermill":{"duration":26.057846,"end_time":"2021-07-27T07:44:00.879207","exception":false,"start_time":"2021-07-27T07:43:34.821361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:47:25.394193Z","iopub.execute_input":"2021-08-11T15:47:25.394640Z","iopub.status.idle":"2021-08-11T15:48:05.382838Z","shell.execute_reply.started":"2021-08-11T15:47:25.394596Z","shell.execute_reply":"2021-08-11T15:48:05.381429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_supervised_model_logr(query_qid):\n    df_predict = parse_supervised_features_into_df(create_supervised_features([query_qid]))\n    scores = clf.predict_proba(np.nan_to_num(df_predict.values))[:,1]\n    candidate_qids = df_predict.reset_index()[\"level_1\"]  # resolve dataframe multi-index\n    results = sorted(list(zip(scores, candidate_qids)))[::-1]\n    return [x[1] for x in results][:RANKED_LIST_SIZE], [x[0] for x in results][:RANKED_LIST_SIZE]  # qid, scores","metadata":{"papermill":{"duration":0.127355,"end_time":"2021-07-27T07:44:01.15055","exception":false,"start_time":"2021-07-27T07:44:01.023195","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:05.384753Z","iopub.execute_input":"2021-08-11T15:48:05.385196Z","iopub.status.idle":"2021-08-11T15:48:05.397978Z","shell.execute_reply.started":"2021-08-11T15:48:05.385150Z","shell.execute_reply":"2021-08-11T15:48:05.395653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_supervised_model_logr(test_query_qids_list[0]))","metadata":{"papermill":{"duration":2.00388,"end_time":"2021-07-27T07:44:03.274201","exception":false,"start_time":"2021-07-27T07:44:01.270321","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:05.399579Z","iopub.execute_input":"2021-08-11T15:48:05.400106Z","iopub.status.idle":"2021-08-11T15:48:08.131696Z","shell.execute_reply.started":"2021-08-11T15:48:05.400058Z","shell.execute_reply":"2021-08-11T15:48:08.130663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_supervised_model_logr = evaluation_process(method_supervised_model_logr)","metadata":{"papermill":{"duration":1373.133818,"end_time":"2021-07-27T08:06:56.527159","exception":false,"start_time":"2021-07-27T07:44:03.393341","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:08.133518Z","iopub.execute_input":"2021-08-11T15:48:08.133970Z","iopub.status.idle":"2021-08-11T15:48:33.205958Z","shell.execute_reply.started":"2021-08-11T15:48:08.133923Z","shell.execute_reply":"2021-08-11T15:48:33.204673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LightGBM classification","metadata":{"papermill":{"duration":0.11223,"end_time":"2021-07-27T08:06:56.75422","exception":false,"start_time":"2021-07-27T08:06:56.64199","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\n\ndf_train = df_supervised.copy()\ntarget_train = np.array(supervised_labels)\neval_set = np.array([True if i < len(df_train)*0.2 else False for i in range(len(df_train))])\nlgb_train = lgb.Dataset(df_train[~eval_set], target_train[~eval_set])\nlgb_eval = lgb.Dataset(df_train[eval_set], target_train[eval_set], reference=lgb_train)\nlgb_all = lgb.Dataset(df_train, target_train)","metadata":{"_kg_hide-input":true,"papermill":{"duration":6.921785,"end_time":"2021-07-27T08:07:03.789327","exception":false,"start_time":"2021-07-27T08:06:56.867542","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:33.208601Z","iopub.execute_input":"2021-08-11T15:48:33.209054Z","iopub.status.idle":"2021-08-11T15:48:42.919156Z","shell.execute_reply.started":"2021-08-11T15:48:33.209007Z","shell.execute_reply":"2021-08-11T15:48:42.917976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n#     'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'monotone_constraints': [1]*len(df_supervised.columns),\n#     'scale_pos_weight': 0.360,\n#     'metric': {'auc'},\n#     'num_leaves': 15,\n#     'learning_rate': 0.05,\n#     'feature_fraction': 0.9,\n#     'bagging_fraction': 0.8,\n#     'bagging_freq': 5,\n    'verbose': -1,\n}\n\ngbm = lgb.train(params,\n                lgb_train,\n                num_boost_round=1000,\n                valid_sets=lgb_eval,\n                verbose_eval=-1,\n                early_stopping_rounds=10)\n\npd.DataFrame({\"feature\": df_train.columns, \"importance\": gbm.feature_importance(importance_type=\"gain\")})[:20]","metadata":{"_kg_hide-input":true,"papermill":{"duration":10.885359,"end_time":"2021-07-27T08:07:14.797532","exception":false,"start_time":"2021-07-27T08:07:03.912173","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:42.920712Z","iopub.execute_input":"2021-08-11T15:48:42.921022Z","iopub.status.idle":"2021-08-11T15:48:50.325450Z","shell.execute_reply.started":"2021-08-11T15:48:42.920993Z","shell.execute_reply":"2021-08-11T15:48:50.324329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_supervised_model_lgbm(query_qid):\n    df_predict = parse_supervised_features_into_df(create_supervised_features([query_qid]))\n    scores = gbm.predict(df_predict)\n    candidate_qids = df_predict.reset_index()[\"level_1\"]\n    results = sorted(list(zip(scores, candidate_qids)))[::-1]\n    return [x[1] for x in results][:RANKED_LIST_SIZE], [x[0] for x in results][:RANKED_LIST_SIZE]  # qid, scores","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.136729,"end_time":"2021-07-27T08:07:15.057392","exception":false,"start_time":"2021-07-27T08:07:14.920663","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:50.326958Z","iopub.execute_input":"2021-08-11T15:48:50.327271Z","iopub.status.idle":"2021-08-11T15:48:50.333794Z","shell.execute_reply.started":"2021-08-11T15:48:50.327232Z","shell.execute_reply":"2021-08-11T15:48:50.332653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_supervised_model_lgbm(test_query_qids_list[0]))","metadata":{"papermill":{"duration":1.957168,"end_time":"2021-07-27T08:07:17.138597","exception":false,"start_time":"2021-07-27T08:07:15.181429","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:50.335122Z","iopub.execute_input":"2021-08-11T15:48:50.335456Z","iopub.status.idle":"2021-08-11T15:48:52.939397Z","shell.execute_reply.started":"2021-08-11T15:48:50.335419Z","shell.execute_reply":"2021-08-11T15:48:52.938420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_supervised_model_lgbm = evaluation_process(method_supervised_model_lgbm)","metadata":{"papermill":{"duration":1355.857062,"end_time":"2021-07-27T08:29:53.179261","exception":false,"start_time":"2021-07-27T08:07:17.322199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:48:52.942830Z","iopub.execute_input":"2021-08-11T15:48:52.943163Z","iopub.status.idle":"2021-08-11T15:49:17.828520Z","shell.execute_reply.started":"2021-08-11T15:48:52.943133Z","shell.execute_reply":"2021-08-11T15:49:17.827293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation for Hand Evaluation Dataset","metadata":{"papermill":{"duration":0.124416,"end_time":"2021-07-27T08:29:53.428422","exception":false,"start_time":"2021-07-27T08:29:53.304006","status":"completed"},"tags":[]}},{"cell_type":"code","source":"method_to_ranklists_scorelists = {\n#     \"random_guess\": results_random_guess,\n    \"overlapping_root_word_count\": results_overlapping_root_word_count,\n    \"boolean\": results_boolean,\n    \"tf_idf\": results_tf_idf,\n    \"bm25\": results_bm25,\n    \"unigram\": results_unigram,\n    \"spacy_embedding_similarity\": results_spacy_embedding_similarity,\n    \"spacy_embedding_similarity_lg\": results_spacy_embedding_similarity_lg,\n    \"wordmover_distance_glovewiki50\": results_wordmover_distance_glovewiki50,\n    \"sentence_vector\": results_sentence_vector,\n    \"supervised_model_logr\": results_supervised_model_logr,\n    \"supervised_model_lgbm\": results_supervised_model_lgbm\n}\n\nimport json\n\ndef parse_ndarray(obj):  # https://stackoverflow.com/a/52604722/5894029\n    if isinstance(obj, np.ndarray):\n        return obj.tolist()\n\n# with open('method_to_ranklists_scorelists.json', 'w') as f:\n#     json.dump(method_to_ranklists_scorelists, f, indent=4, default=parse_ndarray)","metadata":{"papermill":{"duration":4.89802,"end_time":"2021-07-27T08:29:58.451197","exception":false,"start_time":"2021-07-27T08:29:53.553177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:17.829956Z","iopub.execute_input":"2021-08-11T15:49:17.830274Z","iopub.status.idle":"2021-08-11T15:49:17.836910Z","shell.execute_reply.started":"2021-08-11T15:49:17.830240Z","shell.execute_reply":"2021-08-11T15:49:17.835743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUESTIONS_TO_HANDEVAL = set(x-1 for x in [\n    332, 490, 1955, 6319, 9690, 17279, 19619, 20557, 26378, 33734, 38984, \n    49864, 57291, 89903, 116882, 126992, 131214, 144297, 159628, 201409, \n    273666, 284107, 286721, 312887, 318523, 378759, 384832, 405081, \n    405877, 423313, 464279, 480116, 533401])\nHANDEVAL_RANK_THRESHOLD = 10\n\nmap_qid_to_handeval = defaultdict(set)\n\nfor ranklists, scorelists in method_to_ranklists_scorelists.values():\n    for test_qid, ranklist in zip(test_query_qids_list, ranklists):\n        if test_qid in QUESTIONS_TO_HANDEVAL:\n            for candidate_qid in ranklist[:HANDEVAL_RANK_THRESHOLD]:\n                map_qid_to_handeval[test_qid].add(candidate_qid)\n            \nfor qid in map_qid_to_handeval:\n    map_qid_to_handeval[qid] = sorted(map_qid_to_handeval[qid])","metadata":{"papermill":{"duration":0.201874,"end_time":"2021-07-27T08:29:58.840596","exception":false,"start_time":"2021-07-27T08:29:58.638722","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:17.838540Z","iopub.execute_input":"2021-08-11T15:49:17.838853Z","iopub.status.idle":"2021-08-11T15:49:17.852487Z","shell.execute_reply.started":"2021-08-11T15:49:17.838820Z","shell.execute_reply":"2021-08-11T15:49:17.851160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe_columns = [\"test_qid\", \"test_question\", \"candidate_qid\", \"candidate_question\"]\ndataframe_entries = []\nfor qid in sorted(map_qid_to_handeval.keys()):\n    for candidate_qid in map_qid_to_handeval[qid]:\n        line_entry = [qid, qid_to_question[qid], candidate_qid, qid_to_question[candidate_qid]]\n        dataframe_entries.append(line_entry)\n        \nrandom.shuffle(dataframe_entries)\ndataframe_entries = sorted(dataframe_entries, key = lambda x: x[0])","metadata":{"papermill":{"duration":0.197046,"end_time":"2021-07-27T08:29:59.220738","exception":false,"start_time":"2021-07-27T08:29:59.023692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:17.853944Z","iopub.execute_input":"2021-08-11T15:49:17.854277Z","iopub.status.idle":"2021-08-11T15:49:17.873424Z","shell.execute_reply.started":"2021-08-11T15:49:17.854246Z","shell.execute_reply":"2021-08-11T15:49:17.872358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_handeval = pd.DataFrame(dataframe_entries, columns=dataframe_columns)\n# labeller columns\ndf_handeval[\"jh\"] = np.nan\ndf_handeval[\"hk\"] = np.nan\ndf_handeval[\"wt\"] = np.nan\n\n# df_handeval.to_csv(\"df_handeval.csv\", index=None)","metadata":{"papermill":{"duration":0.146718,"end_time":"2021-07-27T08:29:59.492089","exception":false,"start_time":"2021-07-27T08:29:59.345371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:17.875217Z","iopub.execute_input":"2021-08-11T15:49:17.875875Z","iopub.status.idle":"2021-08-11T15:49:17.890062Z","shell.execute_reply.started":"2021-08-11T15:49:17.875821Z","shell.execute_reply":"2021-08-11T15:49:17.889311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate NDCG with Hand Evaluation Dataset\n\nThis calculates NDCG from a snapshot version of `method_to_ranklists_scorelists`, and a hand annotated `df_handeval`.\n\nDue to randomness, the `method_to_ranklists_scorelists` may not be reproduced exactly.","metadata":{"papermill":{"duration":0.123582,"end_time":"2021-07-27T08:29:59.738663","exception":false,"start_time":"2021-07-27T08:29:59.615081","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_handeval = pd.read_csv(\"../input/quoraquestionpairhandannotateddataset/df_handeval.csv\")\nwith open('../input/quoraquestionpairhandannotateddataset/method_to_ranklists_scorelists.json') as f:\n    method_to_ranklists_scorelists = json.load(f)","metadata":{"papermill":{"duration":0.178553,"end_time":"2021-07-27T08:30:00.041441","exception":false,"start_time":"2021-07-27T08:29:59.862888","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:17.891155Z","iopub.execute_input":"2021-08-11T15:49:17.891612Z","iopub.status.idle":"2021-08-11T15:49:19.728302Z","shell.execute_reply.started":"2021-08-11T15:49:17.891581Z","shell.execute_reply":"2021-08-11T15:49:19.727325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\ndef calculate_dcg_at_k(r, k, method=0):\n    if method == 0:\n        logn = [1.] + [1/math.log(i,2) for i in range(2, k+1)]\n    else:\n        logn = [1/math.log(i,2) for i in range(2, k+2)]\n    \n    dcg = 0.\n    for gain,disc in zip(r[:k], logn):\n        dcg += gain*disc\n    return dcg\n\ndef calculate_ndcg_at_k(scores, ref, k=10, method=0):\n    denom = calculate_dcg_at_k(ref, k, method=method)\n    numer = calculate_dcg_at_k(scores, k, method=method)\n    if denom == 0:\n        return 0.\n    return numer/denom","metadata":{"papermill":{"duration":0.130396,"end_time":"2021-07-27T08:30:00.295341","exception":false,"start_time":"2021-07-27T08:30:00.164945","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.730073Z","iopub.execute_input":"2021-08-11T15:49:19.730536Z","iopub.status.idle":"2021-08-11T15:49:19.740911Z","shell.execute_reply.started":"2021-08-11T15:49:19.730480Z","shell.execute_reply":"2021-08-11T15:49:19.739494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_qid_to_candidate_qid_to_scores = collections.defaultdict(dict)\n\nfor _,row in df_handeval.iterrows():\n    test_qid = row[\"test_qid\"]\n    candidate_qid = row[\"candidate_qid\"]\n    score = row[\"average\"]\n    test_qid_to_candidate_qid_to_scores[test_qid][candidate_qid] = score\n    \ntest_qid_to_ideal_scores = collections.defaultdict(list)\nfor test_qid, candidate_qid_to_scores in test_qid_to_candidate_qid_to_scores.items():\n    ideal_scores = sorted(candidate_qid_to_scores.values())[::-1]\n    test_qid_to_ideal_scores[test_qid] = ideal_scores\n\nmethod_to_ndcg_score = collections.defaultdict(list)\ncount_out_of_eval = 0\n\nfor method_name, (ranklists, _) in method_to_ranklists_scorelists.items():\n    for test_qid, ranklist in zip(test_query_qids_list, ranklists):\n        if test_qid in QUESTIONS_TO_HANDEVAL:\n            scores = []\n            for candidate_qid in ranklist[:HANDEVAL_RANK_THRESHOLD]:\n                if candidate_qid not in test_qid_to_candidate_qid_to_scores[test_qid]:\n                    scores.append(1)\n                    print(method_name, len(scores))\n                    count_out_of_eval += 1\n                else:\n                    scores.append(test_qid_to_candidate_qid_to_scores[test_qid][candidate_qid])\n            ref = test_qid_to_ideal_scores[test_qid]\n            ndcg_at_k = calculate_ndcg_at_k(scores, ref)\n            method_to_ndcg_score[method_name].append(ndcg_at_k)\n\ncount_out_of_eval","metadata":{"papermill":{"duration":0.181802,"end_time":"2021-07-27T08:30:00.602248","exception":false,"start_time":"2021-07-27T08:30:00.420446","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.742697Z","iopub.execute_input":"2021-08-11T15:49:19.743195Z","iopub.status.idle":"2021-08-11T15:49:19.907838Z","shell.execute_reply.started":"2021-08-11T15:49:19.743149Z","shell.execute_reply":"2021-08-11T15:49:19.906610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if EVALUATING:\n  for method_name, scores in method_to_ndcg_score.items():\n    scores = scores[3:]  # first three are not labelled\n    print(method_name)\n    print(f\"{sum(scores)/len(scores):.5f}\")\n    print(\" \".join(f\"{x:.2f}\" for x in scores))\n    print()","metadata":{"papermill":{"duration":0.195208,"end_time":"2021-07-27T08:30:00.922944","exception":false,"start_time":"2021-07-27T08:30:00.727736","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.909629Z","iopub.execute_input":"2021-08-11T15:49:19.910066Z","iopub.status.idle":"2021-08-11T15:49:19.916592Z","shell.execute_reply.started":"2021-08-11T15:49:19.910019Z","shell.execute_reply":"2021-08-11T15:49:19.915129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Indexing and Querying of Unseen Questions\n\nThis is the Graphical User Interface that we are presenting","metadata":{"papermill":{"duration":0.126483,"end_time":"2021-07-27T08:30:01.177091","exception":false,"start_time":"2021-07-27T08:30:01.050608","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def index_unseen_question(unseen_question_text_list):\n    unseen_sentence_vectors = model_tf.encode(unseen_question_text_list, show_progress_bar=True)\n    qids_new = [time.time() for _ in unseen_question_text_list]\n\n    for qid_new, unseen_sentence_vector, unseen_question_text in zip(qids_new, unseen_sentence_vectors, unseen_question_text_list):\n        qid_to_question[qid_new] = unseen_question_text\n        \n        # compute and update word embedding\n        token_list = tokenise_then_spellcheck(unseen_question_text)\n        qid_to_vec[qid_new] = to_vec(token_list)\n        qid_to_vec2[qid_new] = to_vec2(token_list)\n\n        # update sentence embedding\n        sentence_vectors[qid_new] = unseen_sentence_vector    \n\n    # update tf-idf\n    qid_to_tokens_, token_to_qids_, tf_, df_, L_  = preprocess_vsm(qids_new)\n    for qid in qid_to_tokens_:\n        qid_to_tokens[qid] = qid_to_tokens_[qid]\n    for token in token_to_qids_:\n        token_to_qids[token].update(token_to_qids_[token])\n    for token in tf_:\n        for qid in tf_[token]:\n            tf[token][qid] += tf_[token][qid]\n    for token in df_:\n        df[token] += df_[token]\n    for qid in L_:\n        L[qid] = L_[qid]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.135531,"end_time":"2021-07-27T08:30:01.437701","exception":false,"start_time":"2021-07-27T08:30:01.30217","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.918625Z","iopub.execute_input":"2021-08-11T15:49:19.919257Z","iopub.status.idle":"2021-08-11T15:49:19.936290Z","shell.execute_reply.started":"2021-08-11T15:49:19.919209Z","shell.execute_reply":"2021-08-11T15:49:19.935001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def query_unseen_question(unseen_question_text, method):\n    qid_new = time.time()\n    qid_to_question[qid_new] = unseen_question_text\n    \n    show_sample_query_results(qid_new, *method(qid_new))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.13253,"end_time":"2021-07-27T08:30:01.693996","exception":false,"start_time":"2021-07-27T08:30:01.561466","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.937402Z","iopub.execute_input":"2021-08-11T15:49:19.937702Z","iopub.status.idle":"2021-08-11T15:49:19.952123Z","shell.execute_reply.started":"2021-08-11T15:49:19.937672Z","shell.execute_reply":"2021-08-11T15:49:19.950846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Choose your query method\n\nThe following cell is the list of methods tested in this repository.\n\nUncomment the line for the method that you want to use.","metadata":{}},{"cell_type":"code","source":"# method = method_random_guess\n# method = method_overlapping_root_word_count  # method 0\n# method = method_boolean\n# method = method_tf_idf  # method 1\n# method = method_bm25  # method 2\n# method = method_unigram  # method 3\n# method = method_spacy_embedding_similarity\n# method = method_spacy_embedding_similarity_lg  # method 4\n# method = method_wordmover_distance_glovewiki50\nmethod = method_sentence_vector  # method 5\n# method = method_supervised_model_logr  # method 6\n# method = method_supervised_model_lgbm","metadata":{"papermill":{"duration":0.130956,"end_time":"2021-07-27T08:30:01.950696","exception":false,"start_time":"2021-07-27T08:30:01.81974","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:19.953728Z","iopub.execute_input":"2021-08-11T15:49:19.954094Z","iopub.status.idle":"2021-08-11T15:49:19.968624Z","shell.execute_reply.started":"2021-08-11T15:49:19.954053Z","shell.execute_reply":"2021-08-11T15:49:19.967540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Query an unseen question\n\nWrite the question you want to query in the following cell.\n\nYou will see the top results retrieved, the score according to the retrieval method, and the retrieved question string.","metadata":{}},{"cell_type":"code","source":"query_unseen_question(\"Why are computer screens dark in color?\", method=method)","metadata":{"papermill":{"duration":0.955657,"end_time":"2021-07-27T08:30:03.031042","exception":false,"start_time":"2021-07-27T08:30:02.075385","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:51:27.119584Z","iopub.execute_input":"2021-08-11T15:51:27.119962Z","iopub.status.idle":"2021-08-11T15:51:27.221909Z","shell.execute_reply.started":"2021-08-11T15:51:27.119932Z","shell.execute_reply":"2021-08-11T15:51:27.220734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Index unseen questions\n\nNow we index two questions of a similar meaning to the queried question.","metadata":{}},{"cell_type":"code","source":"index_unseen_question([\n    \"Why are computer screens black when unpowered?\",\n    \"Why are computer screens manufactured black?\"])","metadata":{"papermill":{"duration":0.375211,"end_time":"2021-07-27T08:30:03.529238","exception":false,"start_time":"2021-07-27T08:30:03.154027","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:51:18.106609Z","iopub.execute_input":"2021-08-11T15:51:18.106969Z","iopub.status.idle":"2021-08-11T15:51:18.337525Z","shell.execute_reply.started":"2021-08-11T15:51:18.106940Z","shell.execute_reply":"2021-08-11T15:51:18.336568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Query a question related to indexed questions\n\nWe make the same query and see that it manages to retrieve the added questions at a high ranking.","metadata":{}},{"cell_type":"code","source":"query_unseen_question(\"Why are computer screens dark in color?\", method=method)","metadata":{"papermill":{"duration":1.065131,"end_time":"2021-07-27T08:30:04.71995","exception":false,"start_time":"2021-07-27T08:30:03.654819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:51:19.117080Z","iopub.execute_input":"2021-08-11T15:51:19.117472Z","iopub.status.idle":"2021-08-11T15:51:19.217441Z","shell.execute_reply.started":"2021-08-11T15:51:19.117438Z","shell.execute_reply":"2021-08-11T15:51:19.215592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Run the following cell if you want to reset the indexes.","metadata":{}},{"cell_type":"code","source":"qid_to_tokens, token_to_qids = deepcopy(qid_to_tokens_original), deepcopy(token_to_qids_original)\ntf, df, L = deepcopy(tf_original), deepcopy(df_original), deepcopy(L_original)","metadata":{"papermill":{"duration":0.132597,"end_time":"2021-07-27T08:30:04.979076","exception":false,"start_time":"2021-07-27T08:30:04.846479","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-11T15:49:20.413739Z","iopub.execute_input":"2021-08-11T15:49:20.414153Z","iopub.status.idle":"2021-08-11T15:49:51.572037Z","shell.execute_reply.started":"2021-08-11T15:49:20.414117Z","shell.execute_reply":"2021-08-11T15:49:51.571002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.128512,"end_time":"2021-07-27T08:30:05.236629","exception":false,"start_time":"2021-07-27T08:30:05.108117","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}