{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%reset -sf","metadata":{"papermill":{"duration":0.126276,"end_time":"2021-06-10T08:53:37.542069","exception":false,"start_time":"2021-06-10T08:53:37.415793","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:04.248928Z","iopub.execute_input":"2021-06-15T03:16:04.249319Z","iopub.status.idle":"2021-06-15T03:16:04.357538Z","shell.execute_reply.started":"2021-06-15T03:16:04.249286Z","shell.execute_reply":"2021-06-15T03:16:04.356787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, collections, random, itertools\n\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nrandom.seed(42)\nnp.random.seed(42)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.034468,"end_time":"2021-06-10T08:53:37.602788","exception":false,"start_time":"2021-06-10T08:53:37.56832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:04.440581Z","iopub.execute_input":"2021-06-15T03:16:04.441311Z","iopub.status.idle":"2021-06-15T03:16:04.446812Z","shell.execute_reply.started":"2021-06-15T03:16:04.441264Z","shell.execute_reply":"2021-06-15T03:16:04.446190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'): \n    for filename in filenames: print(os.path.join(dirname, filename))","metadata":{"papermill":{"duration":0.037115,"end_time":"2021-06-10T08:53:37.666113","exception":false,"start_time":"2021-06-10T08:53:37.628998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:04.690953Z","iopub.execute_input":"2021-06-15T03:16:04.691550Z","iopub.status.idle":"2021-06-15T03:16:04.701945Z","shell.execute_reply.started":"2021-06-15T03:16:04.691496Z","shell.execute_reply":"2021-06-15T03:16:04.700874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\ndf = pd.read_csv(\"/kaggle/input/quora-question-pairs/train.csv.zip\")\ndf[\"question1\"] = df[\"question1\"].astype(str)  # resolve nan\ndf[\"question2\"] = df[\"question2\"].astype(str)  # resolve nan","metadata":{"papermill":{"duration":2.67369,"end_time":"2021-06-10T08:53:40.366914","exception":false,"start_time":"2021-06-10T08:53:37.693224","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:05.036977Z","iopub.execute_input":"2021-06-15T03:16:05.037330Z","iopub.status.idle":"2021-06-15T03:16:07.298924Z","shell.execute_reply.started":"2021-06-15T03:16:05.037301Z","shell.execute_reply":"2021-06-15T03:16:07.298133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(10)","metadata":{"papermill":{"duration":0.065825,"end_time":"2021-06-10T08:53:40.459264","exception":false,"start_time":"2021-06-10T08:53:40.393439","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:07.300307Z","iopub.execute_input":"2021-06-15T03:16:07.300645Z","iopub.status.idle":"2021-06-15T03:16:07.338089Z","shell.execute_reply.started":"2021-06-15T03:16:07.300614Z","shell.execute_reply":"2021-06-15T03:16:07.337407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_SET_SIZE = 1000\nRANKED_LIST_SIZE = 100","metadata":{"papermill":{"duration":0.03432,"end_time":"2021-06-10T08:53:40.520711","exception":false,"start_time":"2021-06-10T08:53:40.486391","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:07.339340Z","iopub.execute_input":"2021-06-15T03:16:07.339712Z","iopub.status.idle":"2021-06-15T03:16:07.343320Z","shell.execute_reply.started":"2021-06-15T03:16:07.339684Z","shell.execute_reply":"2021-06-15T03:16:07.342472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Dataset","metadata":{"papermill":{"duration":0.027079,"end_time":"2021-06-10T08:53:40.575251","exception":false,"start_time":"2021-06-10T08:53:40.548172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# all questions are identified with its qid\nqid_to_question = {}\nfor qid1, qid2, question1, question2 in zip(df[\"qid1\"], df[\"qid2\"], df[\"question1\"], df[\"question2\"]):\n    qid_to_question[qid1] = question1\n    qid_to_question[qid2] = question2","metadata":{"papermill":{"duration":0.623784,"end_time":"2021-06-10T08:53:41.226547","exception":false,"start_time":"2021-06-10T08:53:40.602763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:07.344494Z","iopub.execute_input":"2021-06-15T03:16:07.344791Z","iopub.status.idle":"2021-06-15T03:16:07.917857Z","shell.execute_reply.started":"2021-06-15T03:16:07.344762Z","shell.execute_reply":"2021-06-15T03:16:07.916786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract 1000 questions for testing\ntest_query_qids = set()\n\ndf_duplicate = df[df[\"is_duplicate\"] == 1].sample(frac=1, random_state=42)\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n    if is_duplicate and qid1 not in test_query_qids:\n        test_query_qids.add(qid2)\n    if qid1 in test_query_qids and qid2 in test_query_qids:\n        assert False\n    if len(test_query_qids) == TEST_SET_SIZE:\n        break\n\ntest_query_qids_list = sorted(test_query_qids)\ntrain_query_qids_list = sorted(set(qid_to_question.keys()) - test_query_qids)\nassert test_query_qids_list[:3] == [145, 332, 400]   # to check random state fixed","metadata":{"papermill":{"duration":0.17581,"end_time":"2021-06-10T08:53:41.429779","exception":false,"start_time":"2021-06-10T08:53:41.253969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:08.826700Z","iopub.execute_input":"2021-06-15T03:16:08.827074Z","iopub.status.idle":"2021-06-15T03:16:08.971368Z","shell.execute_reply.started":"2021-06-15T03:16:08.827037Z","shell.execute_reply":"2021-06-15T03:16:08.970475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of training set\n\nqid_to_duplicate_qids = collections.defaultdict(set)\nqid_to_nonduplicate_qids = collections.defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    if not (qid1 in test_query_qids or qid2 in test_query_qids):\n        if is_duplicate:\n            qid_to_duplicate_qids[qid1].add(qid2)\n            qid_to_duplicate_qids[qid2].add(qid1)\n        else:\n            qid_to_nonduplicate_qids[qid1].add(qid2)\n            qid_to_nonduplicate_qids[qid2].add(qid1)","metadata":{"papermill":{"duration":1.665285,"end_time":"2021-06-10T08:53:43.122282","exception":false,"start_time":"2021-06-10T08:53:41.456997","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:09.225678Z","iopub.execute_input":"2021-06-15T03:16:09.226008Z","iopub.status.idle":"2021-06-15T03:16:10.861901Z","shell.execute_reply.started":"2021-06-15T03:16:09.225979Z","shell.execute_reply":"2021-06-15T03:16:10.860982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# complete graph of duplicate relationships\n\nqid_to_duplicate_qids_complete = collections.defaultdict(set)\nqid_to_qid_group_leader = {}\nqid_group_leader_to_duplicate_qid_group = collections.defaultdict(set)\n\nvisited_qids = set()\nfor train_qid in train_query_qids_list:\n    if train_qid in visited_qids:\n        continue\n    current_qids_group = set([train_qid])\n    qid_to_qid_group_leader[train_qid] = train_qid\n    stack = [train_qid]\n    \n    while stack:\n        cur_qid = stack.pop()\n        for nex_qid in qid_to_duplicate_qids[cur_qid]:\n            if nex_qid in current_qids_group:\n                continue\n            qid_to_qid_group_leader[nex_qid] = train_qid\n            stack.append(nex_qid)\n            current_qids_group.add(nex_qid)\n\n    # complete the graph\n    for qid1, qid2 in itertools.combinations(current_qids_group, r=2):\n        qid_to_duplicate_qids_complete[qid1].add(qid2)\n        qid_to_duplicate_qids_complete[qid2].add(qid1)\n    qid_group_leader_to_duplicate_qid_group[train_qid] = current_qids_group\n    visited_qids.update(current_qids_group)","metadata":{"papermill":{"duration":3.450351,"end_time":"2021-06-10T08:53:46.610851","exception":false,"start_time":"2021-06-10T08:53:43.1605","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:10.863518Z","iopub.execute_input":"2021-06-15T03:16:10.864097Z","iopub.status.idle":"2021-06-15T03:16:14.440420Z","shell.execute_reply.started":"2021-06-15T03:16:10.864047Z","shell.execute_reply":"2021-06-15T03:16:14.439375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of the test set\n\ntest_qid_to_duplicate_qids = collections.defaultdict(set)\ntest_qid_to_duplicate_qids_complete = collections.defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n    if qid2 in test_query_qids:\n        qid1, qid2 = qid2, qid1\n    if qid1 in test_query_qids:\n        if qid2 in test_query_qids:\n            continue\n        test_qid_to_duplicate_qids[qid1].add(qid2)\n        test_qid_to_duplicate_qids_complete[qid1].add(qid2)\n        for train_qid in qid_group_leader_to_duplicate_qid_group[qid_to_qid_group_leader[qid2]]:\n            test_qid_to_duplicate_qids_complete[qid1].add(train_qid)","metadata":{"papermill":{"duration":0.199531,"end_time":"2021-06-10T08:53:46.837634","exception":false,"start_time":"2021-06-10T08:53:46.638103","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:14.442146Z","iopub.execute_input":"2021-06-15T03:16:14.442467Z","iopub.status.idle":"2021-06-15T03:16:14.612911Z","shell.execute_reply.started":"2021-06-15T03:16:14.442437Z","shell.execute_reply":"2021-06-15T03:16:14.611907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count inconsistencies in dataset\n\ncnt = 0\nfor qid1, qid_list in qid_to_duplicate_qids_complete.items():\n    for qid2 in qid_list:\n        if qid1 in qid_to_nonduplicate_qids[qid2]:\n            # print(qid1, qid2)\n            cnt += 1\nprint(cnt)","metadata":{"papermill":{"duration":0.236131,"end_time":"2021-06-10T08:53:47.101074","exception":false,"start_time":"2021-06-10T08:53:46.864943","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:14.614458Z","iopub.execute_input":"2021-06-15T03:16:14.614857Z","iopub.status.idle":"2021-06-15T03:16:14.817217Z","shell.execute_reply.started":"2021-06-15T03:16:14.614816Z","shell.execute_reply":"2021-06-15T03:16:14.816208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_mask = (df[\"qid1\"].isin(test_query_qids)) | (df[\"qid2\"].isin(test_query_qids))\ntrain_df = df[~test_mask].copy()\ntest_df = df[test_mask].copy()","metadata":{"papermill":{"duration":0.117128,"end_time":"2021-06-10T08:53:47.247094","exception":false,"start_time":"2021-06-10T08:53:47.129966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:16.590263Z","iopub.execute_input":"2021-06-15T03:16:16.590603Z","iopub.status.idle":"2021-06-15T03:16:16.677015Z","shell.execute_reply.started":"2021-06-15T03:16:16.590574Z","shell.execute_reply":"2021-06-15T03:16:16.676133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean up\ndel qid_to_qid_group_leader, qid_group_leader_to_duplicate_qid_group\ndel cnt\ndel test_query_qids   # not sorted, use test_query_qids_list\ndel df                # all data you can train on is in train_df\n\n# enable use of complete graphs\ntest_qid_to_duplicate_qids = test_qid_to_duplicate_qids_complete\nqid_to_duplicate_qids = qid_to_duplicate_qids_complete","metadata":{"papermill":{"duration":0.138462,"end_time":"2021-06-10T08:53:47.41318","exception":false,"start_time":"2021-06-10T08:53:47.274718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:16.888928Z","iopub.execute_input":"2021-06-15T03:16:16.889324Z","iopub.status.idle":"2021-06-15T03:16:16.998686Z","shell.execute_reply.started":"2021-06-15T03:16:16.889290Z","shell.execute_reply":"2021-06-15T03:16:16.997541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Metrics","metadata":{"execution":{"iopub.execute_input":"2021-06-08T11:21:27.301993Z","iopub.status.busy":"2021-06-08T11:21:27.301617Z","iopub.status.idle":"2021-06-08T11:21:27.305697Z","shell.execute_reply":"2021-06-08T11:21:27.304715Z","shell.execute_reply.started":"2021-06-08T11:21:27.301962Z"},"papermill":{"duration":0.027016,"end_time":"2021-06-10T08:53:47.467677","exception":false,"start_time":"2021-06-10T08:53:47.440661","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def method_random_guess(test_qid):\n    return random.choices(train_query_qids_list, k=RANKED_LIST_SIZE)\n\nmethod_random_guess_ranklists = [method_random_guess(test_qid) for test_qid in test_query_qids_list] # 1000 x 100 (the ranked list of similar qn for each of the 1000 test qns)\nprint(len(method_random_guess_ranklists))\nprint(len(method_random_guess_ranklists[0]))","metadata":{"papermill":{"duration":0.091842,"end_time":"2021-06-10T08:53:47.586952","exception":false,"start_time":"2021-06-10T08:53:47.49511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:17.926833Z","iopub.execute_input":"2021-06-15T03:16:17.927183Z","iopub.status.idle":"2021-06-15T03:16:17.986601Z","shell.execute_reply.started":"2021-06-15T03:16:17.927138Z","shell.execute_reply":"2021-06-15T03:16:17.985863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sample_results(test_qid, method_ranklist, method_scores=[0]*RANKED_LIST_SIZE, num_to_show=8):\n    print(\"Query: {}\".format(qid_to_question[test_qid]))\n    for rank, (score, result_qid) in enumerate(zip(method_scores, method_ranklist[:num_to_show]), start=1):\n        relevance = \"Registered\" if result_qid in test_qid_to_duplicate_qids[test_qid] else \"Unregistered\"\n        print(\"Rank {} - Score {} - {}:  \\t{}\".format(rank, score, relevance, qid_to_question[result_qid]))\n\nsample_results(test_query_qids_list[0], method_random_guess(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.038642,"end_time":"2021-06-10T08:53:47.653291","exception":false,"start_time":"2021-06-10T08:53:47.614649","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:18.251690Z","iopub.execute_input":"2021-06-15T03:16:18.252289Z","iopub.status.idle":"2021-06-15T03:16:18.260837Z","shell.execute_reply.started":"2021-06-15T03:16:18.252253Z","shell.execute_reply":"2021-06-15T03:16:18.259873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_basic_methods(method_ranklists):\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n    ranks = [RANKED_LIST_SIZE]*TEST_SET_SIZE # init rank of first duplicate found for each test qn as 100 (signifying that first duplicate is out of ranked list range)\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n        for rank, result_qid in enumerate(ranklist, start=1):\n            if result_qid in test_qid_to_duplicate_qids[test_qid]: # the first duplicate identified, if any; may not be entered if all duplicates are out of range\n                ranks[i] = rank # rank of the first duplicate identified\n                break\n                \n    plt.figure(figsize=(14,4))\n    plt.title(\"Rank of duplicate questions\")\n    plt.hist(ranks, bins=np.arange(RANKED_LIST_SIZE+1))\n    plt.xlabel(\"Rank\")\n    plt.ylabel(\"Frequency\")\n    plt.show()\n    \n    print(\"Median rank: {}\".format(np.median(ranks)))\n    print(\"Bottom 25% rank: {}\".format(np.percentile(ranks, 75)))\n    print(\"Bottom 10% rank: {}\".format(np.percentile(ranks, 90)))\n    print(\"Logarithmic average rank: {:.2f}\".format(np.exp(np.mean(np.log(ranks)))))\n    print(\"Proportion out of result: {:.3f}\".format(ranks.count(RANKED_LIST_SIZE)/len(ranks)))","metadata":{"papermill":{"duration":0.041196,"end_time":"2021-06-10T08:53:47.722748","exception":false,"start_time":"2021-06-10T08:53:47.681552","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:18.802346Z","iopub.execute_input":"2021-06-15T03:16:18.802736Z","iopub.status.idle":"2021-06-15T03:16:18.813595Z","shell.execute_reply.started":"2021-06-15T03:16:18.802703Z","shell.execute_reply":"2021-06-15T03:16:18.812565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluation_with_basic_methods(method_random_guess_ranklists)","metadata":{"papermill":{"duration":0.442242,"end_time":"2021-06-10T08:53:48.1946","exception":false,"start_time":"2021-06-10T08:53:47.752358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:21.250887Z","iopub.execute_input":"2021-06-15T03:16:21.251266Z","iopub.status.idle":"2021-06-15T03:16:21.655758Z","shell.execute_reply.started":"2021-06-15T03:16:21.251226Z","shell.execute_reply":"2021-06-15T03:16:21.654764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def evaluation_with_auc(method_ranklists, k=10):\n#     # df version for more complicated computations if needed\n#     assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE) # method_ranklists size is (1000,100)\n# #     ranks = [RANKED_LIST_SIZE]*TEST_SET_SIZE\n#     auc_df = pd.DataFrame({\"rank\":[i for i in range(1,k+1)], \"count\":[0 for i in range(1,k+1)]})\n    \n#     ## Identify duplicates among top K ranks for each test\n#     for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)): # iter over 1->1000 tests\n#         # construct df to hold results for top k results\n#         df = pd.DataFrame({\"rank\":[i for i in range(1,k+1)], \"result_qid\":ranklist[:k]})\n#         df[\"is_duplicate\"] = df[\"result_qid\"].apply(lambda result_qid: 1 if result_qid in test_qid_to_duplicate_qids[test_qid] else 0)\n#         df = df.assign(is_duplicate=[random.choice([0,1]) for i in range(1,k+1)]) # uncomment this to test if it works\n#         auc_df[\"count\"] += df[\"is_duplicate\"]\n    \n#     ## Calculate AUC\n#     print(auc_df)\n#     auc = sum(auc_df[\"count\"])\n#     best_auc_at_k = TEST_SET_SIZE * k\n#     return auc/best_auc_at_k\n    \n# evaluation_with_auc(method_random_guess_ranklists)","metadata":{"papermill":{"duration":0.03825,"end_time":"2021-06-10T08:53:48.26178","exception":false,"start_time":"2021-06-10T08:53:48.22353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:24.490952Z","iopub.execute_input":"2021-06-15T03:16:24.491347Z","iopub.status.idle":"2021-06-15T03:16:24.495614Z","shell.execute_reply.started":"2021-06-15T03:16:24.491310Z","shell.execute_reply":"2021-06-15T03:16:24.494590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_auc(method_ranklists, k=10, weights=None):\n    # Comparing models with auc only makes sense if k is kept constant between comparisons\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE) # method_ranklists size is (1000,100)\n    \n    ## Init count and weights\n    counts = np.array([0 for i in range(k)]) # init counts at each of the top k ranks; each count will increase if a duplicate is found at that rank\n    if weights == None:\n        weights = [1 for i in range(k)] # default weights is 1 for each rank; alternatively can weigh higher ranks more by inputting weights list\n    weights = np.array(weights) \n    \n    ## Identify duplicates among top K ranks for each test\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)): # iter over 1->1000 tests\n        topk = ranklist[:k]\n        is_duplicate = np.array([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in topk]) # 1 if result is a duplicate\n#         is_duplicate = np.array([random.choice([0,1]) for result_qid in topk]) # TEST LINE: uncomment this to test if func works; expect auc=0.5\n        counts += is_duplicate # increment the counts\n        # proceed to next test qn\n    \n    ## Calculate AUC\n    counts *= weights # akin to scaling the breadth of each hist bar by a corresponding weight factor\n    auc = sum(counts)\n    best_auc_at_k = TEST_SET_SIZE * sum(weights) # assumes for each test qn: each of the top k has a duplicate found\n    return auc/best_auc_at_k # between [0,1], 1 is perfect\n\nprint(\"Tests (rmb to uncomment TEST LINE before running test)\")\nprint(evaluation_with_auc(method_random_guess_ranklists))\nprint(evaluation_with_auc(method_random_guess_ranklists, weights = [10,9,8,7,6,5,4,3,2,1]))","metadata":{"papermill":{"duration":0.119059,"end_time":"2021-06-10T08:53:48.410326","exception":false,"start_time":"2021-06-10T08:53:48.291267","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:24.891827Z","iopub.execute_input":"2021-06-15T03:16:24.892236Z","iopub.status.idle":"2021-06-15T03:16:24.980414Z","shell.execute_reply.started":"2021-06-15T03:16:24.892197Z","shell.execute_reply":"2021-06-15T03:16:24.979297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def single_r_precision(test_qid, test_qid_to_duplicate_qids_dict, ranklist):\n    # use this to check a single test query\n    num_duplicate = len(test_qid_to_duplicate_qids[test_qid]) # this dict needs to be updated when train:test set separation is updated\n    if num_duplicate == 0:\n        return 0, 0, 0\n    top_r = ranklist[:num_duplicate]\n    num_duplicates_in_top_r = sum([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in top_r])\n    r_precision = num_duplicates_in_top_r/num_duplicate\n    return num_duplicate, num_duplicates_in_top_r, r_precision\n\ndef evaluation_with_r_precision(method_ranklists, report_k=0):\n    assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE) # method_ranklists size is (1000,100)\n    total_num_duplicates = np.array([0 for i in range(TEST_SET_SIZE)])\n    r_precision = np.array([0 for i in range(TEST_SET_SIZE)])\n    \n    ## Iter over 1->1000 tests\n    for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)): # iter over 1->1000 tests\n        total_num_duplicates[i], num_duplicates_in_top_r, r_precision[i] = single_r_precision(test_qid, test_qid_to_duplicate_qids, ranklist)\n    \n    # note: if want do error analysis, intervene here to find test cases with low r precision\n    if report_k > 0:\n        k_lowest_r_precision_idx = np.argpartition(r_precision, k)[:k]\n        k_lowest_r_precision_test_qids = np.array(test_query_qids_list)[k_lowest_r_precision_idx]\n    \n    ## Calculate metrics\n    avg_r_precision = r_precision.mean()\n    weighted_avg_r_precision = np.multiply(r_precision, total_num_duplicates).sum() / total_num_duplicates.sum()\n    if not report_k: return avg_r_precision, weighted_avg_r_precision\n    else:\n        return avg_r_precision, weighted_avg_r_precision, k_lowest_r_precision_test_qids\n       \nevaluation_with_r_precision(method_random_guess_ranklists)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-15T03:16:26.961403Z","iopub.execute_input":"2021-06-15T03:16:26.961747Z","iopub.status.idle":"2021-06-15T03:16:27.011252Z","shell.execute_reply.started":"2021-06-15T03:16:26.961718Z","shell.execute_reply":"2021-06-15T03:16:27.010073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyse_qid(test_qid, method_ranklists=None):\n    qn = qid_to_question[test_qid]\n    dup_qids = test_qid_to_duplicate_qids[test_qid]\n    dup_qns = [qid_to_question[qid] for qid in dup_qids]\n    \n    return_dt = {\"test_qn\": qn, \"dup_qids\": dup_qids, \"dup_qns\":dup_qns}\n    if method_ranklists: \n        idx_ranklist = test_query_qids_list.index(test_qid)\n        ranklist = method_ranklists[idx_ranklist]\n        r_prec = single_r_precision(test_qid, test_qid_to_duplicate_qids, ranklist)\n        return_dt[\"r_prec\" ] = r_prec\n    \n    return return_dt","metadata":{"execution":{"iopub.status.busy":"2021-06-15T03:16:29.726682Z","iopub.execute_input":"2021-06-15T03:16:29.727025Z","iopub.status.idle":"2021-06-15T03:16:29.732788Z","shell.execute_reply.started":"2021-06-15T03:16:29.726996Z","shell.execute_reply":"2021-06-15T03:16:29.731711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lists available for all\n- `train_query_qids_list`\n- `test_query_qids_list`\n\nObjects you are allowed to train with\n- `qid_to_question`  (only the questions NOT in `test_query_qids_list`)\n- `qid_to_duplicate_qids`\n- `qid_to_nonduplicate_qids`\n- `train_df`\n\nObjects you are NOT allowed to train with\n- `qid_to_question`  (only the questions in `test_query_qids_list`)\n- `test_qid_to_duplicate_qids`\n- `test_df`","metadata":{"papermill":{"duration":0.029772,"end_time":"2021-06-10T08:53:48.539089","exception":false,"start_time":"2021-06-10T08:53:48.509317","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Baseline Method\nOrder by the number of overlapping root words","metadata":{"papermill":{"duration":0.029704,"end_time":"2021-06-10T08:53:48.598868","exception":false,"start_time":"2021-06-10T08:53:48.569164","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstopword_set = set(stopwords.words())\nstopword_set.add(\"?\")\n\ndef tokenise(sentence):\n    tokens = word_tokenize(sentence.lower())\n    return set(word for word in tokens if not word in stopword_set)","metadata":{"papermill":{"duration":2.02824,"end_time":"2021-06-10T08:53:50.657158","exception":false,"start_time":"2021-06-10T08:53:48.628918","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:34.392740Z","iopub.execute_input":"2021-06-15T03:16:34.393087Z","iopub.status.idle":"2021-06-15T03:16:34.406120Z","shell.execute_reply.started":"2021-06-15T03:16:34.393057Z","shell.execute_reply":"2021-06-15T03:16:34.405038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(train_query_qids_list):        \n    token_to_qids = collections.defaultdict(list)\n    qid_processed = set()\n    for qid in tqdm.tqdm(train_query_qids_list):\n        if qid in qid_processed:\n            continue\n        qid_processed.add(qid)\n        for token in tokenise(qid_to_question[qid]):\n            token_to_qids[token].append(qid)\n    \n    return token_to_qids","metadata":{"papermill":{"duration":0.038416,"end_time":"2021-06-10T08:53:50.7258","exception":false,"start_time":"2021-06-10T08:53:50.687384","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:38.930531Z","iopub.execute_input":"2021-06-15T03:16:38.930872Z","iopub.status.idle":"2021-06-15T03:16:38.936313Z","shell.execute_reply.started":"2021-06-15T03:16:38.930840Z","shell.execute_reply":"2021-06-15T03:16:38.935297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"token_to_qids = preprocess(train_query_qids_list)","metadata":{"papermill":{"duration":107.018472,"end_time":"2021-06-10T08:55:37.7742","exception":false,"start_time":"2021-06-10T08:53:50.755728","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:16:41.082387Z","iopub.execute_input":"2021-06-15T03:16:41.082948Z","iopub.status.idle":"2021-06-15T03:18:28.150222Z","shell.execute_reply.started":"2021-06-15T03:16:41.082900Z","shell.execute_reply":"2021-06-15T03:18:28.149242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_overlapping_root_word_count(query_qid):\n    query_tokens = tokenise(qid_to_question[query_qid])\n    counter = collections.Counter()\n    \n    for dummy_qid in random.choices(train_query_qids_list, k=RANKED_LIST_SIZE):\n        counter[dummy_qid] = 0.01\n    \n    for query_token in query_tokens:\n        counter += collections.Counter(token_to_qids[query_token])\n    \n    query_results = list(counter.items())\n    random.shuffle(query_results)  # so that qids are not ordered\n    query_results = sorted(query_results, key=lambda x:x[1], reverse=True)[:RANKED_LIST_SIZE]\n\n    return [x[0] for x in query_results], [x[1] for x in query_results]","metadata":{"papermill":{"duration":0.352361,"end_time":"2021-06-10T08:55:38.466409","exception":false,"start_time":"2021-06-10T08:55:38.114048","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:18:28.151537Z","iopub.execute_input":"2021-06-15T03:18:28.151813Z","iopub.status.idle":"2021-06-15T03:18:28.159563Z","shell.execute_reply.started":"2021-06-15T03:18:28.151787Z","shell.execute_reply":"2021-06-15T03:18:28.158652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_results(test_query_qids_list[0], *method_overlapping_root_word_count(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.372296,"end_time":"2021-06-10T08:55:39.195955","exception":false,"start_time":"2021-06-10T08:55:38.823659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:18:28.161103Z","iopub.execute_input":"2021-06-15T03:18:28.161437Z","iopub.status.idle":"2021-06-15T03:18:28.186380Z","shell.execute_reply.started":"2021-06-15T03:18:28.161406Z","shell.execute_reply":"2021-06-15T03:18:28.185518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"method_overlapping_root_word_count_ranklists = [\n    method_overlapping_root_word_count(test_qid)[0] for test_qid in tqdm.tqdm(test_query_qids_list)]","metadata":{"papermill":{"duration":53.925088,"end_time":"2021-06-10T08:56:33.462264","exception":false,"start_time":"2021-06-10T08:55:39.537176","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:18:28.187689Z","iopub.execute_input":"2021-06-15T03:18:28.187957Z","iopub.status.idle":"2021-06-15T03:19:21.091427Z","shell.execute_reply.started":"2021-06-15T03:18:28.187932Z","shell.execute_reply":"2021-06-15T03:19:21.090208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluation_with_basic_methods(method_overlapping_root_word_count_ranklists)","metadata":{"papermill":{"duration":0.799682,"end_time":"2021-06-10T08:56:34.713248","exception":false,"start_time":"2021-06-10T08:56:33.913566","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:19:21.092895Z","iopub.execute_input":"2021-06-15T03:19:21.093244Z","iopub.status.idle":"2021-06-15T03:19:21.468732Z","shell.execute_reply.started":"2021-06-15T03:19:21.093205Z","shell.execute_reply":"2021-06-15T03:19:21.467935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k = 2\nauc = evaluation_with_auc(method_overlapping_root_word_count_ranklists, k=k, weights = [1 for i in range(k)])\nprint(f\"{auc:.2%} of top {k} results are duplicates\")\n\nk = 9\nauc = evaluation_with_auc(method_overlapping_root_word_count_ranklists, k=k, weights = [1 for i in range(k)])\nprint(f\"{auc:.2%} of top {k} results are duplicates\")\n\nk = 51\nauc = evaluation_with_auc(method_overlapping_root_word_count_ranklists, k=k, weights = [1 for i in range(k)])\nprint(f\"{auc:.2%} of top {k} results are duplicates\")","metadata":{"papermill":{"duration":0.619803,"end_time":"2021-06-10T08:56:35.764533","exception":false,"start_time":"2021-06-10T08:56:35.14473","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:19:21.470134Z","iopub.execute_input":"2021-06-15T03:19:21.470512Z","iopub.status.idle":"2021-06-15T03:19:21.597113Z","shell.execute_reply.started":"2021-06-15T03:19:21.470481Z","shell.execute_reply":"2021-06-15T03:19:21.596068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_rp, weighted_avg_rp, k_lowest_r_precision_test_qids = evaluation_with_r_precision(method_overlapping_root_word_count_ranklists, report_k = 10) \nprint(f\"Average R-Precision = {avg_rp:.2%}; Weighted Average R-Precision by proportion of duplicates = {weighted_avg_rp:.2%}\") \nif avg_rp > weighted_avg_rp: print(\"A higher average R-Precisions suggests that there are many test queries with high R-Precision but there are some test queries with high number of duplicates that model is not effective with.\")\n","metadata":{"papermill":{"duration":0.42572,"end_time":"2021-06-10T08:56:36.621289","exception":false,"start_time":"2021-06-10T08:56:36.195569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-15T03:19:21.598232Z","iopub.execute_input":"2021-06-15T03:19:21.598492Z","iopub.status.idle":"2021-06-15T03:19:21.634948Z","shell.execute_reply.started":"2021-06-15T03:19:21.598466Z","shell.execute_reply":"2021-06-15T03:19:21.634005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View the questions for each test qid\n[qid_to_question[qid] for qid in k_lowest_r_precision_test_qids]","metadata":{"execution":{"iopub.status.busy":"2021-06-15T03:19:21.636916Z","iopub.execute_input":"2021-06-15T03:19:21.637271Z","iopub.status.idle":"2021-06-15T03:19:21.649217Z","shell.execute_reply.started":"2021-06-15T03:19:21.637235Z","shell.execute_reply":"2021-06-15T03:19:21.648376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example: pick a qid to analyse\nqid_to_analyse = k_lowest_r_precision_test_qids[0]\nanalyse_qid(qid_to_analyse, method_ranklists=method_overlapping_root_word_count_ranklists)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T03:19:21.650392Z","iopub.execute_input":"2021-06-15T03:19:21.650699Z","iopub.status.idle":"2021-06-15T03:19:21.663278Z","shell.execute_reply.started":"2021-06-15T03:19:21.650662Z","shell.execute_reply":"2021-06-15T03:19:21.662247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}