{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%reset -sf","metadata":{"papermill":{"duration":0.126276,"end_time":"2021-06-10T08:53:37.542069","exception":false,"start_time":"2021-06-10T08:53:37.415793","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:36:51.6168Z","iopub.execute_input":"2021-07-26T18:36:51.617227Z","iopub.status.idle":"2021-07-26T18:36:53.018396Z","shell.execute_reply.started":"2021-07-26T18:36:51.617189Z","shell.execute_reply":"2021-07-26T18:36:53.017523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook covers\n- Dataset Preparation (train-test split)\n- TF-IDF indexes \n- Evaluation algorithm\n- Evaluation procedure with the test set\n- Update indexes with unseen questions\n- Query with unseen questions\n\nThe following process is done on another notebook\n- Spellcheck and SpaCy tokenisation for the training set \n- SentenceTransformer computation of vectors\n- Downloading of the SpaCy and GenSim models","metadata":{}},{"cell_type":"code","source":"# notebook hyperparameters\nTEST_SET_SIZE = 1000\nRANKED_LIST_SIZE = 100\nRANDOM_STATE = 42\nEVALUATING = True # make False if you want to run query quickly","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:36:53.021819Z","iopub.execute_input":"2021-07-26T18:36:53.022339Z","iopub.status.idle":"2021-07-26T18:36:53.026754Z","shell.execute_reply.started":"2021-07-26T18:36:53.022286Z","shell.execute_reply":"2021-07-26T18:36:53.025668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, collections, random, itertools, functools, time, json\n\nfrom collections import defaultdict, Counter\nfrom math import log\n\nimport tqdm.notebook as tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nrandom.seed(RANDOM_STATE)\nnp.random.seed(RANDOM_STATE)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.034468,"end_time":"2021-06-10T08:53:37.602788","exception":false,"start_time":"2021-06-10T08:53:37.56832","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:36:53.028467Z","iopub.execute_input":"2021-07-26T18:36:53.028756Z","iopub.status.idle":"2021-07-26T18:36:53.039668Z","shell.execute_reply.started":"2021-07-26T18:36:53.02873Z","shell.execute_reply":"2021-07-26T18:36:53.038835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\ndf = pd.read_csv(\"/kaggle/input/quora-question-pairs/train.csv.zip\")\ndf[\"question1\"] = df[\"question1\"].astype(str) # resolve nan\ndf[\"question2\"] = df[\"question2\"].astype(str)\ndf[\"qid1\"] -= 1 # start index from zero\ndf[\"qid2\"] -= 1","metadata":{"papermill":{"duration":2.67369,"end_time":"2021-06-10T08:53:40.366914","exception":false,"start_time":"2021-06-10T08:53:37.693224","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-26T18:36:53.041143Z","iopub.execute_input":"2021-07-26T18:36:53.041404Z","iopub.status.idle":"2021-07-26T18:36:54.94188Z","shell.execute_reply.started":"2021-07-26T18:36:53.041378Z","shell.execute_reply":"2021-07-26T18:36:54.940919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(10)","metadata":{"papermill":{"duration":0.065825,"end_time":"2021-06-10T08:53:40.459264","exception":false,"start_time":"2021-06-10T08:53:40.393439","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-26T18:36:54.94331Z","iopub.execute_input":"2021-07-26T18:36:54.943585Z","iopub.status.idle":"2021-07-26T18:36:54.96873Z","shell.execute_reply.started":"2021-07-26T18:36:54.943558Z","shell.execute_reply":"2021-07-26T18:36:54.967786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Dataset","metadata":{"papermill":{"duration":0.027079,"end_time":"2021-06-10T08:53:40.575251","exception":false,"start_time":"2021-06-10T08:53:40.548172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# all questions are identified with its qid\nqid_to_question = {}\nfor qid1, qid2, question1, question2 in zip(df[\"qid1\"], df[\"qid2\"], df[\"question1\"], df[\"question2\"]):\n qid_to_question[qid1] = question1\n qid_to_question[qid2] = question2","metadata":{"papermill":{"duration":0.623784,"end_time":"2021-06-10T08:53:41.226547","exception":false,"start_time":"2021-06-10T08:53:40.602763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-26T18:36:54.970028Z","iopub.execute_input":"2021-07-26T18:36:54.970496Z","iopub.status.idle":"2021-07-26T18:36:55.579003Z","shell.execute_reply.started":"2021-07-26T18:36:54.970465Z","shell.execute_reply":"2021-07-26T18:36:55.578213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract 1000 questions for testing\ntest_query_qids = set()\n\ndf_duplicate = df[df[\"is_duplicate\"] == 1].sample(frac=1, random_state=RANDOM_STATE)\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n if is_duplicate and qid1 not in test_query_qids and len(test_query_qids) < TEST_SET_SIZE:\n test_query_qids.add(qid2)\n if qid1 in test_query_qids and qid2 in test_query_qids:\n # to guarantee that there is a duplicate question in the training set\n test_query_qids.remove(qid1)\n test_query_qids.remove(qid2)\nassert len(test_query_qids) == TEST_SET_SIZE # if fail, change random_state\n\ntest_query_qids_list = sorted(test_query_qids)\ntrain_query_qids_list = sorted(set(qid_to_question.keys()) - test_query_qids)\nassert test_query_qids_list[:3] == [331, 489, 501] # to check random state fixed","metadata":{"papermill":{"duration":0.17581,"end_time":"2021-06-10T08:53:41.429779","exception":false,"start_time":"2021-06-10T08:53:41.253969","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:36:55.580698Z","iopub.execute_input":"2021-07-26T18:36:55.581447Z","iopub.status.idle":"2021-07-26T18:36:55.825869Z","shell.execute_reply.started":"2021-07-26T18:36:55.581398Z","shell.execute_reply":"2021-07-26T18:36:55.825135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # uncomment this to test only limited queries\nif not EVALUATING:\n test_query_qids_list = test_query_qids_list[:10]\n TEST_SET_SIZE = 10","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:36:55.828874Z","iopub.execute_input":"2021-07-26T18:36:55.829525Z","iopub.status.idle":"2021-07-26T18:36:55.834283Z","shell.execute_reply.started":"2021-07-26T18:36:55.829476Z","shell.execute_reply":"2021-07-26T18:36:55.833408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of training set\n\nqid_to_duplicate_qids = defaultdict(set)\nqid_to_nonduplicate_qids = defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n if not (qid1 in test_query_qids or qid2 in test_query_qids):\n if is_duplicate:\n qid_to_duplicate_qids[qid1].add(qid2)\n qid_to_duplicate_qids[qid2].add(qid1)\n else:\n qid_to_nonduplicate_qids[qid1].add(qid2)\n qid_to_nonduplicate_qids[qid2].add(qid1)","metadata":{"papermill":{"duration":1.665285,"end_time":"2021-06-10T08:53:43.122282","exception":false,"start_time":"2021-06-10T08:53:41.456997","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:36:55.836442Z","iopub.execute_input":"2021-07-26T18:36:55.837089Z","iopub.status.idle":"2021-07-26T18:36:57.692884Z","shell.execute_reply.started":"2021-07-26T18:36:55.837016Z","shell.execute_reply":"2021-07-26T18:36:57.692099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# complete graph of duplicate relationships\n\nqid_to_duplicate_qids_complete = defaultdict(set)\nqid_to_qid_group_leader = {}\nqid_group_leader_to_duplicate_qid_group = defaultdict(set)\n\nvisited_qids = set()\nfor train_qid in train_query_qids_list:\n if train_qid in visited_qids:\n continue\n current_qids_group = set([train_qid])\n qid_to_qid_group_leader[train_qid] = train_qid\n stack = [train_qid]\n \n while stack:\n cur_qid = stack.pop()\n for nex_qid in qid_to_duplicate_qids[cur_qid]:\n if nex_qid in current_qids_group:\n continue\n qid_to_qid_group_leader[nex_qid] = train_qid\n stack.append(nex_qid)\n current_qids_group.add(nex_qid)\n\n # complete the graph\n for qid1, qid2 in itertools.combinations(current_qids_group, r=2):\n qid_to_duplicate_qids_complete[qid1].add(qid2)\n qid_to_duplicate_qids_complete[qid2].add(qid1)\n qid_group_leader_to_duplicate_qid_group[train_qid] = current_qids_group\n visited_qids.update(current_qids_group)","metadata":{"papermill":{"duration":3.450351,"end_time":"2021-06-10T08:53:46.610851","exception":false,"start_time":"2021-06-10T08:53:43.1605","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:36:57.694163Z","iopub.execute_input":"2021-07-26T18:36:57.694494Z","iopub.status.idle":"2021-07-26T18:37:01.702549Z","shell.execute_reply.started":"2021-07-26T18:36:57.694461Z","shell.execute_reply":"2021-07-26T18:37:01.701434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract duplicate relationship of the test set\n\ntest_qid_to_duplicate_qids = defaultdict(set)\ntest_qid_to_duplicate_qids_complete = defaultdict(set)\n\nfor qid1, qid2, is_duplicate in zip(df_duplicate[\"qid1\"], df_duplicate[\"qid2\"], df_duplicate[\"is_duplicate\"]):\n if qid2 in test_query_qids:\n qid1, qid2 = qid2, qid1\n if qid1 in test_query_qids:\n if qid2 in test_query_qids:\n continue\n test_qid_to_duplicate_qids[qid1].add(qid2)\n test_qid_to_duplicate_qids_complete[qid1].add(qid2)\n for train_qid in qid_group_leader_to_duplicate_qid_group[qid_to_qid_group_leader[qid2]]:\n test_qid_to_duplicate_qids_complete[qid1].add(train_qid)","metadata":{"papermill":{"duration":0.199531,"end_time":"2021-06-10T08:53:46.837634","exception":false,"start_time":"2021-06-10T08:53:46.638103","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:01.705205Z","iopub.execute_input":"2021-07-26T18:37:01.705813Z","iopub.status.idle":"2021-07-26T18:37:01.863451Z","shell.execute_reply.started":"2021-07-26T18:37:01.705764Z","shell.execute_reply":"2021-07-26T18:37:01.862467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count inconsistencies in dataset\n\ncnt = 0\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n if not is_duplicate and qid1 not in test_query_qids and qid2 not in test_query_qids:\n if qid_to_qid_group_leader[qid1] == qid_to_qid_group_leader[qid2]:\n cnt += 1\nprint(\"Number of inconsistencies: \", cnt) # slightly smaller than 96 because some edges are associated with the test set","metadata":{"papermill":{"duration":0.236131,"end_time":"2021-06-10T08:53:47.101074","exception":false,"start_time":"2021-06-10T08:53:46.864943","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:01.864904Z","iopub.execute_input":"2021-07-26T18:37:01.865562Z","iopub.status.idle":"2021-07-26T18:37:02.264347Z","shell.execute_reply.started":"2021-07-26T18:37:01.865511Z","shell.execute_reply":"2021-07-26T18:37:02.263259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_mask = (df[\"qid1\"].isin(test_query_qids)) | (df[\"qid2\"].isin(test_query_qids))\ntrain_df = df[~test_mask].copy()\ntest_df = df[test_mask].copy()","metadata":{"papermill":{"duration":0.117128,"end_time":"2021-06-10T08:53:47.247094","exception":false,"start_time":"2021-06-10T08:53:47.129966","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.265939Z","iopub.execute_input":"2021-07-26T18:37:02.266542Z","iopub.status.idle":"2021-07-26T18:37:02.359038Z","shell.execute_reply.started":"2021-07-26T18:37:02.266492Z","shell.execute_reply":"2021-07-26T18:37:02.358101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean up\ndel qid_to_qid_group_leader, qid_group_leader_to_duplicate_qid_group\ndel cnt\ndel test_query_qids # not sorted, use test_query_qids_list\ndel df # all data you can train on is in train_df\n\n# enable use of complete graphs\ntest_qid_to_duplicate_qids = test_qid_to_duplicate_qids_complete\nqid_to_duplicate_qids = qid_to_duplicate_qids_complete","metadata":{"papermill":{"duration":0.138462,"end_time":"2021-06-10T08:53:47.41318","exception":false,"start_time":"2021-06-10T08:53:47.274718","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.360311Z","iopub.execute_input":"2021-07-26T18:37:02.360609Z","iopub.status.idle":"2021-07-26T18:37:02.491298Z","shell.execute_reply.started":"2021-07-26T18:37:02.36058Z","shell.execute_reply":"2021-07-26T18:37:02.490295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Metrics","metadata":{"execution":{"iopub.execute_input":"2021-06-08T11:21:27.301993Z","iopub.status.busy":"2021-06-08T11:21:27.301617Z","iopub.status.idle":"2021-06-08T11:21:27.305697Z","shell.execute_reply":"2021-06-08T11:21:27.304715Z","shell.execute_reply.started":"2021-06-08T11:21:27.301962Z"},"papermill":{"duration":0.027016,"end_time":"2021-06-10T08:53:47.467677","exception":false,"start_time":"2021-06-10T08:53:47.440661","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def method_random_guess(test_qid):\n # returns ranklist and scores of each size RANKED_LIST_SIZE\n return random.choices(train_query_qids_list, k=RANKED_LIST_SIZE), [0]*RANKED_LIST_SIZE\n\n# 1000 x 100 (the ranked list of similar qn for each of the 1000 test qns)\nranklists_method_random_guess = [method_random_guess(test_qid)[0] for test_qid in test_query_qids_list]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:02.492902Z","iopub.execute_input":"2021-07-26T18:37:02.49325Z","iopub.status.idle":"2021-07-26T18:37:02.499199Z","shell.execute_reply.started":"2021-07-26T18:37:02.493219Z","shell.execute_reply":"2021-07-26T18:37:02.498169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_sample_query_results(test_qid, method_ranklist, method_scores=[0]*RANKED_LIST_SIZE, num_to_show=10):\n # not a metric, just print a few examples and its scores\n print(\"Query: {}\".format(qid_to_question[test_qid]))\n for rank, (score, result_qid) in enumerate(zip(method_scores, method_ranklist[:num_to_show]), start=1):\n relevance = \"Registered\" if result_qid in test_qid_to_duplicate_qids[test_qid] else \"Unregistered\"\n print(\"Rank {} - Score {:.4f} - {}: \\t{}\".format(rank, score, relevance, qid_to_question[result_qid]))","metadata":{"papermill":{"duration":0.038642,"end_time":"2021-06-10T08:53:47.653291","exception":false,"start_time":"2021-06-10T08:53:47.614649","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.500465Z","iopub.execute_input":"2021-07-26T18:37:02.500758Z","iopub.status.idle":"2021-07-26T18:37:02.513198Z","shell.execute_reply.started":"2021-07-26T18:37:02.500729Z","shell.execute_reply":"2021-07-26T18:37:02.511983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_random_guess(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:02.515033Z","iopub.execute_input":"2021-07-26T18:37:02.515512Z","iopub.status.idle":"2021-07-26T18:37:02.532021Z","shell.execute_reply.started":"2021-07-26T18:37:02.515436Z","shell.execute_reply":"2021-07-26T18:37:02.530928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_first_relevant_rank(method_ranklists, considered=1, eps=10**-6, debug=True, **kwargs):\n # calculation of the statistics of the rank of the first c=considered duplicates\n # if the duplicate does not appear in the ranklist, it has a default rank of RANKED_LIST_SIZE\n assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n reciprocal_ranks = []\n ranks = []\n for test_qid, ranklist in zip(test_query_qids_list, method_ranklists):\n test_qid_to_rank = {result_qid:rank for rank, result_qid in enumerate(ranklist, start=1)}\n rank = [] # may be shorter than `considered` because of lack of duplicates\n for expected_qid in test_qid_to_duplicate_qids[test_qid]:\n if expected_qid in test_qid_to_rank:\n rank.append(test_qid_to_rank[expected_qid])\n else:\n rank.append(RANKED_LIST_SIZE+1)\n rank.sort()\n ranks.extend(rank[:considered])\n if rank[0] > RANKED_LIST_SIZE:\n reciprocal_ranks.append(0)\n else:\n reciprocal_ranks.append(1/rank[0])\n \n plt.figure(figsize=(14,4))\n plt.title(\"Highest rank of duplicate question\")\n plt.hist(ranks, bins=np.arange(RANKED_LIST_SIZE+2))\n plt.xlabel(\"Rank\")\n plt.ylabel(\"Frequency\")\n plt.show()\n \n mrr = sum(reciprocal_ranks)/len(reciprocal_ranks)\n har = 1/(mrr+eps)\n print(f\"Mean Reciprocal Rank (MRR) is {mrr:.2f}\")\n print(f\"Harmonic Average Rank (HAR) is {har:.2f}\") \n \n p50 = np.median(ranks)\n proportion_out_of_result = ranks.count(RANKED_LIST_SIZE+1)/len(ranks)\n if debug:\n print(\"Median rank: {:.2f}\".format(p50))\n print(\"Proportion out of result: {:.3f}\".format(proportion_out_of_result))\n \n return mrr, har, p50, proportion_out_of_result","metadata":{"papermill":{"duration":0.041196,"end_time":"2021-06-10T08:53:47.722748","exception":false,"start_time":"2021-06-10T08:53:47.681552","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.533772Z","iopub.execute_input":"2021-07-26T18:37:02.534275Z","iopub.status.idle":"2021-07-26T18:37:02.547676Z","shell.execute_reply.started":"2021-07-26T18:37:02.534231Z","shell.execute_reply":"2021-07-26T18:37:02.546631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_first_relevant_rank(ranklists_method_random_guess)","metadata":{"papermill":{"duration":0.442242,"end_time":"2021-06-10T08:53:48.1946","exception":false,"start_time":"2021-06-10T08:53:47.752358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-26T18:37:02.549089Z","iopub.execute_input":"2021-07-26T18:37:02.549486Z","iopub.status.idle":"2021-07-26T18:37:02.897877Z","shell.execute_reply.started":"2021-07-26T18:37:02.549441Z","shell.execute_reply":"2021-07-26T18:37:02.896737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_auc(method_ranklists, k=10, weights=None, debug=True, **kwargs):\n assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n \n counts = np.array([0.]*k)\n ## Identify duplicates among top K ranks for each test\n for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n topk = ranklist[:k]\n is_duplicate = np.array([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in topk])\n counts += is_duplicate \n \n ## Calculate AUC\n if weights:\n counts *= np.array(weights)/sum(weights)\n else:\n counts /= k\n \n auc = sum(counts)/(TEST_SET_SIZE)\n \n if debug:\n print(f\"{auc:.2%} of top {k} results are duplicates\")\n\n return auc # between [0,1], 1 is perfect","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.899757Z","iopub.execute_input":"2021-07-26T18:37:02.900216Z","iopub.status.idle":"2021-07-26T18:37:02.90954Z","shell.execute_reply.started":"2021-07-26T18:37:02.900168Z","shell.execute_reply":"2021-07-26T18:37:02.908806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_auc(ranklists_method_random_guess)\n_ = evaluation_with_auc(ranklists_method_random_guess, weights = [10,9,8,7,6,5,4,3,2,1])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:02.910826Z","iopub.execute_input":"2021-07-26T18:37:02.911208Z","iopub.status.idle":"2021-07-26T18:37:02.929413Z","shell.execute_reply.started":"2021-07-26T18:37:02.911175Z","shell.execute_reply":"2021-07-26T18:37:02.928174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def single_r_precision(test_qid, ranklist):\n # use this to check a single test query\n num_duplicate = len(test_qid_to_duplicate_qids[test_qid]) # this dict needs to be updated when train:test set separation is updated\n if num_duplicate == 0:\n return 0, 0, 0\n top_r = ranklist[:num_duplicate]\n num_duplicates_in_top_r = sum([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in top_r])\n r_precision = num_duplicates_in_top_r/num_duplicate\n return num_duplicate, num_duplicates_in_top_r, r_precision\n\n\ndef evaluation_with_r_precision(method_ranklists, k=10, report_k=0, debug=True, **kwargs):\n print(np.array(method_ranklists).shape)\n assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE) # method_ranklists size is (1000,100)\n \n total_num_duplicates = np.array([0 for i in range(TEST_SET_SIZE)])\n r_precision = np.array([0 for i in range(TEST_SET_SIZE)])\n \n ## Iter over 1->1000 tests\n for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)): # iter over 1->1000 tests\n total_num_duplicates[i], num_duplicates_in_top_r, r_precision[i] = single_r_precision(test_qid, ranklist)\n \n # note: if want do error analysis, intervene here to find test cases with low r precision\n if report_k > 0:\n k_lowest_r_precision_idx = np.argpartition(r_precision, k)[:k]\n k_lowest_r_precision_test_qids = np.array(test_query_qids_list)[k_lowest_r_precision_idx]\n\n ## Calculate metrics\n avg_r_precision = r_precision.mean()\n weighted_avg_r_precision = np.multiply(r_precision, total_num_duplicates).sum() / total_num_duplicates.sum()\n \n if debug:\n print(f\"Average R-Precision = {avg_r_precision:.2%}\")\n print(f\"Weighted Average R-Precision by proportion of duplicates = {weighted_avg_r_precision:.2%}\") \n if avg_r_precision > weighted_avg_r_precision:\n print(\"A higher average R-Precisions suggests that there are many test queries with high R-Precision but there are some test queries with high number of duplicates that model is not effective with.\")\n \n if not report_k: return avg_r_precision, weighted_avg_r_precision\n else:\n return avg_r_precision, weighted_avg_r_precision, k_lowest_r_precision_test_qids","metadata":{"papermill":{"duration":0.039167,"end_time":"2021-06-10T08:53:48.479042","exception":false,"start_time":"2021-06-10T08:53:48.439875","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.931125Z","iopub.execute_input":"2021-07-26T18:37:02.931576Z","iopub.status.idle":"2021-07-26T18:37:02.944921Z","shell.execute_reply.started":"2021-07-26T18:37:02.931531Z","shell.execute_reply":"2021-07-26T18:37:02.943859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = evaluation_with_r_precision(ranklists_method_random_guess, k=10)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:02.951931Z","iopub.execute_input":"2021-07-26T18:37:02.952289Z","iopub.status.idle":"2021-07-26T18:37:02.960732Z","shell.execute_reply.started":"2021-07-26T18:37:02.952258Z","shell.execute_reply":"2021-07-26T18:37:02.959852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_precision_recall_at_k(method_ranklists, k=10, exclude_precision=False, exclude_recall=False, debug=True, **kwargs):\n assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n ## Evaluation returns the macro average P@K and R@Kfor test set\n ## Interpretation P@K: what % of top k retrieved is relevant?\n ## Interpretation R@K: what % of all duplicates for query is retrieved within top k?\n \n ## Iter thru each test\n precisions_at_k = []\n recalls_at_k = []\n for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n ## 1. Set rank threshold K, ignore all docs after K\n ## 2. Count num_relevant in top-K\n ## 3. Count total_num_duplicates_for_query\n ## 4. P@K = num_relevant/k\n ## 5. R@K = num_relevant/total_num_duplicates_for_query\n topk = ranklist[:k]\n num_relevant = sum([1 if (result_qid in test_qid_to_duplicate_qids[test_qid]) else 0 for result_qid in topk])\n \n precision_at_k = num_relevant/k\n precisions_at_k.append(precision_at_k)\n \n total_num_duplicates_for_query = len(test_qid_to_duplicate_qids[test_qid])\n recall_at_k = num_relevant/total_num_duplicates_for_query\n recalls_at_k.append(recall_at_k)\n \n mean_precision_at_k = sum(precisions_at_k)/len(precisions_at_k) # macro average\n mean_recall_at_k = sum(recalls_at_k)/len(recalls_at_k) # macro average\n print(f\"Macro Average Precision@k={k} is {mean_precision_at_k:.2%}\")\n print(f\"Macro Average Recall@k={k} is {mean_recall_at_k:.2%}\")\n return (mean_precision_at_k, mean_recall_at_k)\n\n_ = evaluation_with_precision_recall_at_k(ranklists_method_random_guess, k=10)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.963846Z","iopub.execute_input":"2021-07-26T18:37:02.964167Z","iopub.status.idle":"2021-07-26T18:37:02.975736Z","shell.execute_reply.started":"2021-07-26T18:37:02.964133Z","shell.execute_reply":"2021-07-26T18:37:02.974504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_with_map(method_ranklists, debug=True, **kwargs):\n assert np.array(method_ranklists).shape == (TEST_SET_SIZE, RANKED_LIST_SIZE)\n ## Interpretation: what is the average precision for all relevant docs across all queries?\n\n ## Iter thru each test\n average_precisions = []\n for i, (test_qid, ranklist) in enumerate(zip(test_query_qids_list, method_ranklists)):\n ## 1. Find the rank positions of each of the R relevant docs: K1, K2, ... KR and sort \n ## 2. Compute P@K for each K1, K2, ... If K >=RANKED_LIST_SIZE, assume never retrieved\n ## 3. AP = average of P@K for query\n ## 4. MAP = macro average of AP across queries\n\n ## 1. Find the rank positions of each of the R relevant docs: K1, K2, ... and sort \n dup_qids_in_train_set = [dup_qid for dup_qid in test_qid_to_duplicate_qids[test_qid] if dup_qid in train_query_qids_list] # find all the dup_qid that can be found in the train set so you know total dup qn that could be found\n total_num_dup_qid = len(dup_qids_in_train_set) # how many dup qn to expect\n\n dup_ranks = []\n for dup_qid in dup_qids_in_train_set:\n if dup_qid not in ranklist: # not found\n dup_ranks.append(RANKED_LIST_SIZE) # give \"out of range\" rank which would be checked later during calculation\n continue\n dup_ranks.append(list(ranklist).index(dup_qid)+1) # append the rank of the retrieved dup qn\n \n dup_ranks, dup_qids_in_train_set = (list(t) for t in zip(*sorted(zip(dup_ranks, dup_qids_in_train_set)))) # sort by rank\n ## 2. Compute P@K for each K1, K2, ... If K >=RANKED_LIST_SIZE, assume never retrieved\n precisions_at_k = []\n for j, rank in enumerate(dup_ranks, start=1): # dup_ranks is sorted\n if rank >= RANKED_LIST_SIZE: # handle \"unretrieved\" duplicates\n precisions_at_k.append(0)\n else: \n precision_at_k = j / rank # = num_dup_so_far / rank_of_latest_dup_found\n precisions_at_k.append(precision_at_k)\n \n ## 3. AP = average of P@K for query\n average_precisions.append(sum(precisions_at_k)/len(precisions_at_k))\n \n ## Out of test query loop\n ## 4. MAP = macro average of AP across queries\n MAP = sum(average_precisions)/len(average_precisions)\n print(f\"Mean Average Precision (MAP) is {MAP:.2%}\")\n return MAP\n\n_ = evaluation_with_map(ranklists_method_random_guess)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:02.97763Z","iopub.execute_input":"2021-07-26T18:37:02.978048Z","iopub.status.idle":"2021-07-26T18:37:03.188317Z","shell.execute_reply.started":"2021-07-26T18:37:02.978004Z","shell.execute_reply":"2021-07-26T18:37:03.187112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_process(method, test_query_qids_list=test_query_qids_list, \n calculate_metrics=True, use_tqdm=True, **kwargs):\n # executes the method and runs the evaluation functions \n ranklists, scorelists = [], []\n \n iterator = tqdm.tqdm if use_tqdm else iter\n \n for test_qid in iterator(test_query_qids_list):\n ranklist, scores = method(test_qid)\n ranklists.append(ranklist)\n scorelists.append(scores)\n \n if calculate_metrics:\n evaluation_with_first_relevant_rank(ranklists, **kwargs)\n # evaluation_with_auc(ranklists, **kwargs)\n evaluation_with_r_precision(ranklists, **kwargs)\n\n evaluation_with_precision_recall_at_k(ranklists, k=10, **kwargs)\n evaluation_with_map(ranklists, **kwargs)\n\n return ranklists, scorelists","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:03.18964Z","iopub.execute_input":"2021-07-26T18:37:03.189986Z","iopub.status.idle":"2021-07-26T18:37:03.197315Z","shell.execute_reply.started":"2021-07-26T18:37:03.189953Z","shell.execute_reply":"2021-07-26T18:37:03.196094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_random_guess = evaluation_process(method_random_guess)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:03.198855Z","iopub.execute_input":"2021-07-26T18:37:03.199353Z","iopub.status.idle":"2021-07-26T18:37:03.773712Z","shell.execute_reply.started":"2021-07-26T18:37:03.199302Z","shell.execute_reply":"2021-07-26T18:37:03.772551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing the Text","metadata":{"papermill":{"duration":0.029704,"end_time":"2021-06-10T08:53:48.598868","exception":false,"start_time":"2021-06-10T08:53:48.569164","status":"completed"},"tags":[]}},{"cell_type":"code","source":"## This entire cell is important to enable tokeniser pipeline \n## Use this to replace tokenise function if using Tokenise then Spellcheck (TSC) pipeline\n\n######### spacy basic tokenizer\nimport spacy\nprint(\"Spacy version: \", spacy.__version__)\nfrom spacy.tokenizer import Tokenizer # https://spacy.io/api/tokenizer\n\n# !python3 -m spacy download en_core_web_sm\nprint(\"Loading Spacy en_core_web_sm loaded\")\nnlp = spacy.load(\"en_core_web_sm\")\ntokenizer = Tokenizer(nlp.vocab)\ntokenizer.add_special_case(\"[math]\", [{\"ORTH\": \"[math]\"}]) # see qid=7: '[math]23^{24}[/math]' becomes one token\n# add more special cases here if found","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:03.775089Z","iopub.execute_input":"2021-07-26T18:37:03.775422Z","iopub.status.idle":"2021-07-26T18:37:04.767244Z","shell.execute_reply.started":"2021-07-26T18:37:03.77539Z","shell.execute_reply":"2021-07-26T18:37:04.766218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def spacy_tokenise(text, lower=False, split_last_punc=True):\n \"\"\"\n returns a list of tokens given a question text\n note: each punctuation is also considered a token\n note: \"\\n\" is a token\n note: \"'s\" is a token\n note: '(Koh-i-Noor)' is a token\n see tokenizer instantiation code for special cases or to add\n lowercase text only after spell check\n \"\"\"\n if lower: text = text.lower()\n tokens = tokenizer(text)\n token_list = [token.text for token in tokens]\n\n # further split tokens that end with certain punct e.g. \"me?\" => \"me\", \"?\"\n if split_last_punc: \n split_lists = [[token[:-1], token[-1]] if (token[-1] in [\"!\",\"?\",\",\",\":\"]) else [token] for token in token_list]\n token_list = [token for sublist in split_lists for token in sublist]\n return token_list\n\n######### symspell spellchecker\nprint(\"Loading symspell\")\n!pip install symspellpy\nfrom symspellpy.symspellpy import SymSpell, Verbosity # https://github.com/mammothb/symspellpy\nimport pkg_resources\n\n# instantiate spellchecker\nsym = SymSpell(max_dictionary_edit_distance=2, prefix_length=7, count_threshold=1)\n# https://symspellpy.readthedocs.io/en/latest/api/symspellpy.html\ndictionary_path = pkg_resources.resource_filename(\"symspellpy\", \"frequency_dictionary_en_82_765.txt\")\nsym.load_dictionary(dictionary_path, 0, 1) # might take a short while\n\ndef spellcheck_single(word):\n # returns top correct spelling or the same word if no correction found within max_edit_distance\n if not word.isascii(): return word # do not spellcheck non ascii words e.g. シ\n\n # obtain list of suggestions\n suggestions = sym.lookup(word, Verbosity.CLOSEST, max_edit_distance=2,\n include_unknown=True, # a mispelled word with no found corrections is returned as is\n ignore_token=r\"[:,.!?\\\\-]\" # use if want to avoid correcting certain phrases\n )\n # get the term from the suggestItem object\n suggested_words = [suggestion._term for suggestion in suggestions]\n \n # check if the input word is legit and return if so else return corrected word\n word_lower = word.lower()\n if word_lower in suggested_words: return word_lower # do not correct if input is a legit word\n else: return suggested_words[0] # top suggestion\n\ndef spellcheck_compound(sent):\n # spellchecks a sentence\n suggestions = sym.lookup_compound(sent, max_edit_distance=2)\n return suggestions[0]._term # returns the top suggestion\n\n######### tokenise pipeline\ndef tokenise_then_spellcheck(sent):\n # 8 times faster than spellcheck_then_tokenise\n tokens = spacy_tokenise(sent) # NOTE: replace tokenise with spacy_tokenise\n checked_tokens = [spellcheck_single(token).lower() for token in tokens] # lower after spell check\n return checked_tokens\n\ndef spellcheck_then_tokenise(sent):\n checked_sent = spellcheck_compound(sent)\n tokens = spacy_tokenise(checked_sent, lower=True) # lower after spell check\n return tokens","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:04.768703Z","iopub.execute_input":"2021-07-26T18:37:04.768997Z","iopub.status.idle":"2021-07-26T18:37:17.443571Z","shell.execute_reply.started":"2021-07-26T18:37:04.768969Z","shell.execute_reply":"2021-07-26T18:37:17.442592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define tokenisation process\n\nimport pickle\nqid_to_tokens_preprocessed_filename = \"../input/quora-question-pairs-tokenise-pipeline/qid_to_processed_token_list_tokenise_then_spellcheck.pkl\"\nwith open(qid_to_tokens_preprocessed_filename, \"rb\") as f:\n qid_to_tokens_preprocessed = pickle.load(f)\n\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstopword_set = set(stopwords.words())\nstopword_set.update([\"?\", \",\"])\n\ndef nltk_tokenize(sentence):\n return word_tokenize(sentence.lower())\n\ndef tokenise_qid(qid, qid_to_tokens_preprocessed=qid_to_tokens_preprocessed, \n tokenise_method=tokenise_then_spellcheck):\n # return a list of tokens, does not remove stopwords or duplicates\n if qid_to_tokens_preprocessed and qid in qid_to_tokens_preprocessed:\n return qid_to_tokens_preprocessed[qid]\n return tokenise_method(qid_to_question[qid])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:17.446258Z","iopub.execute_input":"2021-07-26T18:37:17.44655Z","iopub.status.idle":"2021-07-26T18:37:20.023209Z","shell.execute_reply.started":"2021-07-26T18:37:17.446521Z","shell.execute_reply":"2021-07-26T18:37:20.022308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_vsm(train_query_qids_list=train_query_qids_list, stopword_set=stopword_set, exclude_stopwords=True):\n '''\n Input:\n qid_to_question = {qid: question string}\n Note: only use the test subset of qids\n \n Outputs:\n qid_to_tokens = {qid: set(tokens)}\n token_to_qids = {token: set(qids)}\n tf = {token: {qid: TF as int}}\n df = {token: DF as int}\n L = {qid: question length as int}\n '''\n qid_to_tokens = defaultdict(set)\n token_to_qids = defaultdict(set)\n tf = defaultdict(Counter)\n df = defaultdict(int)\n L = defaultdict(int)\n\n qid_processed = set()\n for qid in tqdm.tqdm(train_query_qids_list):\n qid_tokenised = tokenise_qid(qid)\n\n for token in set(qid_tokenised):\n if token not in stopword_set or not exclude_stopwords:\n # store qid-to-token mapping\n qid_to_tokens[qid].add(token)\n token_to_qids[token].add(qid)\n\n # compute and store term frequency\n tf[token][qid] += 1 \n\n # store doc frequency in df\n df[token] += 1\n\n # store doc length in L (double-count repeated tokens)\n L[qid] = len(qid_tokenised)\n \n # output\n return qid_to_tokens, token_to_qids, tf, df, L","metadata":{"papermill":{"duration":0.038416,"end_time":"2021-06-10T08:53:50.7258","exception":false,"start_time":"2021-06-10T08:53:50.687384","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:20.025736Z","iopub.execute_input":"2021-07-26T18:37:20.026044Z","iopub.status.idle":"2021-07-26T18:37:20.037705Z","shell.execute_reply.started":"2021-07-26T18:37:20.026017Z","shell.execute_reply":"2021-07-26T18:37:20.036779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qid_to_tokens, token_to_qids, tf, df, L = preprocess_vsm()\n\n# save a copy of the original to allow reset later\nqid_to_tokens_original, token_to_qids_original = qid_to_tokens.copy(), token_to_qids.copy()\ntf_original, df_original, L_original = tf.copy(), df.copy(), L.copy()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:20.03913Z","iopub.execute_input":"2021-07-26T18:37:20.039505Z","iopub.status.idle":"2021-07-26T18:37:34.706609Z","shell.execute_reply.started":"2021-07-26T18:37:20.039429Z","shell.execute_reply":"2021-07-26T18:37:34.705771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline - Overlapping Root Word Count (Working Title)\nOrder by the number of overlapping non-stopword words. Random if tie.","metadata":{}},{"cell_type":"code","source":"def method_overlapping_root_word_count(query_qid, ignore_stopwords=True):\n query_tokens = set(tokenise_qid(query_qid))\n if ignore_stopwords:\n query_tokens = [token for token in query_tokens if token not in stopword_set]\n counter = collections.Counter()\n \n for dummy_qid in random.choices(train_query_qids_list, k=RANKED_LIST_SIZE):\n # prefill with random results to address the possibility of no matches\n counter[dummy_qid] = 0.01\n \n for query_token in query_tokens:\n counter += collections.Counter(token_to_qids[query_token])\n \n query_results = list(counter.items())\n random.shuffle(query_results) # so that qids are not ordered\n query_results = sorted(query_results, key=lambda x:x[1], reverse=True)[:RANKED_LIST_SIZE]\n\n return [x[0] for x in query_results], [x[1] for x in query_results]","metadata":{"papermill":{"duration":0.352361,"end_time":"2021-06-10T08:55:38.466409","exception":false,"start_time":"2021-06-10T08:55:38.114048","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:34.707799Z","iopub.execute_input":"2021-07-26T18:37:34.708249Z","iopub.status.idle":"2021-07-26T18:37:34.770019Z","shell.execute_reply.started":"2021-07-26T18:37:34.708204Z","shell.execute_reply":"2021-07-26T18:37:34.7692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_overlapping_root_word_count(test_query_qids_list[0]))","metadata":{"papermill":{"duration":0.372296,"end_time":"2021-06-10T08:55:39.195955","exception":false,"start_time":"2021-06-10T08:55:38.823659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-26T18:37:34.771023Z","iopub.execute_input":"2021-07-26T18:37:34.771421Z","iopub.status.idle":"2021-07-26T18:37:34.957306Z","shell.execute_reply.started":"2021-07-26T18:37:34.771391Z","shell.execute_reply":"2021-07-26T18:37:34.956182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_overlapping_root_word_count = evaluation_process(method_overlapping_root_word_count)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:34.958673Z","iopub.execute_input":"2021-07-26T18:37:34.958989Z","iopub.status.idle":"2021-07-26T18:37:36.235383Z","shell.execute_reply.started":"2021-07-26T18:37:34.958956Z","shell.execute_reply":"2021-07-26T18:37:36.234413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFIDF","metadata":{}},{"cell_type":"code","source":"def compute_idf(doc_freq, N):\n '''\n Inputs:\n doc_freq = document frequency of some token\n N = corpus size including query\n \n Output:\n idf = IDF as float\n '''\n return log(N/doc_freq)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:36.23858Z","iopub.execute_input":"2021-07-26T18:37:36.23888Z","iopub.status.idle":"2021-07-26T18:37:36.24333Z","shell.execute_reply.started":"2021-07-26T18:37:36.238844Z","shell.execute_reply":"2021-07-26T18:37:36.242524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import reduce\nimport operator\n\ndef prod(iterable):\n return reduce(operator.mul, iterable, 1)\n\n\ndef use_vsm(qid_query, \\\n # qid_to_tokens=qid_to_tokens, tf=tf, df=df, L=L,\n method='tf-idf', compute_idf=compute_idf,\n k1=1.5, k3=1.5, b=0.75,\n smoothing='add-one', alpha=0.75, eps=10**(-6),\n exclude_stopwords=True,\n return_top=RANKED_LIST_SIZE):\n \n '''\n Inputs:\n qid_query = qid of question match # this comes from \"test\" set\n qid_to_tokens = {qid: set(tokens)} # this is the \"training\" corpus\n tf = {token: term freq} # required for all methods\n df = {token: doc freq} # required for method='tf-idf','bm25'\n L = {qid: doc length} # required for method='bm25','unigram'\n\n method = model to apply\n k1, k3, b = tuning params # required for method='bm25'\n smoothing = type of smoothing # required for method='unigram'\n return_top = num of docs to return\n \n Procedure:\n 0. Corpus is already tokenised, tf, df, L already computed\n 1. Tokenise query, expand tf, df, L with query information\n \n if method='boolean':\n Remove idf calculation, then use method='tf-idf'\n\n if method='tf-idf':\n 2. Compute tf-idf weights only for relevant (t,d) pairs\n 3. Compute cosine similarity only for docs containing query terms\n \n if method='bm25':\n 2. Compute RSV summation terms only for relevant (t,d) pairs\n 3. Compute RSV only for docs containing query terms\n \n if method='unigram':\n 2. Compute probabilities only for relevant (t,d) pairs\n 3. Compute query probability only for docs containing query terms\n \n 4. Return docs in ranked order\n\n Output:\n ranking = [qids in decreasing order of match]\n scoring = [corresponding scores]\n '''\n \n assert method in ['boolean','tf-idf','bm25','unigram'], \"Supported methods: 'boolean', 'tf-idf', 'bm25', 'unigram'\"\n assert len(L.keys()) > 0 if method=='bm25' else True, \"Please include L for bm25\"\n assert len(L.keys()) > 0 if method=='unigram' else True, \"Please include L for unigram\"\n assert smoothing in ['add-one','linear-interpolation'] if method=='unigram' else True\n assert alpha >= 0 and alpha <= 1 if smoothing=='linear-interpolation' else True\n\n qid_tmp = time.time()\n\n ''' STEP 1: PROCESS QUERY '''\n query_tokenised = tokenise_qid(qid_query)\n \n for token in set(query_tokenised):\n if token not in stopword_set or not exclude_stopwords:\n # store qid-to-token mapping\n # store query as qid=0 (corpus starts from qid=1)\n qid_to_tokens[qid_tmp].add(token)\n\n # compute and store term frequency\n tf[token][qid_tmp] = sum([1 if t==token else 0 for t in query_tokenised])\n \n # update doc frequency in df\n df[token] += 1\n \n # store query length\n L[qid_tmp] = len(query_tokenised)\n\n if method=='boolean':\n def compute_idf(doc_freq, N):\n return 1\n method = 'tf-idf'\n \n if method=='tf-idf':\n \n ''' STEP 2: COMPUTE TF-IDF WEIGHTS '''\n weights = defaultdict(lambda: defaultdict(float))\n N = len(qid_to_tokens) # original corpus + query\n\n # only bother computing for tokens in the query\n for token in set(query_tokenised):\n if token not in stopword_set or exclude_stopwords==False:\n weights[qid_tmp][token] = tf[token][qid_tmp] * compute_idf(df[token], N)\n \n for qid in tf[token].keys():\n weights[qid][token] = tf[token][qid] * compute_idf(df[token], N)\n \n # also compute weight for other tokens contained by these qids\n # needed for computing qid vector length\n for other_token in qid_to_tokens[qid]:\n weights[qid][other_token] = tf[other_token][qid] * compute_idf(df[other_token], N)\n\n \n ''' STEP 3: COMPUTE COSINE SIMILARITY TO QUERY '''\n cosine_similarities = defaultdict(float)\n # compute denominator (part 1), i.e., |q| * |d|\n query_vector_length = (sum([w**2 for w in weights[qid_tmp].values()]))**0.5\n if query_vector_length == 0:\n print(f\"query={qid_query}\\nweights[qid_tmp].items()={weights[qid_tmp].items()}\")\n \n for qid in weights.keys():\n \n # compute numerator, i.e., dot product of q and d\n cosine_numerator = 0\n \n for token in weights[qid].keys():\n if token in weights[qid_tmp]:\n cosine_numerator += weights[qid][token] * weights[qid_tmp][token]\n \n # compute denominator (part 2), i.e., |q| * |d|\n qid_vector_length = (sum([w**2 for w in weights[qid].values()]))**0.5\n if qid_vector_length == 0: # example: qid=25026 => question='?'\n qid_vector_length = 1e-8\n\n # compute and store cosine similarity between q and d\n cosine_similarities[qid] = cosine_numerator / (query_vector_length+eps) / (qid_vector_length+eps)\n \n scores = cosine_similarities\n\n if method=='bm25':\n\n ''' STEP 2: COMPUTE RSV TERMS '''\n rsv_terms = defaultdict(lambda: defaultdict(float))\n N = len(qid_to_tokens) # original corpus + query\n L_avg = sum(L.values())/len(L.values())\n\n # only bother computing for tokens in the query\n for token in set(query_tokenised):\n for qid in tf[token].keys():\n rsv_terms[qid][token] = compute_idf(df[token], N) \\\n * (k1+1)*tf[token][qid] / (k1*((1-b)+b*L[qid]*L_avg) + tf[token][qid]) \\\n * (k3+1)*tf[token][qid_tmp] / (k3 + tf[token][qid_tmp])\n\n ''' STEP 3: COMPUTE RSV '''\n rsv = {qid: sum(rsv_terms[qid].values()) for qid in rsv_terms.keys()}\n scores = rsv\n \n if method=='unigram':\n \n ''' STEP 2: COMPUTE PROBABILITIES '''\n probabilities = defaultdict(lambda: defaultdict(float))\n corpus_model = defaultdict(float)\n \n # only bother computing for tokens in the query\n for token in set(query_tokenised):\n for qid in tf[token].keys():\n\n if smoothing=='add-one':\n probabilities[qid][token] = (tf[token][qid]+1) / (L[qid]+len(query_tokenised))\n else:\n probabilities[qid][token] = (tf[token][qid]) / (L[qid])\n\n # for linear-interpolation smoothing, build corpus language model\n if smoothing=='linear-interpolation':\n corpus_model[token] += tf[token][qid]\n\n # remaining operations for linear-interpolation smoothing \n if smoothing=='linear-interpolation':\n # finish building corpus language model by dividing corpus tf by corpus L\n total_corpus_length = sum(L.values())\n for token in corpus_model.keys():\n corpus_model[token] = corpus_model[token] / total_corpus_length\n \n # then update the probabilities\n for qid in probabilities.keys():\n for token in probabilities[qid].keys():\n probabilities[qid][token] = alpha*probabilities[qid][token] + (1-alpha)*corpus_model[token]\n\n ''' STEP 3: COMPUTE QUERY PROBABILITY '''\n query_prob = {qid: -log(prod(probabilities[qid].values())) for qid in probabilities.keys()}\n scores = query_prob\n\n ''' STEP 4: RANK DOCUMENTS AND RETURN RESULT '''\n # cleanup\n if qid_tmp in qid_to_tokens:\n del qid_to_tokens[qid_tmp]\n for token in set(query_tokenised):\n if token not in stopword_set or not exclude_stopwords:\n del tf[token][qid_tmp]\n df[token] -= 1\n \n if qid_tmp in scores:\n del scores[qid_tmp] # remove query from result\n ranking = sorted(scores, key=scores.get, reverse=True)\n scoring = sorted(scores.values(), reverse=True)\n\n # if too few documents match the query, add dummy documents\n if len(ranking) < return_top:\n ranking.extend([0]*(return_top-len(ranking)))\n scoring.extend([0]*(return_top-len(ranking)))\n\n # return top k results\n return ranking[:return_top], scoring[:return_top]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:37:36.244518Z","iopub.execute_input":"2021-07-26T18:37:36.244855Z","iopub.status.idle":"2021-07-26T18:37:36.280521Z","shell.execute_reply.started":"2021-07-26T18:37:36.244817Z","shell.execute_reply":"2021-07-26T18:37:36.279446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_boolean(qid):\n return use_vsm(qid, method='boolean')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:36.28216Z","iopub.execute_input":"2021-07-26T18:37:36.282499Z","iopub.status.idle":"2021-07-26T18:37:36.296655Z","shell.execute_reply.started":"2021-07-26T18:37:36.282467Z","shell.execute_reply":"2021-07-26T18:37:36.295829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_boolean(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:36.297951Z","iopub.execute_input":"2021-07-26T18:37:36.29831Z","iopub.status.idle":"2021-07-26T18:37:36.948351Z","shell.execute_reply.started":"2021-07-26T18:37:36.298244Z","shell.execute_reply":"2021-07-26T18:37:36.946997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_boolean = evaluation_process(method_boolean)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:36.950182Z","iopub.execute_input":"2021-07-26T18:37:36.950622Z","iopub.status.idle":"2021-07-26T18:37:41.666714Z","shell.execute_reply.started":"2021-07-26T18:37:36.950572Z","shell.execute_reply":"2021-07-26T18:37:41.665609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_tf_idf(qid):\n return use_vsm(qid, method='tf-idf')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:41.667945Z","iopub.execute_input":"2021-07-26T18:37:41.668262Z","iopub.status.idle":"2021-07-26T18:37:41.673195Z","shell.execute_reply.started":"2021-07-26T18:37:41.668231Z","shell.execute_reply":"2021-07-26T18:37:41.672017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_tf_idf(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:41.674725Z","iopub.execute_input":"2021-07-26T18:37:41.675057Z","iopub.status.idle":"2021-07-26T18:37:42.403041Z","shell.execute_reply.started":"2021-07-26T18:37:41.675025Z","shell.execute_reply":"2021-07-26T18:37:42.402029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_tf_idf = evaluation_process(method_tf_idf)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:42.404462Z","iopub.execute_input":"2021-07-26T18:37:42.404779Z","iopub.status.idle":"2021-07-26T18:37:47.810055Z","shell.execute_reply.started":"2021-07-26T18:37:42.404737Z","shell.execute_reply":"2021-07-26T18:37:47.809004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BM25","metadata":{}},{"cell_type":"code","source":"def method_bm25(qid):\n return use_vsm(qid, method='bm25')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:47.812286Z","iopub.execute_input":"2021-07-26T18:37:47.81271Z","iopub.status.idle":"2021-07-26T18:37:47.81679Z","shell.execute_reply.started":"2021-07-26T18:37:47.812665Z","shell.execute_reply":"2021-07-26T18:37:47.81572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_bm25(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:47.81839Z","iopub.execute_input":"2021-07-26T18:37:47.818786Z","iopub.status.idle":"2021-07-26T18:37:48.031526Z","shell.execute_reply.started":"2021-07-26T18:37:47.818745Z","shell.execute_reply":"2021-07-26T18:37:48.03033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_bm25 = evaluation_process(method_bm25)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:48.03282Z","iopub.execute_input":"2021-07-26T18:37:48.033114Z","iopub.status.idle":"2021-07-26T18:37:51.924912Z","shell.execute_reply.started":"2021-07-26T18:37:48.033086Z","shell.execute_reply":"2021-07-26T18:37:51.923727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_unigram(qid):\n return use_vsm(qid, method='unigram', smoothing='add-one')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:51.926078Z","iopub.execute_input":"2021-07-26T18:37:51.926406Z","iopub.status.idle":"2021-07-26T18:37:51.93159Z","shell.execute_reply.started":"2021-07-26T18:37:51.926376Z","shell.execute_reply":"2021-07-26T18:37:51.930404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_unigram(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:51.933369Z","iopub.execute_input":"2021-07-26T18:37:51.933777Z","iopub.status.idle":"2021-07-26T18:37:52.094971Z","shell.execute_reply.started":"2021-07-26T18:37:51.933733Z","shell.execute_reply":"2021-07-26T18:37:52.094056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_unigram = evaluation_process(method_unigram)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:52.096612Z","iopub.execute_input":"2021-07-26T18:37:52.096973Z","iopub.status.idle":"2021-07-26T18:37:53.5674Z","shell.execute_reply.started":"2021-07-26T18:37:52.09694Z","shell.execute_reply":"2021-07-26T18:37:53.565709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for alpha in [0, 0.25, 0.5, 0.75, 1.0]:\n def method_unigram(qid):\n return use_vsm(qid, method='unigram', smoothing='linear-interpolation', alpha=alpha)\n _ = evaluation_process(method_unigram)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:37:53.568799Z","iopub.execute_input":"2021-07-26T18:37:53.569172Z","iopub.status.idle":"2021-07-26T18:38:04.517186Z","shell.execute_reply.started":"2021-07-26T18:37:53.569127Z","shell.execute_reply":"2021-07-26T18:38:04.515931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word Embeddings - SpaCy","metadata":{}},{"cell_type":"code","source":"def to_vec(token_or_list):\n # converts a token string or a list of tokens into a word or doc vec respectively\n if type(token_or_list) == list:\n # token list needs to be joined into a sentence first\n token_or_list = ' '.join(token_or_list)\n return nlp(token_or_list).vector\n\nnlp2 = spacy.load(\"en_core_web_lg\")\ndef to_vec2(token_or_list):\n # converts a token string or a list of tokens into a word or doc vec respectively\n if type(token_or_list) == list:\n # token list needs to be joined into a sentence first\n token_or_list = ' '.join(token_or_list)\n return nlp2(token_or_list).vector","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:04.518751Z","iopub.execute_input":"2021-07-26T18:38:04.519213Z","iopub.status.idle":"2021-07-26T18:38:11.823412Z","shell.execute_reply.started":"2021-07-26T18:38:04.519167Z","shell.execute_reply":"2021-07-26T18:38:11.822335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pre-processed dict\nwith open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_vec.pkl\", \"rb\") as f:\n qid_to_vec = pickle.load(f)\n\nprint(\"Pre-processed question vector is of shape {}\".format(qid_to_vec[0].shape))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:11.827164Z","iopub.execute_input":"2021-07-26T18:38:11.827479Z","iopub.status.idle":"2021-07-26T18:38:14.206657Z","shell.execute_reply.started":"2021-07-26T18:38:11.827447Z","shell.execute_reply":"2021-07-26T18:38:14.205815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import dot\nfrom numpy.linalg import norm\n\ndef method_spacy_embedding_similarity(test_qid):\n tokens = tokenise_then_spellcheck(qid_to_question[test_qid])\n test_vec = to_vec(tokens)\n \n ## Run baseline model as a filter\n qid_list, scores = method_overlapping_root_word_count(test_qid)\n \n cos_sims = [] # bigger better\n for train_qid in qid_list:# train_query_qids_list:\n train_vec = qid_to_vec[train_qid]\n cos_sim = dot(test_vec, train_vec)/(norm(test_vec)*norm(train_vec))\n cos_sims.append(cos_sim)\n\n cos_sims = np.array(cos_sims)\n qid_list = np.array(qid_list) # train_query_qids_list\n inds = cos_sims.argsort()[::-1] # reverse so biggest come first\n cos_sims = cos_sims[inds]\n ranklist = qid_list[inds] \n\n return ranklist[:RANKED_LIST_SIZE], cos_sims[:RANKED_LIST_SIZE]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:49:22.171342Z","iopub.execute_input":"2021-07-26T18:49:22.17177Z","iopub.status.idle":"2021-07-26T18:49:22.180329Z","shell.execute_reply.started":"2021-07-26T18:49:22.171734Z","shell.execute_reply":"2021-07-26T18:49:22.179055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_spacy_embedding_similarity(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:49:34.454123Z","iopub.execute_input":"2021-07-26T18:49:34.454528Z","iopub.status.idle":"2021-07-26T18:49:34.591234Z","shell.execute_reply.started":"2021-07-26T18:49:34.454495Z","shell.execute_reply":"2021-07-26T18:49:34.590081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_spacy_embedding_similarity = evaluation_process(method_spacy_embedding_similarity)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:49:35.411112Z","iopub.execute_input":"2021-07-26T18:49:35.411475Z","iopub.status.idle":"2021-07-26T18:49:36.882443Z","shell.execute_reply.started":"2021-07-26T18:49:35.411445Z","shell.execute_reply":"2021-07-26T18:49:36.881437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_vec_trf.pkl\", \"rb\") as f: # note, actually lg not trf\n qid_to_vec2 = pickle.load(f)\n\nprint(\"Pre-processed question vector is of shape {}\".format(qid_to_vec2[0].shape)) # 300 dim vec","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:49:38.332217Z","iopub.execute_input":"2021-07-26T18:49:38.332609Z","iopub.status.idle":"2021-07-26T18:49:41.164121Z","shell.execute_reply.started":"2021-07-26T18:49:38.332572Z","shell.execute_reply":"2021-07-26T18:49:41.16306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_spacy_embedding_similarity_lg(test_qid):\n tokens = tokenise_then_spellcheck(qid_to_question[test_qid])\n test_vec = to_vec2(tokens)\n \n ## Run baseline model as a filter\n qid_list, scores = method_overlapping_root_word_count(test_qid)\n \n cos_sims = [] # bigger better\n for train_qid in qid_list:# train_query_qids_list:\n train_vec = qid_to_vec2[train_qid]\n cos_sim = dot(test_vec, train_vec)/(norm(test_vec)*norm(train_vec))\n cos_sims.append(cos_sim)\n\n cos_sims = np.array(cos_sims)\n qid_list = np.array(qid_list) # train_query_qids_list)\n inds = cos_sims.argsort()[::-1] # reverse so biggest come first\n cos_sims = cos_sims[inds]\n ranklist = qid_list[inds] \n\n return ranklist[:RANKED_LIST_SIZE], cos_sims[:RANKED_LIST_SIZE]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:49:41.165847Z","iopub.execute_input":"2021-07-26T18:49:41.166171Z","iopub.status.idle":"2021-07-26T18:49:41.173897Z","shell.execute_reply.started":"2021-07-26T18:49:41.166136Z","shell.execute_reply":"2021-07-26T18:49:41.172797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_spacy_embedding_similarity_lg(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:49:41.175704Z","iopub.execute_input":"2021-07-26T18:49:41.176016Z","iopub.status.idle":"2021-07-26T18:49:41.321239Z","shell.execute_reply.started":"2021-07-26T18:49:41.175982Z","shell.execute_reply":"2021-07-26T18:49:41.320237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_spacy_embedding_similarity_lg = evaluation_process(method_spacy_embedding_similarity_lg)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:18.085675Z","iopub.execute_input":"2021-07-26T18:38:18.085949Z","iopub.status.idle":"2021-07-26T18:38:19.545128Z","shell.execute_reply.started":"2021-07-26T18:38:18.085922Z","shell.execute_reply":"2021-07-26T18:38:19.54395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gensim WordMover Distance on Boolean Retrieval\n* Applies further sorting by wordmover distance on the output ranklist of Boolean Retrieval \n* Current pre-trained model: `glove-wiki-gigaword-50`\n","metadata":{}},{"cell_type":"code","source":"import gensim\nimport gensim.downloader\n# gensim.downloader.info() # find more models to download\n\nfrom gensim.models import KeyedVectors\n\ntry: model = KeyedVectors.load(\"../input/ir-project-download-keyed-vectors/glove-wiki-gigaword-50.keyedvectors\")\nexcept: # gs_model not downloaded\n model = gensim.downloader.load('glove-wiki-gigaword-50')\n # model.save(\"/kaggle/working/glove-wiki-gigaword-50.keyedvectors\") # if not already saved","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:19.546665Z","iopub.execute_input":"2021-07-26T18:38:19.547117Z","iopub.status.idle":"2021-07-26T18:38:19.759101Z","shell.execute_reply.started":"2021-07-26T18:38:19.54705Z","shell.execute_reply":"2021-07-26T18:38:19.757998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_wordmover_distance(test_qid, model):\n # out of box duplicate finder does not work!\n # returns ranklist and scores of each size RANKED_LIST_SIZE\n \n ## Run baseline model as a filter\n ranklist, scores = method_overlapping_root_word_count(test_qid)\n \n ## Process test question\n test_qn = tokenise_qid(test_qid)\n \n ## Get wordmover distance from every candidate\n distances = []\n qid_list = ranklist\n for candidate_qid in qid_list:\n candidate_qn = tokenise_qid(candidate_qid)\n distances.append(1-model.wmdistance(test_qn, candidate_qn))\n \n ## Sort by distance\n sorted_dist_and_candidate_qid = sorted(zip(distances,qid_list))[::-1]\n sorted_candidate_qid = [qid for _,qid in sorted_dist_and_candidate_qid]\n sorted_dist = [dist for dist,_ in sorted_dist_and_candidate_qid]\n return sorted_candidate_qid[:RANKED_LIST_SIZE], sorted_dist[:RANKED_LIST_SIZE]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:38:19.760675Z","iopub.execute_input":"2021-07-26T18:38:19.761103Z","iopub.status.idle":"2021-07-26T18:38:19.769204Z","shell.execute_reply.started":"2021-07-26T18:38:19.761038Z","shell.execute_reply":"2021-07-26T18:38:19.768293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_wordmover_distance_glovewiki50(test_qid):\n return method_wordmover_distance(test_qid, model)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T20:28:34.822188Z","iopub.execute_input":"2021-07-26T20:28:34.822586Z","iopub.status.idle":"2021-07-26T20:28:34.829189Z","shell.execute_reply.started":"2021-07-26T20:28:34.822535Z","shell.execute_reply":"2021-07-26T20:28:34.827936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_wordmover_distance_glovewiki50(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T20:28:34.997057Z","iopub.execute_input":"2021-07-26T20:28:34.997448Z","iopub.status.idle":"2021-07-26T20:28:35.411719Z","shell.execute_reply.started":"2021-07-26T20:28:34.997414Z","shell.execute_reply":"2021-07-26T20:28:35.410672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_wordmover_distance_glovewiki50 = evaluation_process(method_wordmover_distance_glovewiki50)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T20:28:36.661317Z","iopub.execute_input":"2021-07-26T20:28:36.661693Z","iopub.status.idle":"2021-07-26T20:28:43.198688Z","shell.execute_reply.started":"2021-07-26T20:28:36.661662Z","shell.execute_reply":"2021-07-26T20:28:43.197523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_to_try = ['glove-wiki-gigaword-300', 'glove-twitter-50','word2vec-google-news-300','fasttext-wiki-news-subwords-300']\n\nif not EVALUATING:\n models_to_try = []\n\nfor m in models_to_try:\n print(\"Model: \",m)\n try:\n model = KeyedVectors.load(f\"../input/ir-project-download-keyed-vectors/{m}.keyedvectors\")\n except:\n model = gensim.downloader.load(m)\n\n def method_wordmover_distance_new_model(test_qid):\n return method_wordmover_distance(test_qid, model)\n\n show_sample_query_results(test_query_qids_list[0], *method_wordmover_distance_new_model(test_query_qids_list[0]))\n\n _ = evaluation_process(method_wordmover_distance_new_model)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:27.31592Z","iopub.execute_input":"2021-07-26T18:38:27.316258Z","iopub.status.idle":"2021-07-26T18:38:27.32344Z","shell.execute_reply.started":"2021-07-26T18:38:27.316229Z","shell.execute_reply":"2021-07-26T18:38:27.321971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sentence Embeddings\n\nEach sentence can be embedded as a vector with SentenceTransformer","metadata":{}},{"cell_type":"code","source":"!pip install sentence-transformers > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:27.325039Z","iopub.execute_input":"2021-07-26T18:38:27.325512Z","iopub.status.idle":"2021-07-26T18:38:34.302019Z","shell.execute_reply.started":"2021-07-26T18:38:27.325467Z","shell.execute_reply":"2021-07-26T18:38:34.300671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\nmodel_name = 'bert-base-nli-stsb-mean-tokens'\nmodel_tf = SentenceTransformer(model_name)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:34.303761Z","iopub.execute_input":"2021-07-26T18:38:34.304093Z","iopub.status.idle":"2021-07-26T18:38:35.684624Z","shell.execute_reply.started":"2021-07-26T18:38:34.304044Z","shell.execute_reply":"2021-07-26T18:38:35.67885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = \"bert-base-nli-stsb-mean-tokens\"\nsentence_vectors = np.load(f\"../input/quora-question-pairs-bert-sentence-vectors/sentence_vectors_{model_name}.npy\")\nsentence_vectors = {i:vec for i,vec in enumerate(sentence_vectors)}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:38:35.702553Z","iopub.execute_input":"2021-07-26T18:38:35.703082Z","iopub.status.idle":"2021-07-26T18:38:41.705605Z","shell.execute_reply.started":"2021-07-26T18:38:35.703022Z","shell.execute_reply":"2021-07-26T18:38:41.70457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.spatial.distance import cosine\n\ndef method_sentence_vector(query_qid, method_preliminary=method_overlapping_root_word_count, preliminary_factor=1):\n # method_preliminary can be either of the previous methods\n # recommended method_overlapping_root_word_count, method_boolean, method_tf_idf\n\n qid_list, preliminary_scores = method_preliminary(query_qid)\n \n # sort by cosine similarity\n query_sentence_vector = sentence_vectors[query_qid]\n query_results = [(qid, preliminary_factor*preliminary_score+1-abs(cosine(query_sentence_vector, sentence_vectors[qid])))\n for qid,preliminary_score in zip(qid_list,preliminary_scores)]\n query_results = sorted(query_results, key=lambda x:x[1], reverse=True)[:RANKED_LIST_SIZE]\n \n return [x[0] for x in query_results], [x[1] for x in query_results]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:38:41.707722Z","iopub.execute_input":"2021-07-26T18:38:41.708019Z","iopub.status.idle":"2021-07-26T18:38:41.714853Z","shell.execute_reply.started":"2021-07-26T18:38:41.70799Z","shell.execute_reply":"2021-07-26T18:38:41.714136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_sentence_vector(test_query_qids_list[0], preliminary_factor=0))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:41.715907Z","iopub.execute_input":"2021-07-26T18:38:41.716322Z","iopub.status.idle":"2021-07-26T18:38:41.860214Z","shell.execute_reply.started":"2021-07-26T18:38:41.716292Z","shell.execute_reply":"2021-07-26T18:38:41.858965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_sentence_vector(test_query_qids_list[0], preliminary_factor=1))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:41.861413Z","iopub.execute_input":"2021-07-26T18:38:41.861697Z","iopub.status.idle":"2021-07-26T18:38:41.994171Z","shell.execute_reply.started":"2021-07-26T18:38:41.861668Z","shell.execute_reply":"2021-07-26T18:38:41.993138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_sentence_vector = evaluation_process(method_sentence_vector)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:38:41.995612Z","iopub.execute_input":"2021-07-26T18:38:41.996047Z","iopub.status.idle":"2021-07-26T18:38:43.374043Z","shell.execute_reply.started":"2021-07-26T18:38:41.995998Z","shell.execute_reply":"2021-07-26T18:38:43.372935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Supervised Model","metadata":{}},{"cell_type":"code","source":"SUPERVISED_MODEL_TRAINING_SET_SIZE = 10000\nLOAD_DATA_FOR_SUPERVISED = False\nDIR_DATA_FOR_SUPERVISED = \"./\"\n\nsupervised_query_qids = random.sample(set(qid_to_duplicate_qids.keys()) - set(test_query_qids_list), \n SUPERVISED_MODEL_TRAINING_SET_SIZE)\n\ndef create_supervised_features(qids, testing=True):\n \n kwargs = {\"test_query_qids_list\": qids, \"calculate_metrics\": False, \"use_tqdm\": False}\n method_to_ranklists_scorelists_supervised = {\n \"overlapping_root_word_count\": evaluation_process(method_overlapping_root_word_count, **kwargs),\n \"boolean\": evaluation_process(method_boolean, **kwargs),\n \"tf_idf\": evaluation_process(method_tf_idf, **kwargs),\n \"bm25\": evaluation_process(method_bm25, **kwargs),\n \"unigram\": evaluation_process(method_unigram, **kwargs),\n \"spacy_embedding_similarity\": evaluation_process(method_spacy_embedding_similarity, **kwargs),\n \"spacy_embedding_similarity_lg\": evaluation_process(method_spacy_embedding_similarity_lg, **kwargs),\n \"wordmover_distance_glovewiki50\": evaluation_process(method_wordmover_distance_glovewiki50, **kwargs),\n \"sentence_vector\": evaluation_process(method_sentence_vector, **kwargs),\n }\n return method_to_ranklists_scorelists_supervised\n\ndef parse_ndarray(obj): # https://stackoverflow.com/a/52604722/5894029\n if isinstance(obj, np.ndarray):\n return obj.tolist()\n\nif not LOAD_DATA_FOR_SUPERVISED:\n method_to_ranklists_scorelists_supervised = create_supervised_features(supervised_query_qids)\n\n with open(DIR_DATA_FOR_SUPERVISED+'supervised_query_qids.json', 'w') as f:\n json.dump(supervised_query_qids, f, indent=4, default=parse_ndarray) \n\n with open(DIR_DATA_FOR_SUPERVISED+'method_to_ranklists_scorelists_supervised.json', 'w') as f:\n json.dump(method_to_ranklists_scorelists_supervised, f, indent=4, default=parse_ndarray)\n\nwith open(DIR_DATA_FOR_SUPERVISED+'supervised_query_qids.json') as f:\n supervised_query_qids = json.load(f)\n\nwith open(DIR_DATA_FOR_SUPERVISED+'method_to_ranklists_scorelists_supervised.json') as f:\n method_to_ranklists_scorelists_supervised = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:22.899668Z","iopub.execute_input":"2021-07-26T18:54:22.900113Z","iopub.status.idle":"2021-07-26T18:54:50.283543Z","shell.execute_reply.started":"2021-07-26T18:54:22.900058Z","shell.execute_reply":"2021-07-26T18:54:50.282391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_supervised_features_into_df(method_to_ranklists_scorelists_supervised, training=False, \n supervised_query_qids_set=set(supervised_query_qids)):\n supervised_scores = defaultdict(dict)\n for method, (ranklists, scorelists) in method_to_ranklists_scorelists_supervised.items():\n for supervised_query_qid, ranklist, scorelist in zip(supervised_query_qids, ranklists, scorelists):\n for candidate_qid, score in zip(ranklist, scorelist):\n if training and candidate_qid in supervised_query_qids_set:\n continue\n supervised_scores[supervised_query_qid, candidate_qid][method] = score\n \n df_supervised = pd.DataFrame.from_dict(supervised_scores, orient='index')\n return df_supervised\n\ndef extract_supervised_labels_from_df(df_supervised):\n supervised_labels = [int(candidate_qid in qid_to_duplicate_qids[supervised_query_qid]) \n for supervised_query_qid, candidate_qid in df_supervised.index]\n return supervised_labels\n\ndf_supervised = parse_supervised_features_into_df(method_to_ranklists_scorelists_supervised, training=True)\nsupervised_labels = extract_supervised_labels_from_df(df_supervised)\n\n# extracted and total number of positive labels\nsum(supervised_labels), sum(len(qid_to_duplicate_qids[supervised_query_qid]) for supervised_query_qid in supervised_query_qids)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:50.285226Z","iopub.execute_input":"2021-07-26T18:54:50.285552Z","iopub.status.idle":"2021-07-26T18:54:50.387776Z","shell.execute_reply.started":"2021-07-26T18:54:50.285519Z","shell.execute_reply":"2021-07-26T18:54:50.38693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Logistic Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression(random_state=0, class_weight='balanced').fit(np.nan_to_num(df_supervised.values), supervised_labels)\nfor coef, feature in zip(clf.coef_[0], df_supervised.columns):\n print(\"{:.4f}\".format(coef), feature)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:50.389193Z","iopub.execute_input":"2021-07-26T18:54:50.389618Z","iopub.status.idle":"2021-07-26T18:54:50.553254Z","shell.execute_reply.started":"2021-07-26T18:54:50.389587Z","shell.execute_reply":"2021-07-26T18:54:50.552193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_supervised_model_logr(query_qid):\n df_predict = parse_supervised_features_into_df(create_supervised_features([query_qid]))\n scores = clf.predict_proba(np.nan_to_num(df_predict.values))[:,1]\n candidate_qids = df_predict.reset_index()[\"level_1\"] # resolve dataframe multi-index\n results = sorted(list(zip(scores, candidate_qids)))[::-1]\n return [x[1] for x in results][:RANKED_LIST_SIZE], [x[0] for x in results][:RANKED_LIST_SIZE] # qid, scores","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:50.55541Z","iopub.execute_input":"2021-07-26T18:54:50.556149Z","iopub.status.idle":"2021-07-26T18:54:50.565058Z","shell.execute_reply.started":"2021-07-26T18:54:50.556095Z","shell.execute_reply":"2021-07-26T18:54:50.563939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_supervised_model_logr(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:50.566902Z","iopub.execute_input":"2021-07-26T18:54:50.567646Z","iopub.status.idle":"2021-07-26T18:54:53.316423Z","shell.execute_reply.started":"2021-07-26T18:54:50.567597Z","shell.execute_reply":"2021-07-26T18:54:53.315221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_supervised_model_logr = evaluation_process(method_supervised_model_logr)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:54:53.318043Z","iopub.execute_input":"2021-07-26T18:54:53.31857Z","iopub.status.idle":"2021-07-26T18:55:17.72415Z","shell.execute_reply.started":"2021-07-26T18:54:53.318514Z","shell.execute_reply":"2021-07-26T18:55:17.723129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LightGBM classification","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\ndf_train = df_supervised.copy()\ntarget_train = np.array(supervised_labels)\neval_set = np.array([True if i < len(df_train)*0.2 else False for i in range(len(df_train))])\nlgb_train = lgb.Dataset(df_train[~eval_set], target_train[~eval_set])\nlgb_eval = lgb.Dataset(df_train[eval_set], target_train[eval_set], reference=lgb_train)\nlgb_all = lgb.Dataset(df_train, target_train)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:55:17.725665Z","iopub.execute_input":"2021-07-26T18:55:17.725985Z","iopub.status.idle":"2021-07-26T18:55:17.754704Z","shell.execute_reply.started":"2021-07-26T18:55:17.725954Z","shell.execute_reply":"2021-07-26T18:55:17.753907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n# 'boosting_type': 'gbdt',\n 'objective': 'binary',\n 'monotone_constraints': [1]*len(df_supervised.columns),\n# 'scale_pos_weight': 0.360,\n# 'metric': {'auc'},\n# 'num_leaves': 15,\n# 'learning_rate': 0.05,\n# 'feature_fraction': 0.9,\n# 'bagging_fraction': 0.8,\n# 'bagging_freq': 5,\n 'verbose': -1,\n}\n\ngbm = lgb.train(params,\n lgb_train,\n num_boost_round=1000,\n valid_sets=lgb_eval,\n verbose_eval=-1,\n early_stopping_rounds=10)\n\npd.DataFrame({\"feature\": df_train.columns, \"importance\": gbm.feature_importance(importance_type=\"gain\")})[:20]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:59:59.094564Z","iopub.execute_input":"2021-07-26T18:59:59.094941Z","iopub.status.idle":"2021-07-26T18:59:59.144735Z","shell.execute_reply.started":"2021-07-26T18:59:59.094909Z","shell.execute_reply":"2021-07-26T18:59:59.143586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def method_supervised_model_lgbm(query_qid):\n df_predict = parse_supervised_features_into_df(create_supervised_features([query_qid]))\n scores = gbm.predict(df_predict)\n candidate_qids = df_predict.reset_index()[\"level_1\"]\n results = sorted(list(zip(scores, candidate_qids)))[::-1]\n return [x[1] for x in results][:RANKED_LIST_SIZE], [x[0] for x in results][:RANKED_LIST_SIZE] # qid, scores","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T19:00:10.815808Z","iopub.execute_input":"2021-07-26T19:00:10.816206Z","iopub.status.idle":"2021-07-26T19:00:10.825202Z","shell.execute_reply.started":"2021-07-26T19:00:10.816175Z","shell.execute_reply":"2021-07-26T19:00:10.823966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_sample_query_results(test_query_qids_list[0], *method_supervised_model_lgbm(test_query_qids_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T19:00:11.968747Z","iopub.execute_input":"2021-07-26T19:00:11.969163Z","iopub.status.idle":"2021-07-26T19:00:14.745911Z","shell.execute_reply.started":"2021-07-26T19:00:11.969123Z","shell.execute_reply":"2021-07-26T19:00:14.745006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_supervised_model_lgbm = evaluation_process(method_supervised_model_lgbm)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T19:00:14.747521Z","iopub.execute_input":"2021-07-26T19:00:14.747895Z","iopub.status.idle":"2021-07-26T19:00:39.261549Z","shell.execute_reply.started":"2021-07-26T19:00:14.747861Z","shell.execute_reply":"2021-07-26T19:00:39.260372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation for Hand Evaluation Dataset","metadata":{}},{"cell_type":"code","source":"method_to_ranklists_scorelists = {\n# \"random_guess\": results_random_guess,\n \"overlapping_root_word_count\": results_overlapping_root_word_count,\n \"boolean\": results_boolean,\n \"tf_idf\": results_tf_idf,\n \"bm25\": results_bm25,\n \"unigram\": results_unigram,\n \"spacy_embedding_similarity\": results_spacy_embedding_similarity,\n \"spacy_embedding_similarity_lg\": results_spacy_embedding_similarity_lg,\n \"wordmover_distance_glovewiki50\": results_wordmover_distance_glovewiki50,\n \"sentence_vector\": results_sentence_vector,\n \"supervised_model_logr\": results_supervised_model_logr,\n \"supervised_model_lgbm\": results_supervised_model_lgbm\n}\n\nimport json\n\ndef parse_ndarray(obj): # https://stackoverflow.com/a/52604722/5894029\n if isinstance(obj, np.ndarray):\n return obj.tolist()\n\nwith open('method_to_ranklists_scorelists.json', 'w') as f:\n json.dump(method_to_ranklists_scorelists, f, indent=4, default=parse_ndarray)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.072672Z","iopub.execute_input":"2021-07-26T18:44:11.072983Z","iopub.status.idle":"2021-07-26T18:44:11.124945Z","shell.execute_reply.started":"2021-07-26T18:44:11.072952Z","shell.execute_reply":"2021-07-26T18:44:11.123867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUESTIONS_TO_HANDEVAL = set(x-1 for x in [\n 332, 490, 1955, 6319, 9690, 17279, 19619, 20557, 26378, 33734, 38984, \n 49864, 57291, 89903, 116882, 126992, 131214, 144297, 159628, 201409, \n 273666, 284107, 286721, 312887, 318523, 378759, 384832, 405081, \n 405877, 423313, 464279, 480116, 533401])\nHANDEVAL_RANK_THRESHOLD = 10\n\nmap_qid_to_handeval = defaultdict(set)\n\nfor ranklists, scorelists in method_to_ranklists_scorelists.values():\n for test_qid, ranklist in zip(test_query_qids_list, ranklists):\n if test_qid in QUESTIONS_TO_HANDEVAL:\n for candidate_qid in ranklist[:HANDEVAL_RANK_THRESHOLD]:\n map_qid_to_handeval[test_qid].add(candidate_qid)\n \nfor qid in map_qid_to_handeval:\n map_qid_to_handeval[qid] = sorted(map_qid_to_handeval[qid])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.12635Z","iopub.execute_input":"2021-07-26T18:44:11.12667Z","iopub.status.idle":"2021-07-26T18:44:11.135371Z","shell.execute_reply.started":"2021-07-26T18:44:11.126642Z","shell.execute_reply":"2021-07-26T18:44:11.134356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe_columns = [\"test_qid\", \"test_question\", \"candidate_qid\", \"candidate_question\"]\ndataframe_entries = []\nfor qid in sorted(map_qid_to_handeval.keys()):\n for candidate_qid in map_qid_to_handeval[qid]:\n line_entry = [qid, qid_to_question[qid], candidate_qid, qid_to_question[candidate_qid]]\n dataframe_entries.append(line_entry)\n \nrandom.shuffle(dataframe_entries)\ndataframe_entries = sorted(dataframe_entries, key = lambda x: x[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.13648Z","iopub.execute_input":"2021-07-26T18:44:11.136769Z","iopub.status.idle":"2021-07-26T18:44:11.151847Z","shell.execute_reply.started":"2021-07-26T18:44:11.136739Z","shell.execute_reply":"2021-07-26T18:44:11.150923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_handeval = pd.DataFrame(dataframe_entries, columns=dataframe_columns)\n# labeller columns\ndf_handeval[\"jh\"] = np.nan\ndf_handeval[\"hk\"] = np.nan\ndf_handeval[\"wt\"] = np.nan\n\ndf_handeval.to_csv(\"df_handeval.csv\", index=None)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.152959Z","iopub.execute_input":"2021-07-26T18:44:11.153415Z","iopub.status.idle":"2021-07-26T18:44:11.169951Z","shell.execute_reply.started":"2021-07-26T18:44:11.153385Z","shell.execute_reply":"2021-07-26T18:44:11.168689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate NDCG with Hand Evaluation Dataset\n\nThis calculates NDCG from a snapshot version of `method_to_ranklists_scorelists`, and a hand annotated `df_handeval`\n\nDue to randomness, the `method_to_ranklists_scorelists` may not be reproduced exactly.","metadata":{}},{"cell_type":"code","source":"df_handeval = pd.read_csv(\"../input/quoraquestionpairhandannotateddataset/df_handeval.csv\")\nwith open('../input/quoraquestionpairhandannotateddataset/method_to_ranklists_scorelists.json') as f:\n method_to_ranklists_scorelists = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.171515Z","iopub.execute_input":"2021-07-26T18:44:11.171831Z","iopub.status.idle":"2021-07-26T18:44:11.19977Z","shell.execute_reply.started":"2021-07-26T18:44:11.17179Z","shell.execute_reply":"2021-07-26T18:44:11.198326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\ndef calculate_dcg_at_k(r, k, method=0):\n if method == 0:\n logn = [1.] + [1/math.log(i,2) for i in range(2, k+1)]\n else:\n logn = [1/math.log(i,2) for i in range(2, k+2)]\n \n dcg = 0.\n for gain,disc in zip(r[:k], logn):\n dcg += gain*disc\n return dcg\n\ndef calculate_ndcg_at_k(scores, ref, k=10, method=0):\n denom = calculate_dcg_at_k(ref, k, method=method)\n numer = calculate_dcg_at_k(scores, k, method=method)\n if denom == 0:\n return 0.\n return numer/denom","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.204148Z","iopub.execute_input":"2021-07-26T18:44:11.204818Z","iopub.status.idle":"2021-07-26T18:44:11.214236Z","shell.execute_reply.started":"2021-07-26T18:44:11.20476Z","shell.execute_reply":"2021-07-26T18:44:11.213259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_qid_to_candidate_qid_to_scores = collections.defaultdict(dict)\n\nfor _,row in df_handeval.iterrows():\n test_qid = row[\"test_qid\"]\n candidate_qid = row[\"candidate_qid\"]\n score = row[\"average\"]\n test_qid_to_candidate_qid_to_scores[test_qid][candidate_qid] = score\n \ntest_qid_to_ideal_scores = collections.defaultdict(list)\nfor test_qid, candidate_qid_to_scores in test_qid_to_candidate_qid_to_scores.items():\n ideal_scores = sorted(candidate_qid_to_scores.values())[::-1]\n test_qid_to_ideal_scores[test_qid] = ideal_scores\n\nmethod_to_ndcg_score = collections.defaultdict(list)\ncount_out_of_eval = 0\n\nfor method_name, (ranklists, _) in method_to_ranklists_scorelists.items():\n for test_qid, ranklist in zip(test_query_qids_list, ranklists):\n if test_qid in QUESTIONS_TO_HANDEVAL:\n scores = []\n for candidate_qid in ranklist[:HANDEVAL_RANK_THRESHOLD]:\n if candidate_qid not in test_qid_to_candidate_qid_to_scores[test_qid]:\n scores.append(1)\n print(method_name, len(scores))\n count_out_of_eval += 1\n else:\n scores.append(test_qid_to_candidate_qid_to_scores[test_qid][candidate_qid])\n ref = test_qid_to_ideal_scores[test_qid]\n ndcg_at_k = calculate_ndcg_at_k(scores, ref)\n method_to_ndcg_score[method_name].append(ndcg_at_k)\n\ncount_out_of_eval","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.215449Z","iopub.execute_input":"2021-07-26T18:44:11.215721Z","iopub.status.idle":"2021-07-26T18:44:11.275812Z","shell.execute_reply.started":"2021-07-26T18:44:11.215694Z","shell.execute_reply":"2021-07-26T18:44:11.274692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for method_name, scores in method_to_ndcg_score.items():\n print(method_name)\n print(sum(scores)/len(scores))\n print(\" \".join(f\"{x:.2f}\" for x in scores))\n print()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:44:11.277059Z","iopub.execute_input":"2021-07-26T18:44:11.277358Z","iopub.status.idle":"2021-07-26T18:44:11.286708Z","shell.execute_reply.started":"2021-07-26T18:44:11.27733Z","shell.execute_reply":"2021-07-26T18:44:11.285972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Indexing and Querying of Unseen Questions\n\nThis is probably the Graphical User Interface that we will present","metadata":{}},{"cell_type":"code","source":"def index_unseen_question(unseen_question_text_list):\n unseen_sentence_vectors = model_tf.encode(unseen_question_text_list, show_progress_bar=True)\n qids_new = [time.time() for _ in unseen_question_text_list]\n\n for qid_new, unseen_sentence_vector, unseen_question_text in zip(qids_new, unseen_sentence_vectors, unseen_question_text_list):\n qid_to_question[qid_new] = unseen_question_text\n \n # compute and update word embedding\n token_list = tokenise_then_spellcheck(unseen_question_text)\n qid_to_vec[qid_new] = to_vec(token_list)\n qid_to_vec2[qid_new] = to_vec2(token_list)\n\n # update sentence embedding\n sentence_vectors[qid_new] = unseen_sentence_vector \n\n # update tf-idf\n qid_to_tokens_, token_to_qids_, tf_, df_, L_ = preprocess_vsm(qids_new)\n for qid in qid_to_tokens_:\n qid_to_tokens[qid] = qid_to_tokens_[qid]\n for token in token_to_qids_:\n token_to_qids[token].update(token_to_qids_[token])\n for token in tf_:\n for qid in tf_[token]:\n tf[token][qid] += tf_[token][qid]\n for token in df_:\n df[token] += df_[token]\n for qid in L_:\n L[qid] = L_[qid]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:44:11.287951Z","iopub.execute_input":"2021-07-26T18:44:11.28827Z","iopub.status.idle":"2021-07-26T18:44:11.303548Z","shell.execute_reply.started":"2021-07-26T18:44:11.288232Z","shell.execute_reply":"2021-07-26T18:44:11.302497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def query_unseen_question(unseen_question_text, method):\n qid_new = time.time()\n qid_to_question[qid_new] = unseen_question_text\n \n # update word embedding\n token_list = tokenise_then_spellcheck(unseen_question_text)\n qid_to_vec[qid_new] = to_vec(token_list)\n qid_to_vec2[qid_new] = to_vec2(token_list)\n \n # update sentence embedding\n sentence_vectors[qid_new] = model_tf.encode(unseen_question_text, show_progress_bar=False)\n \n show_sample_query_results(qid_new, *method(qid_new))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-26T18:44:11.304995Z","iopub.execute_input":"2021-07-26T18:44:11.30536Z","iopub.status.idle":"2021-07-26T18:44:11.315311Z","shell.execute_reply.started":"2021-07-26T18:44:11.305326Z","shell.execute_reply":"2021-07-26T18:44:11.314254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of methods, uncomment to select\nmethod = method_random_guess\nmethod = method_overlapping_root_word_count\nmethod = method_boolean\nmethod = method_tf_idf\nmethod = method_bm25\nmethod = method_unigram\nmethod = method_spacy_embedding_similarity\nmethod = method_spacy_embedding_similarity_lg\nmethod = method_wordmover_distance_glovewiki50\nmethod = method_sentence_vector\nmethod = method_supervised_model_logr\n# method = method_supervised_model_lgbm","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:45:53.971921Z","iopub.execute_input":"2021-07-26T18:45:53.972613Z","iopub.status.idle":"2021-07-26T18:45:53.977661Z","shell.execute_reply.started":"2021-07-26T18:45:53.972558Z","shell.execute_reply":"2021-07-26T18:45:53.976901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_unseen_question(\"Why are computer screens dark in color?\", method=method)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T20:11:43.959976Z","iopub.execute_input":"2021-07-26T20:11:43.960457Z","iopub.status.idle":"2021-07-26T20:11:44.860989Z","shell.execute_reply.started":"2021-07-26T20:11:43.960408Z","shell.execute_reply":"2021-07-26T20:11:44.85928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_unseen_question([\n \"Why are computer screens black when unpowered?\",\n \"Why are computer screens manufactured black?\"])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T19:02:45.73164Z","iopub.status.idle":"2021-07-26T19:02:45.732128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_unseen_question(\"Why are computer screens dark in color?\", method=method)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T19:02:45.733129Z","iopub.status.idle":"2021-07-26T19:02:45.733559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## uncomment if you want to reset the indexing\n# qid_to_tokens, token_to_qids = qid_to_tokens_original.copy(), token_to_qids_original.copy()\n# tf, df, L = tf_original.copy(), df_original.copy(), L_original.copy()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T19:02:45.734421Z","iopub.status.idle":"2021-07-26T19:02:45.734861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}