{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%reset -sf","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:08.302328Z","iopub.execute_input":"2021-08-03T20:13:08.302793Z","iopub.status.idle":"2021-08-03T20:13:08.405180Z","shell.execute_reply.started":"2021-08-03T20:13:08.302705Z","shell.execute_reply":"2021-08-03T20:13:08.404245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Visualisation\n\nThis notebook is solely use to generate visualisations on the dataset.","metadata":{}},{"cell_type":"code","source":"import os, collections, random, itertools\n\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-03T20:13:08.406697Z","iopub.execute_input":"2021-08-03T20:13:08.406997Z","iopub.status.idle":"2021-08-03T20:13:08.421154Z","shell.execute_reply.started":"2021-08-03T20:13:08.406970Z","shell.execute_reply":"2021-08-03T20:13:08.420065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'): \n    for filename in filenames: print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:08.423140Z","iopub.execute_input":"2021-08-03T20:13:08.423587Z","iopub.status.idle":"2021-08-03T20:13:08.459937Z","shell.execute_reply.started":"2021-08-03T20:13:08.423553Z","shell.execute_reply":"2021-08-03T20:13:08.458914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\ndf = pd.read_csv(\"/kaggle/input/quora-question-pairs/train.csv.zip\")\ndf[\"question1\"] = df[\"question1\"].astype(str)  # resolve nan\ndf[\"question2\"] = df[\"question2\"].astype(str)\ndf[\"qid1\"] = df[\"qid1\"] - 1\ndf[\"qid2\"] = df[\"qid2\"] - 1\nmaxidx = max(max(df[\"qid1\"]), max(df[\"qid2\"])) + 1","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:08.463554Z","iopub.execute_input":"2021-08-03T20:13:08.463871Z","iopub.status.idle":"2021-08-03T20:13:11.332892Z","shell.execute_reply.started":"2021-08-03T20:13:08.463844Z","shell.execute_reply":"2021-08-03T20:13:11.331728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:11.334581Z","iopub.execute_input":"2021-08-03T20:13:11.335025Z","iopub.status.idle":"2021-08-03T20:13:11.372224Z","shell.execute_reply.started":"2021-08-03T20:13:11.334982Z","shell.execute_reply":"2021-08-03T20:13:11.371139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Indexing the questions","metadata":{}},{"cell_type":"code","source":"# all questions are identified with its qid\nqid_to_question = {}\nfor qid1, qid2, question1, question2 in zip(df[\"qid1\"], df[\"qid2\"], df[\"question1\"], df[\"question2\"]):\n    qid_to_question[qid1] = question1\n    qid_to_question[qid2] = question2","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:11.373561Z","iopub.execute_input":"2021-08-03T20:13:11.373868Z","iopub.status.idle":"2021-08-03T20:13:11.984076Z","shell.execute_reply.started":"2021-08-03T20:13:11.373833Z","shell.execute_reply":"2021-08-03T20:13:11.983079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Simple Analysis of the dataset","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:52:53.643176Z","iopub.execute_input":"2021-06-14T14:52:53.643954Z","iopub.status.idle":"2021-06-14T14:52:53.649019Z","shell.execute_reply.started":"2021-06-14T14:52:53.643899Z","shell.execute_reply":"2021-06-14T14:52:53.648171Z"}}},{"cell_type":"code","source":"print(\"Number of questions\", len(qid_to_question))\nprint(\"Number of duplicate pairs\", sum(df[\"is_duplicate\"]))\nprint(\"Percentage of pairs that are duplicate {:.3f}%\".format(sum(df[\"is_duplicate\"])/len(qid_to_question)*100))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:11.985414Z","iopub.execute_input":"2021-08-03T20:13:11.985731Z","iopub.status.idle":"2021-08-03T20:13:12.102739Z","shell.execute_reply.started":"2021-08-03T20:13:11.985702Z","shell.execute_reply":"2021-08-03T20:13:12.101670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qid_to_labelled_qids = collections.defaultdict(set)\nqid_to_duplicate_qids = collections.defaultdict(set)\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    qid_to_labelled_qids[qid1].add(qid2)\n    qid_to_labelled_qids[qid2].add(qid1)\n    if is_duplicate:\n        qid_to_duplicate_qids[qid1].add(qid2)\n        qid_to_duplicate_qids[qid2].add(qid1)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:12.105512Z","iopub.execute_input":"2021-08-03T20:13:12.105804Z","iopub.status.idle":"2021-08-03T20:13:14.126337Z","shell.execute_reply.started":"2021-08-03T20:13:12.105776Z","shell.execute_reply":"2021-08-03T20:13:14.125249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,4))\nplt.title(\"Number of labels for each question, and how many of which are duplicate\")\n\nlabel_sizes = [len(qid_to_labelled_qids[qid]) for qid in qid_to_question]\ncount, bins = np.histogram(label_sizes, bins=range(max(label_sizes)+2))\ncount = count*bins[:-1]   # convert number of groups to population\nplt.bar(bins[1:-1], count[1:], width=1, label=\"total\")\n\nduplicate_sizes = [len(qid_to_duplicate_qids[qid]) for qid in qid_to_question]\ncount, bins = np.histogram(duplicate_sizes, bins=range(max(duplicate_sizes)+2))\ncount = count*bins[:-1]   # convert number of groups to population\nplt.bar(bins[1:-1], count[1:], width=1, label=\"duplicate\")\nplt.xlim(0,50)\nplt.xlabel(\"Count\")\nplt.ylabel(\"Dataset Frequency\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:14.128129Z","iopub.execute_input":"2021-08-03T20:13:14.128463Z","iopub.status.idle":"2021-08-03T20:13:16.273946Z","shell.execute_reply.started":"2021-08-03T20:13:14.128426Z","shell.execute_reply":"2021-08-03T20:13:16.272895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Largest label sizes:\", sorted(label_sizes)[-20:])\nprint(\"Largest duplicate sizes:\", sorted(duplicate_sizes)[-20:])","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:16.275049Z","iopub.execute_input":"2021-08-03T20:13:16.275321Z","iopub.status.idle":"2021-08-03T20:13:16.348913Z","shell.execute_reply.started":"2021-08-03T20:13:16.275295Z","shell.execute_reply":"2021-08-03T20:13:16.347739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Connecting similar questions","metadata":{}},{"cell_type":"code","source":"import typing\n\n\nclass DisjointSet:\n    # https://github.com/not522/ac-library-python/blob/master/atcoder/dsu.py\n    def __init__(self, n: int = 0) -> None:\n        self._n = n\n        self.parent_or_size = [-1] * n\n\n    def union(self, a: int, b: int) -> int:\n        assert 0 <= a < self._n\n        assert 0 <= b < self._n\n\n        x = self.leader(a)\n        y = self.leader(b)\n\n        if x == y:\n            return x\n\n        if -self.parent_or_size[x] < -self.parent_or_size[y]:\n            x, y = y, x\n\n        self.parent_or_size[x] += self.parent_or_size[y]\n        self.parent_or_size[y] = x\n\n        return x\n\n    def same(self, a: int, b: int) -> bool:\n        assert 0 <= a < self._n\n        assert 0 <= b < self._n\n\n        return self.leader(a) == self.leader(b)\n\n    def find(self, a: int) -> int:\n        return self.leader(a)\n    \n    def leader(self, a: int) -> int:\n        assert 0 <= a < self._n\n\n        parent = self.parent_or_size[a]\n        while parent >= 0:\n            if self.parent_or_size[parent] < 0:\n                return parent\n            self.parent_or_size[a], a, parent = (\n                self.parent_or_size[parent],\n                self.parent_or_size[parent],\n                self.parent_or_size[self.parent_or_size[parent]]\n            )\n\n        return a\n\n    def size(self, a: int) -> int:\n        assert 0 <= a < self._n\n\n        return -self.parent_or_size[self.leader(a)]\n\n    def groups(self) -> typing.List[typing.List[int]]:\n        leader_buf = [self.leader(i) for i in range(self._n)]\n\n        result: typing.List[typing.List[int]] = [[] for _ in range(self._n)]\n        for i in range(self._n):\n            result[leader_buf[i]].append(i)\n\n        return list(filter(lambda r: r, result))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:16.350302Z","iopub.execute_input":"2021-08-03T20:13:16.350719Z","iopub.status.idle":"2021-08-03T20:13:16.366949Z","shell.execute_reply.started":"2021-08-03T20:13:16.350689Z","shell.execute_reply":"2021-08-03T20:13:16.365619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all questions are identified with its qid\ndisjoint_set = DisjointSet(maxidx)\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    if is_duplicate:\n        disjoint_set.union(qid1, qid2)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:16.368328Z","iopub.execute_input":"2021-08-03T20:13:16.368771Z","iopub.status.idle":"2021-08-03T20:13:16.892248Z","shell.execute_reply.started":"2021-08-03T20:13:16.368728Z","shell.execute_reply":"2021-08-03T20:13:16.891234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Counting the number of inconsistent nonduplicate labels","metadata":{}},{"cell_type":"code","source":"cnt = 0\nfor qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n    if not is_duplicate:\n        if disjoint_set.find(qid1) == disjoint_set.find(qid2):\n            cnt += 1\n            if cnt < 10:\n                print(qid_to_question[qid1], \"\\n\", qid_to_question[qid2], \"\\n\")\nprint(cnt)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:16.893689Z","iopub.execute_input":"2021-08-03T20:13:16.894235Z","iopub.status.idle":"2021-08-03T20:13:17.486356Z","shell.execute_reply.started":"2021-08-03T20:13:16.894172Z","shell.execute_reply":"2021-08-03T20:13:17.485661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualising the group size of similar questions","metadata":{}},{"cell_type":"code","source":"group_sizes = np.array([len(group) for group in disjoint_set.groups()])\ncount, bins = np.histogram(group_sizes, bins=range(max(group_sizes)+2))\ncount = count*bins[:-1]   # convert number of groups to population\nplt.figure(figsize=(14,4))\nplt.title(\"Group size of similar questions\")\nplt.bar(bins[2:-1], count[2:], width=1)\nplt.xlim(0,50)\nplt.xlabel(\"Group Size\")\nplt.ylabel(\"Dataset Frequency\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:17.487529Z","iopub.execute_input":"2021-08-03T20:13:17.487987Z","iopub.status.idle":"2021-08-03T20:13:19.096773Z","shell.execute_reply.started":"2021-08-03T20:13:17.487955Z","shell.execute_reply":"2021-08-03T20:13:19.096095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Largest group sizes:\", sorted(group_sizes)[-20:])","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:19.097861Z","iopub.execute_input":"2021-08-03T20:13:19.098318Z","iopub.status.idle":"2021-08-03T20:13:19.305916Z","shell.execute_reply.started":"2021-08-03T20:13:19.098288Z","shell.execute_reply":"2021-08-03T20:13:19.304900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Counting the number of augmented connections","metadata":{}},{"cell_type":"code","source":"initial_connection_count = sum(duplicate_sizes)\nfinal_connection_count = sum(group_sizes)\ninitial_connection_count, final_connection_count, final_connection_count - initial_connection_count","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:19.307431Z","iopub.execute_input":"2021-08-03T20:13:19.307762Z","iopub.status.idle":"2021-08-03T20:13:19.498159Z","shell.execute_reply.started":"2021-08-03T20:13:19.307729Z","shell.execute_reply":"2021-08-03T20:13:19.497156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualise distribution of overlapping word count for duplicate pairs","metadata":{}},{"cell_type":"code","source":"# define tokenisation process\n\nimport pickle, functools\nqid_to_tokens_preprocessed_filename = \"../input/quora-question-pairs-tokenise-pipeline/qid_to_processed_token_list_tokenise_then_spellcheck.pkl\"\nwith open(qid_to_tokens_preprocessed_filename, \"rb\") as f:\n    qid_to_tokens_preprocessed = pickle.load(f)\n\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstopword_set = set(stopwords.words())\nstopword_set.update([\"?\"])\n\n@functools.lru_cache(maxsize=None)\ndef tokenise_qid(qid, qid_to_tokens_preprocessed=qid_to_tokens_preprocessed):\n    if qid_to_tokens_preprocessed:\n        return qid_to_tokens_preprocessed[qid]\n    \n    sentence = qid_to_question[qid]\n    return word_tokenize(sentence.lower())","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:19.500284Z","iopub.execute_input":"2021-08-03T20:13:19.500743Z","iopub.status.idle":"2021-08-03T20:13:24.809652Z","shell.execute_reply.started":"2021-08-03T20:13:19.500696Z","shell.execute_reply":"2021-08-03T20:13:24.808543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = disjoint_set.groups()\noverlap_count_duplicate = []\nfor group in tqdm.tqdm(groups):\n    for qid1,qid2 in itertools.combinations(group, r=2):\n        overlapping_tokens = set(tokenise_qid(qid1)) & set(tokenise_qid(qid2))\n        overlapping_tokens = list(token for token in overlapping_tokens if token not in stopword_set)\n        overlap_count_duplicate.append(len(overlapping_tokens))\n\n\noverlap_count_random = []\nsample1 = random.sample(qid_to_question.keys(), 20000)\nsample2 = random.sample(qid_to_question.keys(), 20000)\nfor qid1, qid2 in zip(sample1, sample2):\n    overlapping_tokens = set(tokenise_qid(qid1)) & set(tokenise_qid(qid2))\n    overlapping_tokens = list(token for token in overlapping_tokens if token not in stopword_set)\n    overlap_count_random.append(len(overlapping_tokens))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:24.811225Z","iopub.execute_input":"2021-08-03T20:13:24.811688Z","iopub.status.idle":"2021-08-03T20:13:27.886550Z","shell.execute_reply.started":"2021-08-03T20:13:24.811641Z","shell.execute_reply":"2021-08-03T20:13:27.885432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,4))\nplt.hist(overlap_count_duplicate, bins=range(15), density=True, alpha=0.5, label=\"duplicate pair\")\nplt.hist(overlap_count_random, bins=range(15), density=True, alpha=0.5, label=\"random pair\")\nplt.title(\"Distribution of overlapping non-root word tokens for duplicate pairs and random pairs\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:27.887994Z","iopub.execute_input":"2021-08-03T20:13:27.888445Z","iopub.status.idle":"2021-08-03T20:13:29.658864Z","shell.execute_reply.started":"2021-08-03T20:13:27.888402Z","shell.execute_reply":"2021-08-03T20:13:29.658111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Understand the most frequent non rootword tokens","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstopword_set = set(stopwords.words())","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:29.660179Z","iopub.execute_input":"2021-08-03T20:13:29.660789Z","iopub.status.idle":"2021-08-03T20:13:29.671908Z","shell.execute_reply.started":"2021-08-03T20:13:29.660745Z","shell.execute_reply":"2021-08-03T20:13:29.670961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport random\n\nfrom nltk.corpus import stopwords\nstopword_set = set(stopwords.words())\n\nwith open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_processed_token_list_spellcheck_then_tokenise.pkl\", \"rb\") as f:\n    qid_to_tokens = pickle.load(f)\n\n# with open(\"../input/quora-question-pairs-tokenise-pipeline/token_to_qid_tokenise_then_spellcheck.pkl\", \"rb\") as f:\n#     token_to_qids = pickle.load(f)\n\ntoken_to_qids = collections.defaultdict(set)\nfor qid, tokens in qid_to_tokens.items():\n    for token in tokens:\n        token_to_qids[token].add(qid)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:29.673326Z","iopub.execute_input":"2021-08-03T20:13:29.673746Z","iopub.status.idle":"2021-08-03T20:13:36.281735Z","shell.execute_reply.started":"2021-08-03T20:13:29.673704Z","shell.execute_reply":"2021-08-03T20:13:36.280988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# most common non-stop words, the question mark has been excluded\nsorted([(len(v),k) for k,v in token_to_qids.items() if k not in stopword_set], reverse=True)[:20]","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:36.283016Z","iopub.execute_input":"2021-08-03T20:13:36.283536Z","iopub.status.idle":"2021-08-03T20:13:36.382612Z","shell.execute_reply.started":"2021-08-03T20:13:36.283505Z","shell.execute_reply":"2021-08-03T20:13:36.381707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualise distribution of the number of questions to compare against","metadata":{}},{"cell_type":"code","source":"token_length_sizes = []\nconsidered_set_sizes = []\nfor qid in tqdm.tqdm(random.sample(qid_to_tokens.keys(), 10000)):\n    considered_set = set()\n    for token in qid_to_tokens[qid]:\n        if token in stopword_set:\n            continue\n        if token in token_to_qids:  # some tokens are not found in the token_to_qids (probably from test set)\n            for considered_qid in token_to_qids[token]:\n                considered_set.add(considered_qid)\n    token_length_sizes.append(len(set(qid_to_tokens[qid])))\n    considered_set_sizes.append(len(considered_set))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:13:36.385711Z","iopub.execute_input":"2021-08-03T20:13:36.385994Z","iopub.status.idle":"2021-08-03T20:14:23.941110Z","shell.execute_reply.started":"2021-08-03T20:13:36.385968Z","shell.execute_reply":"2021-08-03T20:14:23.940233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,4))\nplt.hist(considered_set_sizes, bins=np.arange(0,70000,1000), density=True)\nplt.title(\"How many other questions has at least one common non-rootword token\")\nplt.xlabel(\"Query comparison size\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:14:23.942597Z","iopub.execute_input":"2021-08-03T20:14:23.942872Z","iopub.status.idle":"2021-08-03T20:14:24.341793Z","shell.execute_reply.started":"2021-08-03T20:14:23.942845Z","shell.execute_reply":"2021-08-03T20:14:24.340925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,4))\nplt.scatter(considered_set_sizes, token_length_sizes, alpha=0.1)\nplt.title(\"Relationship between number of unique tokens and query comparison size\")\nplt.ylabel(\"Number of unique tokens\")\nplt.xlabel(\"Query comparison size\")\nplt.xlim(None,70000)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:14:24.342882Z","iopub.execute_input":"2021-08-03T20:14:24.343178Z","iopub.status.idle":"2021-08-03T20:14:24.685506Z","shell.execute_reply.started":"2021-08-03T20:14:24.343152Z","shell.execute_reply":"2021-08-03T20:14:24.684474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualise distribution of sentence vectors","metadata":{}},{"cell_type":"code","source":"# model_name = \"bert-base-nli-stsb-mean-tokens\"\n# sentence_vectors = np.load(f\"../input/quora-question-pairs-bert-sentence-vectors/sentence_vectors_{model_name}.npy\")","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:14:24.686700Z","iopub.execute_input":"2021-08-03T20:14:24.687034Z","iopub.status.idle":"2021-08-03T20:14:48.648587Z","shell.execute_reply.started":"2021-08-03T20:14:24.686994Z","shell.execute_reply":"2021-08-03T20:14:48.647595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/quora-question-pairs-tokenise-pipeline/qid_to_vec_trf.pkl\", 'rb') as f:\n    qid_to_vec = pickle.load(f)\nsentence_vectors = []\nfor idx in sorted(qid_to_vec.keys()):\n    sentence_vectors.append(qid_to_vec[idx])\nsentence_vectors = np.array(sentence_vectors)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:18:10.243300Z","iopub.execute_input":"2021-08-03T20:18:10.243676Z","iopub.status.idle":"2021-08-03T20:18:13.030877Z","shell.execute_reply.started":"2021-08-03T20:18:10.243647Z","shell.execute_reply":"2021-08-03T20:18:13.029836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_LARGEST_GROUPS = 1\nlargest_groups = sorted(disjoint_set.groups(), key=len)[-NUM_LARGEST_GROUPS:]\nqids_of_largest_groups = np.array(sum(largest_groups, []))  # flatten","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:20:03.974634Z","iopub.execute_input":"2021-08-03T20:20:03.975041Z","iopub.status.idle":"2021-08-03T20:20:04.726509Z","shell.execute_reply.started":"2021-08-03T20:20:03.975002Z","shell.execute_reply":"2021-08-03T20:20:04.725661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=2)\nsentence_pca = pca.fit_transform(sentence_vectors)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:20:04.727797Z","iopub.execute_input":"2021-08-03T20:20:04.728279Z","iopub.status.idle":"2021-08-03T20:20:10.486178Z","shell.execute_reply.started":"2021-08-03T20:20:04.728231Z","shell.execute_reply":"2021-08-03T20:20:10.485324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import collections  as mc\n\ndef plot_2d_distribution(vectors_2d, qids_to_connect, title=\"\"):\n    qids_to_connect = set(qids_to_connect)\n    plt.figure(figsize=(10,8))\n    plt.scatter(*list(zip(*vectors_2d))[:2], s=1, alpha=0.1)\n    lines = []\n    for qid1, qid2, is_duplicate in zip(df[\"qid1\"], df[\"qid2\"], df[\"is_duplicate\"]):\n        if is_duplicate and qid1 in qids_to_connect and qid2 in qids_to_connect:\n            lines.append([vectors_2d[qid1][:2], vectors_2d[qid2][:2]])\n    lc = mc.LineCollection(lines, color=\"red\", alpha=0.2,\n                           linewidths=[1/(0.1 + (a-c)**2 + (b-d)**2)**0.5 for (a,b),(c,d) in lines])\n    plt.gca().add_collection(lc)\n    plt.title(title)\n\n    mx, my = np.nanmean(vectors_2d, axis=0)\n    sx, sy = np.nanstd(vectors_2d, axis=0)\n    plt.xlim(mx - 4*sx, mx + 4*sx)\n    plt.ylim(my - 4*sy, mx + 4*sy)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:43:20.737685Z","iopub.execute_input":"2021-08-03T20:43:20.738199Z","iopub.status.idle":"2021-08-03T20:43:20.760108Z","shell.execute_reply.started":"2021-08-03T20:43:20.738150Z","shell.execute_reply":"2021-08-03T20:43:20.758656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_2d_distribution(sentence_pca, qids_of_largest_groups, \n                     \"Plot of PCA projection of all word embeddings, {} largest group(s) highlighted\".format(NUM_LARGEST_GROUPS))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:23:28.535646Z","iopub.execute_input":"2021-08-03T20:23:28.536038Z","iopub.status.idle":"2021-08-03T20:23:37.091411Z","shell.execute_reply.started":"2021-08-03T20:23:28.536009Z","shell.execute_reply":"2021-08-03T20:23:37.090299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/DmitryUlyanov/Multicore-TSNE.git > /dev/null && cd Multicore-TSNE/ && pip install . > /dev/null && rm -rf ./Multicore-TSNE ","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:20:20.478080Z","iopub.execute_input":"2021-08-03T20:20:20.478375Z","iopub.status.idle":"2021-08-03T20:20:21.319378Z","shell.execute_reply.started":"2021-08-03T20:20:20.478336Z","shell.execute_reply":"2021-08-03T20:20:21.318330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_LARGEST_GROUPS = 20\nlargest_groups = sorted(disjoint_set.groups(), key=len)[-NUM_LARGEST_GROUPS:]\nqids_of_largest_groups = np.array(sum(largest_groups, []))  # flatten","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:43:24.543341Z","iopub.execute_input":"2021-08-03T20:43:24.543874Z","iopub.status.idle":"2021-08-03T20:43:27.840660Z","shell.execute_reply.started":"2021-08-03T20:43:24.543824Z","shell.execute_reply":"2021-08-03T20:43:27.839184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from MulticoreTSNE import MulticoreTSNE as TSNE\n\n# it was recommended by scipy that we first reduce the dimensions\nqids_to_fit_tsne = np.array(list(set(qids_of_largest_groups) | set(random.sample(qid_to_question.keys(), 20000))))\nsentence_pca = PCA(n_components=50).fit_transform(sentence_vectors[qids_to_fit_tsne])\ntsne = TSNE(n_jobs=4)\nsentence_tsne = np.empty((sentence_vectors.shape[0], 2))\nsentence_tsne[:] = np.nan\nsentence_tsne[qids_to_fit_tsne] = tsne.fit_transform(sentence_pca)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:43:27.842857Z","iopub.execute_input":"2021-08-03T20:43:27.843347Z","iopub.status.idle":"2021-08-03T20:43:38.234201Z","shell.execute_reply.started":"2021-08-03T20:43:27.843297Z","shell.execute_reply":"2021-08-03T20:43:38.232960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_2d_distribution(sentence_tsne, qids_of_largest_groups,\n                     \"Plot of T-SNE projection of {} largest groups and 20000 other questions\".format(NUM_LARGEST_GROUPS))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:43:38.236037Z","iopub.execute_input":"2021-08-03T20:43:38.236502Z","iopub.status.idle":"2021-08-03T20:43:49.933807Z","shell.execute_reply.started":"2021-08-03T20:43:38.236449Z","shell.execute_reply":"2021-08-03T20:43:49.932614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Duplicate questions with largest cosine distance of sentence vectors","metadata":{}},{"cell_type":"code","source":"import itertools\nfrom scipy.spatial.distance import cosine\n\ndistances = []\nfor group in tqdm.tqdm(disjoint_set.groups()):\n    for qid1,qid2 in itertools.combinations(group, r=2):\n        distance = cosine(sentence_vectors[qid1], sentence_vectors[qid2])\n        distances.append((distance,qid1,qid2))\n\ndistances = sorted(distances)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:15:31.218689Z","iopub.status.idle":"2021-08-03T20:15:31.219123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for distance,qid1,qid2 in distances[-10:]:\n    print(f\"Distance: {distance:.2f}\\n{qid_to_question[qid1]}\\n{qid_to_question[qid2]}\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-03T20:15:31.220010Z","iopub.status.idle":"2021-08-03T20:15:31.220452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}