{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport polars as pl\nimport glob\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-26T03:27:38.422946Z","iopub.execute_input":"2024-04-26T03:27:38.423768Z","iopub.status.idle":"2024-04-26T03:27:40.464523Z","shell.execute_reply.started":"2024-04-26T03:27:38.423725Z","shell.execute_reply":"2024-04-26T03:27:40.462866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. patent data","metadata":{}},{"cell_type":"code","source":"allpatentdata = []\n\nfor filename in tqdm(glob.glob(\"/kaggle/input/uspto-explainable-ai/patent_data/*\")):\n    allpatentdata.append(pl.read_parquet(filename,columns=[0]))","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:27:40.468449Z","iopub.execute_input":"2024-04-26T03:27:40.469380Z","iopub.status.idle":"2024-04-26T03:28:21.925942Z","shell.execute_reply.started":"2024-04-26T03:27:40.469317Z","shell.execute_reply":"2024-04-26T03:28:21.924385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allpatentdata = pl.concat(allpatentdata)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:28:21.927603Z","iopub.execute_input":"2024-04-26T03:28:21.928053Z","iopub.status.idle":"2024-04-26T03:28:22.254447Z","shell.execute_reply.started":"2024-04-26T03:28:21.928011Z","shell.execute_reply":"2024-04-26T03:28:22.253059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allpatentdata = allpatentdata[\"publication_number\"].to_list()","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:28:22.257443Z","iopub.execute_input":"2024-04-26T03:28:22.257914Z","iopub.status.idle":"2024-04-26T03:28:24.158263Z","shell.execute_reply.started":"2024-04-26T03:28:22.257869Z","shell.execute_reply":"2024-04-26T03:28:24.156708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allpatentdata[:3]","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:26:32.705880Z","iopub.execute_input":"2024-04-26T04:26:32.706469Z","iopub.status.idle":"2024-04-26T04:26:32.715414Z","shell.execute_reply.started":"2024-04-26T04:26:32.706433Z","shell.execute_reply":"2024-04-26T04:26:32.714163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(allpatentdata)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:28:24.160383Z","iopub.execute_input":"2024-04-26T03:28:24.160857Z","iopub.status.idle":"2024-04-26T03:28:24.177077Z","shell.execute_reply.started":"2024-04-26T03:28:24.160817Z","shell.execute_reply":"2024-04-26T03:28:24.175327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. nearest_neighbors","metadata":{}},{"cell_type":"code","source":"allunique_patent = []\n\nfor num,chunk in tqdm(enumerate(pd.read_csv(\"/kaggle/input/uspto-explainable-ai/nearest_neighbors.csv\",chunksize=1000000))):\n    \n    unique_values = pd.unique(chunk.values.ravel())\n    unique_values = list(unique_values)\n    \n    allunique_patent.extend(unique_values)\n    allunique_patent = list(set(allunique_patent))   \n","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:28:24.179503Z","iopub.execute_input":"2024-04-26T03:28:24.179981Z","iopub.status.idle":"2024-04-26T03:56:47.644478Z","shell.execute_reply.started":"2024-04-26T03:28:24.179929Z","shell.execute_reply":"2024-04-26T03:56:47.642114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allunique_patent[:3]","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:26:50.725624Z","iopub.execute_input":"2024-04-26T04:26:50.726412Z","iopub.status.idle":"2024-04-26T04:26:50.740994Z","shell.execute_reply.started":"2024-04-26T04:26:50.726355Z","shell.execute_reply":"2024-04-26T04:26:50.739399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(allunique_patent)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T03:56:47.647394Z","iopub.execute_input":"2024-04-26T03:56:47.647931Z","iopub.status.idle":"2024-04-26T03:56:47.661577Z","shell.execute_reply.started":"2024-04-26T03:56:47.647893Z","shell.execute_reply":"2024-04-26T03:56:47.659886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib_venn import venn2\n\nset1 = set(allpatentdata)\nset2 = set(allunique_patent)\n\nplt.figure(figsize=(8, 8))\nvenn_diagram = venn2([set1, set2], set_labels=('set1', 'set2'))\n\nplt.title(\"Venn Diagram of patent_data and nearest_neighbors.csv\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:30:23.919092Z","iopub.execute_input":"2024-04-26T04:30:23.919620Z","iopub.status.idle":"2024-04-26T04:31:03.743901Z","shell.execute_reply.started":"2024-04-26T04:30:23.919583Z","shell.execute_reply":"2024-04-26T04:31:03.742046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"only_in_set1 = set1 - set2\n\nonly_in_set2 = set2 - set1","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:29:38.061432Z","iopub.execute_input":"2024-04-26T04:29:38.061930Z","iopub.status.idle":"2024-04-26T04:29:47.170581Z","shell.execute_reply.started":"2024-04-26T04:29:38.061894Z","shell.execute_reply":"2024-04-26T04:29:47.169193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.save(only_in_set1,\"only_in_set1.bin\")\ntorch.save(only_in_set2,\"only_in_set2.bin\")","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:24:20.843141Z","iopub.execute_input":"2024-04-26T04:24:20.844125Z","iopub.status.idle":"2024-04-26T04:24:20.857370Z","shell.execute_reply.started":"2024-04-26T04:24:20.844060Z","shell.execute_reply":"2024-04-26T04:24:20.855304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### only in nearest_neighbors.csv","metadata":{}},{"cell_type":"code","source":"only_in_set2","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:20:16.619335Z","iopub.execute_input":"2024-04-26T04:20:16.619804Z","iopub.status.idle":"2024-04-26T04:20:16.630470Z","shell.execute_reply.started":"2024-04-26T04:20:16.619770Z","shell.execute_reply":"2024-04-26T04:20:16.628995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## only in patent_data","metadata":{}},{"cell_type":"code","source":"only_in_set1","metadata":{"execution":{"iopub.status.busy":"2024-04-26T04:20:01.052173Z","iopub.execute_input":"2024-04-26T04:20:01.052674Z","iopub.status.idle":"2024-04-26T04:20:01.090716Z","shell.execute_reply.started":"2024-04-26T04:20:01.052634Z","shell.execute_reply":"2024-04-26T04:20:01.089163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}