{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":7731449,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\n\ndef count_query_tokens(query: str):\n    tokens = [i for i in re.split('[\\s+()]', query) if i]\n    return len(tokens), str(tokens)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-27T05:46:57.669034Z","iopub.execute_input":"2024-04-27T05:46:57.669413Z","iopub.status.idle":"2024-04-27T05:46:57.706585Z","shell.execute_reply.started":"2024-04-27T05:46:57.669384Z","shell.execute_reply":"2024-04-27T05:46:57.705709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:seaGreen;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">▶️ Queries cannot include more than 50 tokens ◀️</div>","metadata":{}},{"cell_type":"code","source":"query = \"(cpc:AO1B33/00)\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.708216Z","iopub.execute_input":"2024-04-27T05:46:57.708881Z","iopub.status.idle":"2024-04-27T05:46:57.716732Z","shell.execute_reply.started":"2024-04-27T05:46:57.708844Z","shell.execute_reply":"2024-04-27T05:46:57.715682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query = \"cpc:AO1B33/00 AND cpc:B60H1/00885\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.718036Z","iopub.execute_input":"2024-04-27T05:46:57.719008Z","iopub.status.idle":"2024-04-27T05:46:57.731568Z","shell.execute_reply.started":"2024-04-27T05:46:57.718972Z","shell.execute_reply":"2024-04-27T05:46:57.730462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query = \"ti:bread ADJ5 ti:cheese\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.734734Z","iopub.execute_input":"2024-04-27T05:46:57.735652Z","iopub.status.idle":"2024-04-27T05:46:57.746926Z","shell.execute_reply.started":"2024-04-27T05:46:57.735587Z","shell.execute_reply":"2024-04-27T05:46:57.745796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query = \"(cpc:AO1B33/00 AND cpc:B60H1/00885) OR (ti:bread ADJ5 ti:cheese)\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.748570Z","iopub.execute_input":"2024-04-27T05:46:57.749312Z","iopub.status.idle":"2024-04-27T05:46:57.762608Z","shell.execute_reply.started":"2024-04-27T05:46:57.749276Z","shell.execute_reply":"2024-04-27T05:46:57.761332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query = \"(cpc:AO1B33/00 AND cpc:B60H1/00885) AND ( (ti:bread ADJ5 ti:cheese) OR (ab:bread OR ab:cheese) )\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.764410Z","iopub.execute_input":"2024-04-27T05:46:57.764846Z","iopub.status.idle":"2024-04-27T05:46:57.779242Z","shell.execute_reply.started":"2024-04-27T05:46:57.764811Z","shell.execute_reply":"2024-04-27T05:46:57.778015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query = \"(cpc:AO1B33/00 AND cpc:B60H1/00885) AND ( (ti:bread ADJ5 ti:cheese) OR (ab:bread OR ab:cheese) ) NOT (clm:method)\"\ncount_query_tokens(query)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.780581Z","iopub.execute_input":"2024-04-27T05:46:57.780926Z","iopub.status.idle":"2024-04-27T05:46:57.796230Z","shell.execute_reply.started":"2024-04-27T05:46:57.780900Z","shell.execute_reply":"2024-04-27T05:46:57.795131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OR, AND, NOT, XOR, ADJ{0-9}, NEAR{0-9}","metadata":{}},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:seaGreen;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">▶️ Maximum tokens => 50 include opertors too, so maximum words in query can be 25-27 ◀️</div>","metadata":{}},{"cell_type":"code","source":"import whoosh_utils\ntrain_idx = whoosh_utils.load_index('/kaggle/input/uspto-explainable-ai/train_index')\nsearcher = whoosh_utils.get_searcher(train_idx)\nqp = whoosh_utils.get_query_parser()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:46:57.797733Z","iopub.execute_input":"2024-04-27T05:46:57.798088Z","iopub.status.idle":"2024-04-27T05:48:19.678753Z","shell.execute_reply.started":"2024-04-27T05:46:57.798061Z","shell.execute_reply":"2024-04-27T05:48:19.677729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:seaGreen;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">▶️ Whoosh ◀️</div>","metadata":{}},{"cell_type":"code","source":"import re\nimport whoosh\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\n\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t\n                \nBRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n        'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\ntext_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:48:19.679904Z","iopub.execute_input":"2024-04-27T05:48:19.680751Z","iopub.status.idle":"2024-04-27T05:48:19.687723Z","shell.execute_reply.started":"2024-04-27T05:48:19.680720Z","shell.execute_reply":"2024-04-27T05:48:19.686668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport pandas as pd\npatent_details = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_data/2020_1.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:48:19.690481Z","iopub.execute_input":"2024-04-27T05:48:19.690889Z","iopub.status.idle":"2024-04-27T05:48:34.185180Z","shell.execute_reply.started":"2024-04-27T05:48:19.690862Z","shell.execute_reply":"2024-04-27T05:48:34.184343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfields = list(patent_details.keys())\nfields","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:48:34.186040Z","iopub.execute_input":"2024-04-27T05:48:34.186400Z","iopub.status.idle":"2024-04-27T05:48:34.194371Z","shell.execute_reply.started":"2024-04-27T05:48:34.186376Z","shell.execute_reply":"2024-04-27T05:48:34.193305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:seaGreen;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">▶️ Parse title, abstract, claims and description -> Tokens ◀️</div>","metadata":{}},{"cell_type":"code","source":"from IPython.core.display import display, HTML\ndef parse_data(section, text):\n    display(HTML(f\"<b style='color:blue'>{section.title()}</b>: <span style='color:blue'>{text}<span>\"))\n    tokens = text_analyzer(text)\n    display(HTML(f\"<b style='color:green'>{section.title()} Tokens</b>: <span style='color:green'>{str([token.text for token in tokens])}</span>\"))\n    \nfor i in range(len(patent_details['publication_number'])):\n    if i > 0:\n        break\n    \n    pubnum = patent_details['publication_number'][i]\n    title = patent_details['title'][i]\n    abstract = patent_details['abstract'][i]\n    claims = patent_details['claims'][i]\n    description = patent_details['description'][i]\n    display(HTML(f\"<br/>Publication number: <b style='color:red'>{pubnum}</b>\"))\n    parse_data('title', title)\n    parse_data('abstract', abstract)\n    parse_data('claims', claims)\n    parse_data('description', description)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:49:53.319492Z","iopub.execute_input":"2024-04-27T05:49:53.320226Z","iopub.status.idle":"2024-04-27T05:49:53.356423Z","shell.execute_reply.started":"2024-04-27T05:49:53.320194Z","shell.execute_reply":"2024-04-27T05:49:53.355263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:seaGreen;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">▶️ TF IDF instead BM25 score ◀️</div>","metadata":{}},{"cell_type":"code","source":"def get_searcher(whoosh_index):\n    return whoosh_index.searcher(weighting=whoosh.scoring.TF_IDF())","metadata":{"execution":{"iopub.status.busy":"2024-04-27T05:48:34.363102Z","iopub.execute_input":"2024-04-27T05:48:34.363413Z","iopub.status.idle":"2024-04-27T05:48:34.369677Z","shell.execute_reply.started":"2024-04-27T05:48:34.363388Z","shell.execute_reply":"2024-04-27T05:48:34.368687Z"},"trusted":true},"execution_count":null,"outputs":[]}]}