{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":7731449,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8246447,"sourceType":"datasetVersion","datasetId":4892374},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\ntest_df = pd.read_csv(\"/kaggle/input/uspto-explainable-ai-validation-index/neighbors_small.csv\")\ntest_df = test_df[0:2500]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-29T19:20:08.619352Z","iopub.execute_input":"2024-04-29T19:20:08.619761Z","iopub.status.idle":"2024-04-29T19:20:09.986886Z","shell.execute_reply.started":"2024-04-29T19:20:08.619730Z","shell.execute_reply":"2024-04-29T19:20:09.985689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nallunique_pubnums = []\nfor num,chunk in tqdm(enumerate(pd.read_csv(\"/kaggle/input/uspto-explainable-ai-validation-index/neighbors_small.csv\",chunksize=1000))):\n    \n    unique_values = pd.unique(chunk.values.ravel())\n    unique_values = list(unique_values)\n    \n    allunique_pubnums.extend(unique_values)\n    allunique_pubnums = list(set(allunique_pubnums))\nlen(allunique_pubnums)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:09.988801Z","iopub.execute_input":"2024-04-29T19:20:09.989197Z","iopub.status.idle":"2024-04-29T19:20:10.352553Z","shell.execute_reply.started":"2024-04-29T19:20:09.989165Z","shell.execute_reply":"2024-04-29T19:20:10.351402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npatent_metadata_df = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_metadata.parquet\")\npatent_metadata_df","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:10.354320Z","iopub.execute_input":"2024-04-29T19:20:10.354670Z","iopub.status.idle":"2024-04-29T19:20:38.574193Z","shell.execute_reply.started":"2024-04-29T19:20:10.354640Z","shell.execute_reply":"2024-04-29T19:20:38.572986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_patent_metadata = patent_metadata_df[patent_metadata_df['publication_number'].isin(allunique_pubnums)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:38.575931Z","iopub.execute_input":"2024-04-29T19:20:38.576285Z","iopub.status.idle":"2024-04-29T19:20:44.974344Z","shell.execute_reply.started":"2024-04-29T19:20:38.576256Z","shell.execute_reply":"2024-04-29T19:20:44.973127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:44.975583Z","iopub.execute_input":"2024-04-29T19:20:44.975919Z","iopub.status.idle":"2024-04-29T19:20:44.994409Z","shell.execute_reply.started":"2024-04-29T19:20:44.975890Z","shell.execute_reply":"2024-04-29T19:20:44.993112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_patent_metadata['total_cpc_codes'] = test_patent_metadata['cpc_codes'].apply(lambda x:len(x))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:44.996857Z","iopub.execute_input":"2024-04-29T19:20:44.997249Z","iopub.status.idle":"2024-04-29T19:20:45.138099Z","shell.execute_reply.started":"2024-04-29T19:20:44.997209Z","shell.execute_reply":"2024-04-29T19:20:45.136837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ndef make_generic(value, i):\n    prefix = value[0:i]\n    if (len(value)-i+1) > 0:\n        repeat = f\"\\${(len(value)-i+1)}\"\n    else:\n        repeat = \"\"\n    return prefix+repeat\nfor i in tqdm(range(1,16),total=15):\n    test_patent_metadata[f'cpc_codes_{i}'] =  test_patent_metadata['cpc_codes'].apply(lambda x: list(set([make_generic(y,i) for y in x if len(x) > 0])))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:20:45.141066Z","iopub.execute_input":"2024-04-29T19:20:45.141451Z","iopub.status.idle":"2024-04-29T19:21:13.040437Z","shell.execute_reply.started":"2024-04-29T19:20:45.141421Z","shell.execute_reply":"2024-04-29T19:21:13.039049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata['cpc_codes_all'] = test_patent_metadata['cpc_codes']\ntest_patent_metadata","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:21:13.041646Z","iopub.execute_input":"2024-04-29T19:21:13.041953Z","iopub.status.idle":"2024-04-29T19:21:14.250922Z","shell.execute_reply.started":"2024-04-29T19:21:13.041928Z","shell.execute_reply":"2024-04-29T19:21:14.249618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata_hash = {}\nfor i in tqdm(range(1,16),total=15):\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes_{i}']","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:21:14.252296Z","iopub.execute_input":"2024-04-29T19:21:14.252659Z","iopub.status.idle":"2024-04-29T19:24:50.150329Z","shell.execute_reply.started":"2024-04-29T19:21:14.252629Z","shell.execute_reply":"2024-04-29T19:24:50.149232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in [\"all\"]:\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes_{i}']","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:24:50.152034Z","iopub.execute_input":"2024-04-29T19:24:50.152765Z","iopub.status.idle":"2024-04-29T19:25:03.828770Z","shell.execute_reply.started":"2024-04-29T19:24:50.152732Z","shell.execute_reply":"2024-04-29T19:25:03.827800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\ndef select_cpc(row, limit):\n    all_codes = []\n    for pubnum in row.values:\n        if pubnum in test_patent_metadata_hash:\n            all_codes.extend(test_patent_metadata_hash[pubnum][f'cpc_codes_all'])\n    selected_codes = [code.strip(\"/\") for code,freq in Counter(all_codes).most_common(1)]\n    return selected_codes\n\ndef select_cpc(row, limit):\n    all_codes = []\n    for pubnum in row.values:\n        #print(pubnum)\n        if pubnum in test_patent_metadata_hash:\n            all_codes.extend(test_patent_metadata_hash[pubnum][f'cpc_codes_all'])\n        if len(all_codes) > 0:\n            break\n    selected_codes = [code.strip(\"/\") for code,freq in Counter(all_codes).most_common(1)]\n    return selected_codes\n\ndef build_cpc_query(cpc_codes):\n    #print(cpc_codes)\n    return \" OR \".join([  'cpc:'+x.replace(\"\\\\\",\"\") for x in cpc_codes])\n\nqueries = []\nfor index, row in tqdm(test_df.iterrows(), total=len(test_df)):\n    selected_codes = []\n    for i in range(15,0,-1):\n        selected_codes = select_cpc(row, 'all')\n        if len(selected_codes) < 4:\n             break\n    \n    if len(selected_codes) == 0:\n        selected_codes = [\"B\",\"H\"]\n        \n    queries.append(build_cpc_query(selected_codes))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:30:17.127582Z","iopub.execute_input":"2024-04-29T19:30:17.128010Z","iopub.status.idle":"2024-04-29T19:30:17.335625Z","shell.execute_reply.started":"2024-04-29T19:30:17.127981Z","shell.execute_reply":"2024-04-29T19:30:17.334505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for query in queries[0:50]:\n    print(len(query.split(' OR ')),\" ==> \", query)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:30:19.619555Z","iopub.execute_input":"2024-04-29T19:30:19.620239Z","iopub.status.idle":"2024-04-29T19:30:19.627505Z","shell.execute_reply.started":"2024-04-29T19:30:19.620204Z","shell.execute_reply":"2024-04-29T19:30:19.626155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport whoosh_utils\ntrain_idx = whoosh_utils.load_index('/kaggle/input/uspto-explainable-ai-validation-index/validation/validation_index')\nsearcher = whoosh_utils.get_searcher(train_idx)\nqp = whoosh_utils.get_query_parser()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:25:51.468857Z","iopub.execute_input":"2024-04-29T19:25:51.469272Z","iopub.status.idle":"2024-04-29T19:27:20.740670Z","shell.execute_reply.started":"2024-04-29T19:25:51.469242Z","shell.execute_reply":"2024-04-29T19:27:20.738183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npredicted = []\nfor query in tqdm(queries, total=len(queries)):\n    results = whoosh_utils.execute_query(query, qp, searcher)[:50]\n    predicted.append(results)\n    #print(len(results), query)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T19:30:22.608908Z","iopub.execute_input":"2024-04-29T19:30:22.609326Z","iopub.status.idle":"2024-04-29T19:30:24.592320Z","shell.execute_reply.started":"2024-04-29T19:30:22.609297Z","shell.execute_reply":"2024-04-29T19:30:24.590986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}