{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is an adaptation of @seshurajup's public notebook.\n\nCheck it out here: https://www.kaggle.com/code/seshurajup/lb-0-11-uspto-single-cpc-query","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Load testing data\ntest_df = pd.read_csv(\"/kaggle/input/uspto-explainable-ai/test.csv\")\ntest_df.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-24T12:21:14.351491Z","iopub.execute_input":"2024-05-24T12:21:14.352382Z","iopub.status.idle":"2024-05-24T12:21:14.378645Z","shell.execute_reply.started":"2024-05-24T12:21:14.352351Z","shell.execute_reply":"2024-05-24T12:21:14.377541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n# Store the unique patents by publication_number\nallunique_pubnums = []\nfor num,chunk in tqdm(enumerate(pd.read_csv(\"/kaggle/input/uspto-explainable-ai/test.csv\",chunksize=1000))):\n    \n    unique_values = pd.unique(chunk.values.ravel())\n    unique_values = list(unique_values)\n    \n    allunique_pubnums.extend(unique_values)\n    allunique_pubnums = list(set(allunique_pubnums))\nlen(allunique_pubnums)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:22:50.327633Z","iopub.execute_input":"2024-05-24T12:22:50.327987Z","iopub.status.idle":"2024-05-24T12:22:50.356318Z","shell.execute_reply.started":"2024-05-24T12:22:50.327963Z","shell.execute_reply":"2024-05-24T12:22:50.355563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load metadata\nmeta_df = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_metadata.parquet\")\nmeta_df.tail(7)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:31:26.801576Z","iopub.execute_input":"2024-05-24T12:31:26.801978Z","iopub.status.idle":"2024-05-24T12:31:26.818044Z","shell.execute_reply.started":"2024-05-24T12:31:26.801949Z","shell.execute_reply":"2024-05-24T12:31:26.816971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter metadata to keep only patents in test\ntest_patent_metadata = meta_df[meta_df['publication_number'].isin(allunique_pubnums)].reset_index(drop=True)\ntest_patent_metadata","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:24:38.088285Z","iopub.execute_input":"2024-05-24T12:24:38.088686Z","iopub.status.idle":"2024-05-24T12:24:39.131855Z","shell.execute_reply.started":"2024-05-24T12:24:38.088656Z","shell.execute_reply":"2024-05-24T12:24:39.130676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count cpc codes by patent\ntest_patent_metadata['total_cpc_codes'] = test_patent_metadata['cpc_codes'].apply(lambda x:len(x))\n\n# Split cpc codes in unique parts\nfor i in range(1,16):\n    test_patent_metadata[f'cpc_codes_{i}'] =  test_patent_metadata['cpc_codes'].apply(lambda x: list(set([f\"{y[0:i]}\" for y in x if len(x) > 0])))\ntest_patent_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:25:47.674616Z","iopub.execute_input":"2024-05-24T12:25:47.674984Z","iopub.status.idle":"2024-05-24T12:25:47.755250Z","shell.execute_reply.started":"2024-05-24T12:25:47.674955Z","shell.execute_reply":"2024-05-24T12:25:47.754487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n# Transform unique cpc parts into dictionary\ntest_patent_metadata_hash = {}\nfor i in tqdm(range(1,16), total=15):\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes_{i}']","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:37:57.869988Z","iopub.execute_input":"2024-05-24T12:37:57.870377Z","iopub.status.idle":"2024-05-24T12:37:58.440046Z","shell.execute_reply.started":"2024-05-24T12:37:57.870346Z","shell.execute_reply":"2024-05-24T12:37:58.438893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm([\"all\"], total=1):\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes']","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:37:59.409725Z","iopub.execute_input":"2024-05-24T12:37:59.410613Z","iopub.status.idle":"2024-05-24T12:37:59.461950Z","shell.execute_reply.started":"2024-05-24T12:37:59.410565Z","shell.execute_reply":"2024-05-24T12:37:59.461011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\n# Select first n cpc from a patent\ndef select_cpc(row, limit):\n    all_codes = []\n    for pubnum in row.values:\n        if pubnum in test_patent_metadata_hash:\n            all_codes.extend(test_patent_metadata_hash[pubnum][f'cpc_codes_all'])\n        if len(all_codes) > 0:\n            break\n    selected_codes = [code.strip(\"/\") for code,freq in Counter(all_codes).most_common(18)]\n    return selected_codes\n\n# Transform cpc codes into a query string\ndef build_cpc_query(cpc_codes):\n    return \" OR \".join(['cpc:'+x.replace(\"\\\\\",\"\") for x in cpc_codes])\n\n\n# Create queries\nqueries = []\nfor index, row in tqdm(test_df.iterrows(), total=len(test_df)):\n    selected_codes = []\n    for i in range(15,0,-1):\n        selected_codes = select_cpc(row, 'all')\n        if len(selected_codes) < 4:\n             break\n    \n    if len(selected_codes) == 0:\n        selected_codes = [\"B\",\"H\"]\n        \n    queries.append(build_cpc_query(selected_codes))\n\nqueries","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:38:33.658367Z","iopub.execute_input":"2024-05-24T12:38:33.658757Z","iopub.status.idle":"2024-05-24T12:38:33.678310Z","shell.execute_reply.started":"2024-05-24T12:38:33.658730Z","shell.execute_reply":"2024-05-24T12:38:33.677168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh_utils\n\n# Transform string to Whoosh query (less than 50 tokens) and ensure catching errors\nqueryValidator = whoosh_utils.QueryValidator()\nfinal_queries = []\nfor query in queries:\n    final_query = query\n    try:\n        queryValidator.validate_query(query)\n        final_query = query\n    except:\n        final_query = \"ti:device\"\n    if whoosh_utils.count_query_tokens(query) > 50:\n        final_query = \"ti:mobile\"\n    final_queries.append(final_query)\nfor query in final_queries[0:15]:\n    print(\"\\n\", whoosh_utils.count_query_tokens(query), query)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:39:14.721648Z","iopub.execute_input":"2024-05-24T12:39:14.722044Z","iopub.status.idle":"2024-05-24T12:39:14.730958Z","shell.execute_reply.started":"2024-05-24T12:39:14.722017Z","shell.execute_reply":"2024-05-24T12:39:14.729474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assign queries to submission\nsub = pd.read_csv(\"/kaggle/input/uspto-explainable-ai/sample_submission.csv\")\nsub['query'] = final_queries\nsub","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:39:46.401605Z","iopub.execute_input":"2024-05-24T12:39:46.401986Z","iopub.status.idle":"2024-05-24T12:39:46.416276Z","shell.execute_reply.started":"2024-05-24T12:39:46.401960Z","shell.execute_reply":"2024-05-24T12:39:46.415051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T12:39:48.813732Z","iopub.execute_input":"2024-05-24T12:39:48.814131Z","iopub.status.idle":"2024-05-24T12:39:48.825128Z","shell.execute_reply.started":"2024-05-24T12:39:48.814099Z","shell.execute_reply":"2024-05-24T12:39:48.824042Z"},"trusted":true},"execution_count":null,"outputs":[]}]}