{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":7731449,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8370701,"sourceType":"datasetVersion","datasetId":4976301},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\ntest_df = pd.read_csv(\"/kaggle/input/uspto-explainable-ai/test.csv\")\ntest_df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-03T23:00:50.009962Z","iopub.execute_input":"2024-05-03T23:00:50.010672Z","iopub.status.idle":"2024-05-03T23:00:51.204797Z","shell.execute_reply.started":"2024-05-03T23:00:50.010633Z","shell.execute_reply":"2024-05-03T23:00:51.203739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nallunique_pubnums = []\nfor num,chunk in tqdm(enumerate(pd.read_csv(\"/kaggle/input/uspto-explainable-ai/test.csv\",chunksize=1000))):\n    \n    unique_values = pd.unique(chunk.values.ravel())\n    unique_values = list(unique_values)\n    \n    allunique_pubnums.extend(unique_values)\n    allunique_pubnums = list(set(allunique_pubnums))\nlen(allunique_pubnums)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:00:51.207161Z","iopub.execute_input":"2024-05-03T23:00:51.207904Z","iopub.status.idle":"2024-05-03T23:00:51.244253Z","shell.execute_reply.started":"2024-05-03T23:00:51.207850Z","shell.execute_reply":"2024-05-03T23:00:51.243052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patent_metadata_df = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_metadata.parquet\")\npatent_metadata_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:00:51.245877Z","iopub.execute_input":"2024-05-03T23:00:51.246539Z","iopub.status.idle":"2024-05-03T23:01:17.863104Z","shell.execute_reply.started":"2024-05-03T23:00:51.246503Z","shell.execute_reply":"2024-05-03T23:01:17.862091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata = patent_metadata_df[patent_metadata_df['publication_number'].isin(allunique_pubnums)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:17.865294Z","iopub.execute_input":"2024-05-03T23:01:17.865619Z","iopub.status.idle":"2024-05-03T23:01:18.912804Z","shell.execute_reply.started":"2024-05-03T23:01:17.865593Z","shell.execute_reply":"2024-05-03T23:01:18.911707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:18.914014Z","iopub.execute_input":"2024-05-03T23:01:18.914319Z","iopub.status.idle":"2024-05-03T23:01:18.936714Z","shell.execute_reply.started":"2024-05-03T23:01:18.914295Z","shell.execute_reply":"2024-05-03T23:01:18.935631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import ast\n# test_patent_metadata['cpc_codes'] = test_patent_metadata['cpc_codes'].apply(ast.l)\ntest_patent_metadata['total_cpc_codes'] = test_patent_metadata['cpc_codes'].apply(lambda x:len(x))","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:18.938059Z","iopub.execute_input":"2024-05-03T23:01:18.938443Z","iopub.status.idle":"2024-05-03T23:01:18.947022Z","shell.execute_reply.started":"2024-05-03T23:01:18.938417Z","shell.execute_reply":"2024-05-03T23:01:18.945746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,16):\n    test_patent_metadata[f'cpc_codes_{i}'] =  test_patent_metadata['cpc_codes'].apply(lambda x: list(set([f\"{y[0:i]}\" for y in x if len(x) > 0])))","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:18.948110Z","iopub.execute_input":"2024-05-03T23:01:18.948434Z","iopub.status.idle":"2024-05-03T23:01:18.994540Z","shell.execute_reply.started":"2024-05-03T23:01:18.948410Z","shell.execute_reply":"2024-05-03T23:01:18.993271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_patent_metadata","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:18.995778Z","iopub.execute_input":"2024-05-03T23:01:18.996109Z","iopub.status.idle":"2024-05-03T23:01:19.082751Z","shell.execute_reply.started":"2024-05-03T23:01:18.996083Z","shell.execute_reply":"2024-05-03T23:01:19.081564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ntest_patent_metadata_hash = {}\nfor i in tqdm(range(1,16), total=15):\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes_{i}']","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:19.084255Z","iopub.execute_input":"2024-05-03T23:01:19.085338Z","iopub.status.idle":"2024-05-03T23:01:19.712329Z","shell.execute_reply.started":"2024-05-03T23:01:19.085290Z","shell.execute_reply":"2024-05-03T23:01:19.710942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm([\"all\"], total=1):\n    for _, row in test_patent_metadata.iterrows():\n        if row['publication_number'] not in test_patent_metadata_hash:\n            test_patent_metadata_hash[row['publication_number']] = {}\n        if f'cpc_codes_{i}' not in test_patent_metadata_hash[row['publication_number']]:\n            test_patent_metadata_hash[row['publication_number']][f'cpc_codes_{i}'] = row[f'cpc_codes']","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:19.716390Z","iopub.execute_input":"2024-05-03T23:01:19.716732Z","iopub.status.idle":"2024-05-03T23:01:19.767965Z","shell.execute_reply.started":"2024-05-03T23:01:19.716706Z","shell.execute_reply":"2024-05-03T23:01:19.766634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\ndef select_cpc(row, limit):\n    all_codes = []\n    for pubnum in row.values:\n        if pubnum in test_patent_metadata_hash:\n            all_codes.extend(test_patent_metadata_hash[pubnum][f'cpc_codes_all'])\n    selected_codes = [code.strip(\"/\") for code,freq in Counter(all_codes).most_common(1)]\n    return selected_codes\n\ndef select_cpc(row, limit):\n    all_codes = []\n    for pubnum in row.values:\n        #print(pubnum)\n        if pubnum in test_patent_metadata_hash:\n            all_codes.extend(test_patent_metadata_hash[pubnum][f'cpc_codes_all'])\n        if len(all_codes) > 0:\n            break\n    selected_codes = [code.strip(\"/\") for code,freq in Counter(all_codes).most_common(1)]\n    return selected_codes\n\ndef build_cpc_query(cpc_codes):\n    #print(cpc_codes)\n    return \" OR \".join(['cpc:'+x.replace(\"\\\\\",\"\") for x in cpc_codes])\n\nqueries = []\nfor index, row in tqdm(test_df.iterrows(), total=len(test_df)):\n    selected_codes = []\n    for i in range(15,0,-1):\n        selected_codes = select_cpc(row, 'all')\n        if len(selected_codes) < 4:\n             break\n    \n    if len(selected_codes) == 0:\n        selected_codes = [\"B\",\"H\"]\n        \n    queries.append(build_cpc_query(selected_codes))","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:19.769188Z","iopub.execute_input":"2024-05-03T23:01:19.769513Z","iopub.status.idle":"2024-05-03T23:01:19.786915Z","shell.execute_reply.started":"2024-05-03T23:01:19.769487Z","shell.execute_reply":"2024-05-03T23:01:19.785364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh_utils\nqueryValidator = whoosh_utils.QueryValidator()\nfinal_queries = []\nfor query in queries:\n    final_query = query\n    try:\n        queryValidator.validate_query(query)\n        final_query = query\n    except:\n        final_query = \"ti:device\"\n    if whoosh_utils.count_query_tokens(query) > 50:\n        final_query = \"ti:mobile\"\n    final_queries.append(final_query)\nfor query in final_queries[0:15]:\n    print(\"\\n\", whoosh_utils.count_query_tokens(query), query)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:19.788090Z","iopub.execute_input":"2024-05-03T23:01:19.788749Z","iopub.status.idle":"2024-05-03T23:01:52.253056Z","shell.execute_reply.started":"2024-05-03T23:01:19.788707Z","shell.execute_reply":"2024-05-03T23:01:52.251721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/uspto-explainable-ai/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.255699Z","iopub.execute_input":"2024-05-03T23:01:52.256096Z","iopub.status.idle":"2024-05-03T23:01:52.266230Z","shell.execute_reply.started":"2024-05-03T23:01:52.256066Z","shell.execute_reply":"2024-05-03T23:01:52.264608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['query'] = final_queries","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.267419Z","iopub.execute_input":"2024-05-03T23:01:52.267768Z","iopub.status.idle":"2024-05-03T23:01:52.274951Z","shell.execute_reply.started":"2024-05-03T23:01:52.267739Z","shell.execute_reply":"2024-05-03T23:01:52.273798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.276326Z","iopub.execute_input":"2024-05-03T23:01:52.276903Z","iopub.status.idle":"2024-05-03T23:01:52.296209Z","shell.execute_reply.started":"2024-05-03T23:01:52.276860Z","shell.execute_reply":"2024-05-03T23:01:52.294864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.297616Z","iopub.execute_input":"2024-05-03T23:01:52.298046Z","iopub.status.idle":"2024-05-03T23:01:52.309257Z","shell.execute_reply.started":"2024-05-03T23:01:52.298017Z","shell.execute_reply":"2024-05-03T23:01:52.308213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import whoosh_utils\n# train_idx = whoosh_utils.load_index('/kaggle/input/uspto-explainable-ai/train_index')\n# searcher = whoosh_utils.get_searcher(train_idx)\n# qp = whoosh_utils.get_query_parser()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.312036Z","iopub.execute_input":"2024-05-03T23:01:52.312702Z","iopub.status.idle":"2024-05-03T23:01:52.317781Z","shell.execute_reply.started":"2024-05-03T23:01:52.312659Z","shell.execute_reply":"2024-05-03T23:01:52.316826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# query = 'cpc:G01N33*'\n# whoosh_utils.execute_query(query, qp, searcher)[:50]","metadata":{"execution":{"iopub.status.busy":"2024-05-03T23:01:52.319289Z","iopub.execute_input":"2024-05-03T23:01:52.319980Z","iopub.status.idle":"2024-05-03T23:01:52.328645Z","shell.execute_reply.started":"2024-05-03T23:01:52.319949Z","shell.execute_reply":"2024-05-03T23:01:52.327617Z"},"trusted":true},"execution_count":null,"outputs":[]}]}