{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8323913,"sourceType":"datasetVersion","datasetId":4944579},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":148861315,"sourceType":"kernelVersion"},{"sourceId":148877407,"sourceType":"kernelVersion"},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":176449835,"sourceType":"kernelVersion"},{"sourceId":5111,"sourceType":"modelInstanceVersion","modelInstanceId":3899}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport gc\n\nmeta_df = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_metadata.parquet\")\nprint(meta_df)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-06T08:30:21.500725Z","iopub.execute_input":"2024-06-06T08:30:21.501053Z","iopub.status.idle":"2024-06-06T08:30:43.021810Z","shell.execute_reply.started":"2024-06-06T08:30:21.501030Z","shell.execute_reply":"2024-06-06T08:30:43.020758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/input/uspto-explainable-ai/sample_submission.csv\").drop(\"query\", axis=1)\nprint(sub_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:30:43.023714Z","iopub.execute_input":"2024-06-06T08:30:43.024433Z","iopub.status.idle":"2024-06-06T08:30:43.042122Z","shell.execute_reply.started":"2024-06-06T08:30:43.024395Z","shell.execute_reply":"2024-06-06T08:30:43.041274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df = meta_df[meta_df[\"publication_number\"].isin(sub_df[\"publication_number\"])]\nprint(meta_df)\ngc.collect()\nmeta_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:30:43.043153Z","iopub.execute_input":"2024-06-06T08:30:43.043435Z","iopub.status.idle":"2024-06-06T08:30:46.163568Z","shell.execute_reply.started":"2024-06-06T08:30:43.043411Z","shell.execute_reply":"2024-06-06T08:30:46.162682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patent_df = pd.read_parquet(\"/kaggle/input/uspto-all-patents-after-1975/all_patents.parquet\")\npatent_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:30:46.165800Z","iopub.execute_input":"2024-06-06T08:30:46.166068Z","iopub.status.idle":"2024-06-06T08:31:40.405438Z","shell.execute_reply.started":"2024-06-06T08:30:46.166043Z","shell.execute_reply":"2024-06-06T08:31:40.384751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df = meta_df.merge(patent_df, on=\"publication_number\").reset_index(drop=True)\nmeta_df[\"title\"] = meta_df[\"title\"].fillna(\"\")\nmeta_df[\"abstract\"] = meta_df[\"abstract\"].fillna(\"\")\n\ndel patent_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:31:40.407084Z","iopub.execute_input":"2024-06-06T08:31:40.407397Z","iopub.status.idle":"2024-06-06T08:31:51.471209Z","shell.execute_reply.started":"2024-06-06T08:31:40.407371Z","shell.execute_reply":"2024-06-06T08:31:51.470391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q -U accelerate --no-index --find-links ../input/llm-detect-pip/\n\n!pip install -q -U bitsandbytes --no-index --find-links ../input/llm-detect-pip/\n\n!pip install -q -U transformers --no-index --find-links ../input/llm-detect-p","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:31:51.472491Z","iopub.execute_input":"2024-06-06T08:31:51.472729Z","iopub.status.idle":"2024-06-06T08:32:32.447095Z","shell.execute_reply.started":"2024-06-06T08:31:51.472709Z","shell.execute_reply":"2024-06-06T08:32:32.446142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh_utils\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\nimport torch\n\nmodel_name = '/kaggle/input/mistral/pytorch/7b-v0.1-hf/1'\ntokenizer = AutoTokenizer.from_pretrained(model_name) \n\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit= True,\n    bnb_4bit_quant_type=\"fp4\",\n    bnb_4bit_compute_dtype= torch.float16,\n    bnb_4bit_use_double_quant= False,\n)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n        model_name,\n        #quantization_config=bnb_config,\n        torch_dtype=torch.float16,\n        device_map=\"auto\",\n        trust_remote_code=True,\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:32:32.448502Z","iopub.execute_input":"2024-06-06T08:32:32.448806Z","iopub.status.idle":"2024-06-06T08:34:21.461995Z","shell.execute_reply.started":"2024-06-06T08:32:32.448779Z","shell.execute_reply":"2024-06-06T08:34:21.461134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_keyword(row):\n    response_start = \"Here are the keywords comma separated: \"\n\n    prompt = f'''\n    {row.title}\n    {row.abstract}\n\n    -------\n    Given the patent above, provide several keywords to search similar patents.\n\n    '''\n\n    messages = [\n        {\"role\": \"user\", \"content\": prompt},\n        {\"role\": \"assistant\", \"content\": response_start}\n    ]\n\n    model_inputs = tokenizer.apply_chat_template(messages, return_tensors=\"pt\")\n    model_inputs = model_inputs.to(\"cuda\")\n    model_inputs = model_inputs[:, :-2]\n    \n    generated_ids = model.generate(model_inputs, max_new_tokens=10, \n                                   pad_token_id=tokenizer.eos_token_id,\n                                   do_sample=False, begin_suppress_tokens=[13, 28740])\n\n    decoded = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n    answer = decoded[0].split(response_start.strip())[-1].strip()\n    \n    try:\n        answer = str(eval(answer))\n    except:\n        pass\n    \n    return prompt, answer\n\nanswers = []\n\nfor i, row in tqdm(meta_df.iterrows(), total=meta_df.shape[0]):\n    prompt, answer = gen_keyword(row)\n    answers.append(answer)","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:34:21.463123Z","iopub.execute_input":"2024-06-06T08:34:21.463556Z","iopub.status.idle":"2024-06-06T08:34:28.908830Z","shell.execute_reply.started":"2024-06-06T08:34:21.463529Z","shell.execute_reply":"2024-06-06T08:34:28.907967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport whoosh\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\n\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t\n\nBRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n        'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\n\ncustom_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:34:28.910116Z","iopub.execute_input":"2024-06-06T08:34:28.910467Z","iopub.status.idle":"2024-06-06T08:34:28.917256Z","shell.execute_reply.started":"2024-06-06T08:34:28.910441Z","shell.execute_reply":"2024-06-06T08:34:28.916096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"queries = []\nbackup_query = \"ti:device\"\n\nfor i, row in tqdm(meta_df.iterrows()):\n    cpc = row[\"cpc_codes\"]\n    pn = row[\"publication_number\"]\n    \n    try:\n        tokens = [token.text for token in custom_analyzer(answers[i])]\n        tokens = list(set(tokens))\n        \n        if len(tokens) > 0:\n            cpc  = \"(\" + \" OR \".join(cpc[:15]) + \")\"\n            keywords = \"(\" + \" OR \".join(tokens) + \")\"\n            query = f\"detd:{keywords} AND cpc:{cpc}\"\n        else:\n            # prevent scoring to hang when there is no valid keyword\n            query = f\"cpc:{cpc[0]}\"\n\n        n_tokens = whoosh_utils.count_query_tokens(query)\n        if n_tokens >= 50:\n            query = backup_query\n    except:\n        query = backup_query\n    \n    queries.append(query)","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:34:28.919955Z","iopub.execute_input":"2024-06-06T08:34:28.920266Z","iopub.status.idle":"2024-06-06T08:34:28.934813Z","shell.execute_reply.started":"2024-06-06T08:34:28.920243Z","shell.execute_reply":"2024-06-06T08:34:28.933957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df[\"query\"] = queries\n\nmeta_df.to_csv(\"submission.csv\", index=False, columns=[\"publication_number\", \"query\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:34:28.935925Z","iopub.execute_input":"2024-06-06T08:34:28.936252Z","iopub.status.idle":"2024-06-06T08:34:28.947864Z","shell.execute_reply.started":"2024-06-06T08:34:28.936222Z","shell.execute_reply":"2024-06-06T08:34:28.947067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(meta_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-06T08:48:00.550809Z","iopub.execute_input":"2024-06-06T08:48:00.551210Z","iopub.status.idle":"2024-06-06T08:48:00.561741Z","shell.execute_reply.started":"2024-06-06T08:48:00.551180Z","shell.execute_reply":"2024-06-06T08:48:00.560688Z"},"trusted":true},"execution_count":null,"outputs":[]}]}