{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":7731449,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":174185912,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initialization","metadata":{}},{"cell_type":"code","source":"import mlxtend\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport whoosh_utils\nfrom collections import Counter\nfrom mlxtend import frequent_patterns\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\n\nis_kaggle = Path(\"/kaggle\").is_dir()\n\nif is_kaggle:\n    data_root = Path(\"/kaggle/input\")\n    output_root = Path(\"/kaggle/working\")\nelse:\n    data_root = Path.cwd().parent / \"data\"\n    output_root = Path.cwd().parent / \"output\"\n\nmax_query_tokens = 50\n\nprint(f\"{is_kaggle=}\")\nprint(f\"{data_root=}\")\nprint(f\"{output_root=}\")\nprint(f\"{max_query_tokens=}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:28:19.348625Z","start_time":"2024-05-06T16:28:18.440216Z"},"execution":{"iopub.status.busy":"2024-05-06T17:25:03.094294Z","iopub.execute_input":"2024-05-06T17:25:03.094907Z","iopub.status.idle":"2024-05-06T17:25:38.821164Z","shell.execute_reply.started":"2024-05-06T17:25:03.094844Z","shell.execute_reply":"2024-05-06T17:25:38.819938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pl.read_csv(data_root / \"uspto-explainable-ai\" / \"test.csv\")\ndf_test","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:28:19.386049Z","start_time":"2024-05-06T16:28:19.350871Z"},"execution":{"iopub.status.busy":"2024-05-06T17:25:38.824175Z","iopub.execute_input":"2024-05-06T17:25:38.824598Z","iopub.status.idle":"2024-05-06T17:25:38.927361Z","shell.execute_reply.started":"2024-05-06T17:25:38.824546Z","shell.execute_reply":"2024-05-06T17:25:38.925835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Query Builder","metadata":{}},{"cell_type":"code","source":"df_patent_metadata = pl.read_parquet(data_root / \"uspto-explainable-ai\" / \"patent_metadata.parquet\")\ndf_patent_metadata = df_patent_metadata.with_columns(family_id=df_patent_metadata[\"family_id\"].cast(pl.Int32))\ndf_patent_metadata","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:28:23.045972Z","start_time":"2024-05-06T16:28:19.387893Z"},"execution":{"iopub.status.busy":"2024-05-06T17:25:38.928743Z","iopub.execute_input":"2024-05-06T17:25:38.929279Z","iopub.status.idle":"2024-05-06T17:25:46.981688Z","shell.execute_reply.started":"2024-05-06T17:25:38.929246Z","shell.execute_reply":"2024-05-06T17:25:46.980588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"publication_number_index = {v: i for i, v in enumerate(df_patent_metadata[\"publication_number\"].to_list())}\npublication_number_index[\"US-1-A\"], publication_number_index[\"US-2017082634-A1\"]","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:28:30.838285Z","start_time":"2024-05-06T16:28:23.047140Z"},"execution":{"iopub.status.busy":"2024-05-06T17:25:46.983302Z","iopub.execute_input":"2024-05-06T17:25:46.983710Z","iopub.status.idle":"2024-05-06T17:25:56.144715Z","shell.execute_reply.started":"2024-05-06T17:25:46.983673Z","shell.execute_reply":"2024-05-06T17:25:56.143814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"term_counter = Counter()\nterm_counter_patents = 0\n\nstopwords_pattern = \"|\".join(f\"\\\\b{word}\\\\b\" for word in [\n    \"an\", \"are\", \"by\", \"for\", \"if\", \"into\", \"is\", \"no\", \"not\", \"of\", \"on\", \"such\",\n    \"that\", \"the\", \"their\", \"then\", \"there\", \"these\", \"they\", \"this\", \"to\", \"was\", \"will\",\n])\n\ndef add_terms_to_counter(terms: pl.Series, prefix: str) -> None:\n    for term, count in terms.value_counts(parallel=True).rows():\n        if term is not None and term != \"\":\n            term_counter[f\"{prefix}:{term}\"] += count\n\ndef add_texts_to_counter(texts: pl.Series, prefix: str) -> None:\n    terms = texts \\\n        .str.to_lowercase() \\\n        .str.replace_all(fr\"\\d+|{stopwords_pattern}\", \"\") \\\n        .str.split(\" \") \\\n        .list.unique() \\\n        .list.explode()\n\n    add_terms_to_counter(terms, prefix)\n\nadd_terms_to_counter(df_patent_metadata[\"cpc_codes\"].list.explode(), \"cpc\")\n\nfor patent_data_file in tqdm(list((data_root / \"uspto-explainable-ai\" / \"patent_data\").iterdir())):\n    df = pl.read_parquet(patent_data_file, columns=[\"title\"])\n\n    add_texts_to_counter(df[\"title\"], \"ti\")\n    # add_texts_to_counter(df[\"abstract\"], \"ab\")\n    term_counter_patents += df.shape[0]\n\nterm_selectivity = {term: count / term_counter_patents for term, count in term_counter.items()}\n\nmost_common = term_counter.most_common(5)\nmost_common, {term: term_selectivity[term] for term, _ in most_common}","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:17:13.214983Z","start_time":"2024-05-06T17:16:26.435314Z"},"execution":{"iopub.status.busy":"2024-05-06T17:25:56.147453Z","iopub.execute_input":"2024-05-06T17:25:56.148347Z","iopub.status.idle":"2024-05-06T17:27:46.106129Z","shell.execute_reply.started":"2024-05-06T17:25:56.148316Z","shell.execute_reply":"2024-05-06T17:27:46.104963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"whoosh_text_analyzer = whoosh_utils._define_uspto_whoosh_schema()[\"ti\"].analyzer\n\ndef extract_text_tokens(text: str) -> list[str]:\n    return [token.text for token in whoosh_text_analyzer(text)]\n\ndef get_terms_by_publication_number(publication_number: str) -> set[str]:\n    metadata_idx = publication_number_index[publication_number]\n\n    publication_date = df_patent_metadata[\"publication_date\"][metadata_idx]\n    if publication_date is not None:\n        year, month = publication_date.year, publication_date.month\n    else:\n        year, month = \"nan\", \"nan\"\n\n    df_patent_data = pl.scan_parquet(data_root / \"uspto-explainable-ai\" / \"patent_data\" / f\"{year}_{month}.parquet\") \\\n        .filter(pl.col(\"publication_number\") == publication_number) \\\n        .limit(1) \\\n        .select(pl.col([\"title\"])) \\\n        .collect(streaming=True)\n\n    cpc_codes = df_patent_metadata[\"cpc_codes\"][metadata_idx]\n    cpc_terms = [f\"cpc:{code}\" for code in cpc_codes]\n\n    title = df_patent_data[\"title\"][0]\n    title_terms = [f\"ti:{token}\" for token in extract_text_tokens(title)]\n\n    # abstract = df_patent_data[\"abstract\"][0]\n    # abstract_terms = [f\"ab:{token}\" for token in extract_text_tokens(abstract)]\n\n    return set(cpc_terms + title_terms)\n\nget_terms_by_publication_number(\"US-2017082634-A1\")","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:22:24.993056Z","start_time":"2024-05-06T17:22:24.972959Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.107595Z","iopub.execute_input":"2024-05-06T17:27:46.109851Z","iopub.status.idle":"2024-05-06T17:27:46.146354Z","shell.execute_reply.started":"2024-05-06T17:27:46.109815Z","shell.execute_reply":"2024-05-06T17:27:46.145373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_term_group(group: list[str]) -> str:\n    if len(group) == 1:\n        return group[0]\n    else:\n        return \"(\" + \" \".join(group) + \")\"\n\ndef serialize_term_groups(or_groups: list[list[str]], xor_groups: list[list[str]]) -> str:\n    or_subquery = \" OR \".join(map(serialize_term_group, or_groups))\n    xor_subquery = \" XOR \".join(map(serialize_term_group, xor_groups))\n\n    if len(or_groups) > 0 and len(xor_groups) > 0:\n        return f\"({or_subquery}) XOR {xor_subquery}\"\n    elif len(or_groups) > 0:\n        return or_subquery\n    elif len(xor_groups) > 0:\n        return xor_subquery\n    else:\n        return \"\"\n\nserialize_term_groups([[\"a\", \"b\"], [\"c\", \"d\"]], [[\"e\", \"f\"], [\"g\", \"h\", \"i\"]])","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:29:22.333163Z","start_time":"2024-05-06T16:29:22.326434Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.147607Z","iopub.execute_input":"2024-05-06T17:27:46.148622Z","iopub.status.idle":"2024-05-06T17:27:46.159734Z","shell.execute_reply.started":"2024-05-06T17:27:46.148591Z","shell.execute_reply":"2024-05-06T17:27:46.158870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_itemset_selectivity(itemset: frozenset) -> float:\n    selectivity = 1\n\n    for term in itemset:\n        selectivity *= term_selectivity.get(term, 1)\n\n    return selectivity\n\nget_itemset_selectivity(frozenset([\"ti:and\"])), get_itemset_selectivity(frozenset([\"ti:device\"])), get_itemset_selectivity(frozenset([\"ti:and\", \"ti:device\"]))","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:29:22.342423Z","start_time":"2024-05-06T16:29:22.335369Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.161383Z","iopub.execute_input":"2024-05-06T17:27:46.161802Z","iopub.status.idle":"2024-05-06T17:27:46.173457Z","shell.execute_reply.started":"2024-05-06T17:27:46.161764Z","shell.execute_reply":"2024-05-06T17:27:46.172123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_query(row: tuple[str, ...]) -> str:\n    targets = row[1:]\n\n    terms_by_target = list(map(get_terms_by_publication_number, targets))\n    unique_terms = set().union(*terms_by_target)\n\n    if len(unique_terms) == 0:\n        return \"ti:device\"\n\n    df_mlxtend = {term: [] for term in unique_terms}\n    for target_terms in terms_by_target:\n        for term in unique_terms:\n            df_mlxtend[term].append(term in target_terms)\n\n    df_mlxtend = pd.DataFrame(df_mlxtend)\n    df_mlxtend = mlxtend.frequent_patterns.fpgrowth(df_mlxtend, min_support=1 / len(targets) * 1, max_len=2, use_colnames=True)\n    df_mlxtend[\"selectivity\"] = np.round(df_mlxtend[\"itemsets\"].apply(get_itemset_selectivity), 8)\n    df_mlxtend[\"score\"] = df_mlxtend[\"support\"] * df_mlxtend[\"selectivity\"]\n    df_mlxtend = df_mlxtend.sort_values([\"score\", \"support\"], ascending=[True, False])\n\n    groups: list[tuple[list[str], list[int]]] = []\n    target_coverage = [0] * len(targets)\n\n    for row in df_mlxtend.itertuples():\n        group = row.itemsets\n\n        new_query = serialize_term_groups([list(group)], [list(tup[0]) for tup in groups])\n        if whoosh_utils.count_query_tokens(new_query) > max_query_tokens:\n            break\n\n        covered_targets = []\n        covers_new_target = False\n\n        for i, target_terms in enumerate(terms_by_target):\n            if len(group & target_terms) == len(group):\n                covered_targets.append(i)\n                covers_new_target = covers_new_target or target_coverage[i] == 0\n\n        if not covers_new_target:\n            continue\n\n        groups.append((list(group), covered_targets))\n        for target in covered_targets:\n            target_coverage[target] += 1\n\n    or_groups: list[list[str]] = []\n    xor_groups: list[list[str]] = []\n\n    for group, covered_targets in groups:\n        if all(target_coverage[target] == 1 for target in covered_targets):\n            xor_groups.append(group)\n        else:\n            or_groups.append(group)\n\n    if len(or_groups) == 0 and len(xor_groups) == 0:\n        return list(unique_terms)[0]\n\n    return serialize_term_groups(or_groups, xor_groups)\n\ndef build_query_safe(row: tuple[str, ...]) -> str:\n    try:\n        return build_query(row)\n    except:\n        return \"ti:device\"\n\nbuild_query(df_test.row(0))","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:22:36.613427Z","start_time":"2024-05-06T17:22:36.130168Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.174865Z","iopub.execute_input":"2024-05-06T17:27:46.175221Z","iopub.status.idle":"2024-05-06T17:27:46.858118Z","shell.execute_reply.started":"2024-05-06T17:27:46.175194Z","shell.execute_reply":"2024-05-06T17:27:46.857236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"if not is_kaggle:\n    validation_query_parser = whoosh_utils.get_query_parser()\n    validation_index = whoosh_utils.load_index(str(data_root / \"uspto-explainable-ai-validation-index\" / \"validation\" / \"validation_index\"))\n    validation_searcher = whoosh_utils.get_searcher(validation_index)\n\n    def execute_validation_query(query: str) -> list[str]:\n        return whoosh_utils.execute_query(query, validation_query_parser, validation_searcher)\n\n    # https://patents.google.com/patent/US10098246B1\n    print(execute_validation_query(\"ti:display AND ti:system AND ti:and AND ti:method AND ti:use\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:29:30.064089Z","start_time":"2024-05-06T16:29:22.812661Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.859346Z","iopub.execute_input":"2024-05-06T17:27:46.859908Z","iopub.status.idle":"2024-05-06T17:27:46.865698Z","shell.execute_reply.started":"2024-05-06T17:27:46.859863Z","shell.execute_reply":"2024-05-06T17:27:46.864571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_kaggle:\n    df_validation = pl.read_csv(data_root / \"uspto-explainable-ai-validation-index\" / \"neighbors_small.csv\")\nelse:\n    df_validation = None\n\ndf_validation","metadata":{"ExecuteTime":{"end_time":"2024-05-06T16:29:30.097684Z","start_time":"2024-05-06T16:29:30.065716Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.866808Z","iopub.execute_input":"2024-05-06T17:27:46.867189Z","iopub.status.idle":"2024-05-06T17:27:46.880435Z","shell.execute_reply.started":"2024-05-06T17:27:46.867161Z","shell.execute_reply":"2024-05-06T17:27:46.878977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_kaggle:\n    scores = []\n\n    progress = tqdm(total=df_validation.shape[0])\n    for row in df_validation.rows():\n        query = build_query(row)\n        result = set(execute_validation_query(query))\n\n        hits = sum(target in result for target in row[1:])\n        score = hits / (len(row) - 1)\n\n        # progress.write(f\"{score:,.3f} {query}\")\n        scores.append(score)\n\n        progress.set_postfix({\"score\": f\"{np.mean(scores):,.3f}±{np.std(scores):,.3f}\", \"last_score\": f\"{score:,.3f}\"})\n        progress.update()","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:23:51.884611Z","start_time":"2024-05-06T17:22:40.587927Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.882049Z","iopub.execute_input":"2024-05-06T17:27:46.882586Z","iopub.status.idle":"2024-05-06T17:27:46.893001Z","shell.execute_reply.started":"2024-05-06T17:27:46.882509Z","shell.execute_reply":"2024-05-06T17:27:46.891694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pl.DataFrame({\n    \"publication_number\": df_test[\"publication_number\"],\n    \"query\": df_test.map_rows(build_query_safe),\n})\n\nsubmission","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:23:57.250819Z","start_time":"2024-05-06T17:23:54.289786Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:46.894378Z","iopub.execute_input":"2024-05-06T17:27:46.894802Z","iopub.status.idle":"2024-05-06T17:27:51.201612Z","shell.execute_reply.started":"2024-05-06T17:27:46.894773Z","shell.execute_reply":"2024-05-06T17:27:51.200691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_root.mkdir(parents=True, exist_ok=True)\nsubmission.write_csv(output_root / \"submission.csv\")","metadata":{"ExecuteTime":{"end_time":"2024-05-06T17:24:01.316650Z","start_time":"2024-05-06T17:24:01.312111Z"},"execution":{"iopub.status.busy":"2024-05-06T17:27:51.204666Z","iopub.execute_input":"2024-05-06T17:27:51.205435Z","iopub.status.idle":"2024-05-06T17:27:51.213557Z","shell.execute_reply.started":"2024-05-06T17:27:51.205402Z","shell.execute_reply":"2024-05-06T17:27:51.212350Z"},"trusted":true},"execution_count":null,"outputs":[]}]}