{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8704137,"sourceType":"datasetVersion","datasetId":5220722},{"sourceId":8750992,"sourceType":"datasetVersion","datasetId":5256179},{"sourceId":8751047,"sourceType":"datasetVersion","datasetId":5256229},{"sourceId":9013364,"sourceType":"datasetVersion","datasetId":5374626},{"sourceId":188158247,"sourceType":"kernelVersion"}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%writefile CMakeLists.txt\ncmake_minimum_required(VERSION 3.16)\nproject(uspto-explainable-ai)\n\nset(CMAKE_CXX_STANDARD 17)\nadd_definitions(\n -DANTLR4_USE_THREAD_LOCAL_CACHE=1\n -DIDMAPFN\n -DISR_PATSPEC\n -DPSP_ESTIM\n -DTA_SURR)\n\nfind_package(antlr4-runtime CONFIG REQUIRED)\nfind_package(bshoshany-thread-pool CONFIG REQUIRED)\nfind_package(fast-cpp-csv-parser CONFIG REQUIRED)\nfind_package(fmt CONFIG REQUIRED)\nfind_package(indicators CONFIG REQUIRED)\nfind_package(roaring CONFIG REQUIRED)\nfind_package(spdlog CONFIG REQUIRED)\nfind_package(unordered_dense CONFIG REQUIRED)\n\noption(GRAFANA_ENABLED \"Enable the Grafana integration\" OFF)\nif (GRAFANA_ENABLED)\n add_definitions(-DGRAFANA_ENABLED)\n find_package(libpqxx CONFIG REQUIRED)\nendif ()\n\nfile(GLOB_RECURSE common_sources src/*.cpp vendor/*.c)\nlist(FILTER common_sources EXCLUDE REGEX \"\\/uspto\\/tools\\/\")\n\nset(common_includes src vendor)\nset(common_libraries\n antlr4_static\n bshoshany-thread-pool::bshoshany-thread-pool\n fast-cpp-csv-parser::fast-cpp-csv-parser\n fmt::fmt\n indicators::indicators\n roaring::roaring\n spdlog::spdlog\n unordered_dense::unordered_dense)\n\nif (GRAFANA_ENABLED)\n list(APPEND common_libraries libpqxx::pqxx)\nendif ()\n\nfile(GLOB target_files src/uspto/tools/*.cpp)\nforeach (target_file ${target_files})\n get_filename_component(target_name ${target_file} NAME_WE)\n\n add_executable(${target_name} ${target_file} ${common_sources})\n target_include_directories(${target_name} PRIVATE ${common_includes})\n target_link_libraries(${target_name} PRIVATE ${common_libraries})\nendforeach ()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:08.551385Z","iopub.execute_input":"2024-07-24T22:02:08.551870Z","iopub.status.idle":"2024-07-24T22:02:08.561586Z","shell.execute_reply.started":"2024-07-24T22:02:08.551832Z","shell.execute_reply":"2024-07-24T22:02:08.560302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p src/uspto/tools","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:08.564330Z","iopub.execute_input":"2024-07-24T22:02:08.564799Z","iopub.status.idle":"2024-07-24T22:02:09.748299Z","shell.execute_reply.started":"2024-07-24T22:02:08.564760Z","shell.execute_reply":"2024-07-24T22:02:09.746707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p src/uspto/whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:09.750331Z","iopub.execute_input":"2024-07-24T22:02:09.750801Z","iopub.status.idle":"2024-07-24T22:02:10.921947Z","shell.execute_reply.started":"2024-07-24T22:02:09.750753Z","shell.execute_reply":"2024-07-24T22:02:10.920143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/config.h\n#pragma once\n\n#include \n#include \n#include \n\n#include \n\ninline bool IS_KAGGLE = std::filesystem::exists(\"/kaggle\");\n\ninline std::string getEnv(const std::string& variable) {\n const char* value = std::getenv(variable.c_str());\n if (value == nullptr) {\n spdlog::error(\"{} environment variable is not set\", variable);\n std::exit(1);\n }\n\n return value;\n}\n\ninline std::filesystem::path getPathFromEnv(const std::string& variable) {\n return std::filesystem::current_path() / getEnv(variable);\n}\n\ninline std::filesystem::path getCompetitionDataDirectory() {\n if (IS_KAGGLE) {\n return \"/kaggle/input/uspto-explainable-ai\";\n }\n\n return getPathFromEnv(\"COMPETITION_DATA_DIRECTORY\");\n}\n\ninline std::filesystem::path getValidationDataDirectory() {\n if (IS_KAGGLE) {\n spdlog::error(\"getValidationDataDirectory() is not supported when running on Kaggle\");\n std::exit(1);\n }\n\n return getPathFromEnv(\"VALIDATION_DATA_DIRECTORY\");\n}\n\ninline std::filesystem::path getProjectDirectory() {\n if (IS_KAGGLE) {\n spdlog::error(\"getProjectDirectory() is not supported when running on Kaggle\");\n std::exit(1);\n }\n\n return getPathFromEnv(\"PROJECT_DIRECTORY\");\n}\n\ninline std::filesystem::path getOutputDirectory() {\n if (IS_KAGGLE) {\n return \"/kaggle/working\";\n }\n\n return getPathFromEnv(\"OUTPUT_DIRECTORY\");\n}\n\ninline std::filesystem::path getReformattedPatentDataDirectory() {\n if (IS_KAGGLE) {\n return \"/kaggle/input/uspto-explainable-ai-reformatted-patent-data\";\n }\n\n return getOutputDirectory() / \"patents\";\n}\n\ninline std::filesystem::path getValidationIndexDirectory() {\n if (IS_KAGGLE) {\n spdlog::error(\"getValidationIndexDirectory() is not supported when running on Kaggle\");\n std::exit(1);\n }\n\n return getOutputDirectory() / \"validation-index\";\n}\n\ninline std::filesystem::path getFullIndexDirectory() {\n if (IS_KAGGLE) {\n return \"/kaggle/input/uspto-explainable-ai-full-search-index\";\n }\n\n return getOutputDirectory() / \"full-index\";\n}\n\ninline std::filesystem::path getSubmissionIndexDirectory() {\n return getOutputDirectory() / \"submission-index\";\n}\n\ninline bool isGrafanaEnabled() {\n if (IS_KAGGLE) {\n return false;\n }\n\n return getEnv(\"GRAFANA_ENABLED\") == \"true\";\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:10.924706Z","iopub.execute_input":"2024-07-24T22:02:10.925162Z","iopub.status.idle":"2024-07-24T22:02:10.935703Z","shell.execute_reply.started":"2024-07-24T22:02:10.925120Z","shell.execute_reply":"2024-07-24T22:02:10.934153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/csv.h\n#pragma once\n\n#include \n#include \n#include \n\n#include \n\ntemplate\nvoid readNeighbors(const std::filesystem::path& file, F&& consumer, int maxRows = -1) {\n io::CSVReader<51> in(file);\n\n std::string publicationNumber;\n std::vector neighbors(50);\n\n bool isHeader = true;\n\n while (in.read_row(\n publicationNumber,\n neighbors[0], neighbors[1], neighbors[2], neighbors[3], neighbors[4], neighbors[5], neighbors[6], neighbors[7],\n neighbors[8], neighbors[9], neighbors[10], neighbors[11], neighbors[12], neighbors[13], neighbors[14],\n neighbors[15], neighbors[16], neighbors[17], neighbors[18], neighbors[19], neighbors[20], neighbors[21],\n neighbors[22], neighbors[23], neighbors[24], neighbors[25], neighbors[26], neighbors[27], neighbors[28],\n neighbors[29], neighbors[30], neighbors[31], neighbors[32], neighbors[33], neighbors[34], neighbors[35],\n neighbors[36], neighbors[37], neighbors[38], neighbors[39], neighbors[40], neighbors[41], neighbors[42],\n neighbors[43], neighbors[44], neighbors[45], neighbors[46], neighbors[47], neighbors[48], neighbors[49])) {\n if (isHeader) {\n isHeader = false;\n continue;\n }\n\n consumer(publicationNumber, neighbors);\n\n if (maxRows == -1) {\n continue;\n }\n\n --maxRows;\n if (maxRows <= 0) {\n return;\n }\n }\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:10.940253Z","iopub.execute_input":"2024-07-24T22:02:10.940829Z","iopub.status.idle":"2024-07-24T22:02:10.954898Z","shell.execute_reply.started":"2024-07-24T22:02:10.940778Z","shell.execute_reply":"2024-07-24T22:02:10.953619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/files.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\nclass FileReader {\n std::ifstream in;\n\npublic:\n explicit FileReader(const std::filesystem::path& file) {\n if (!std::filesystem::exists(file)) {\n spdlog::error(\"{} does not exist\", file.generic_string());\n std::exit(1);\n }\n\n in = std::ifstream(file);\n }\n\n std::uint64_t getPosition() {\n return in.tellg();\n }\n\n void seek(std::uint64_t offset) {\n in.seekg(offset, std::ios::beg);\n }\n\n bool isEOF() {\n return in.peek() == std::ifstream::traits_type::eof();\n }\n\n std::vector readRaw(std::uint64_t size) {\n std::vector buffer(size);\n in.read(buffer.data(), size);\n return buffer;\n }\n\n template\n T readScalar() {\n T value;\n in.read(reinterpret_cast(&value), sizeof(T));\n return value;\n }\n\n template\n std::string readString() {\n auto length = readScalar();\n\n std::string str(length, '\\0');\n in.read(str.data(), length);\n\n return str;\n }\n};\n\nclass FileWriter {\n std::ofstream out;\n\npublic:\n explicit FileWriter(const std::filesystem::path& path) {\n if (!std::filesystem::exists(path)) {\n std::filesystem::create_directories(path.parent_path());\n }\n\n out = std::ofstream(path);\n }\n\n std::uint64_t getPosition() {\n return out.tellp();\n }\n\n void writeRaw(const char* buffer, std::uint64_t size) {\n out.write(buffer, size);\n }\n\n template\n void writeScalar(T value) {\n out.write(reinterpret_cast(&value), sizeof(T));\n }\n\n template\n void writeString(const std::string& value) {\n writeScalar(value.length());\n out << value;\n }\n};\n\ntemplate\nclass DataReader : public FileReader {\n std::filesystem::path directory;\n std::shared_ptr> index;\n\npublic:\n explicit DataReader(const std::filesystem::path& directory)\n : FileReader(directory / \"data.bin\"),\n directory(directory),\n index(std::make_shared>()) {\n FileReader indexReader(directory / \"index.bin\");\n\n std::uint64_t offset = 0;\n while (!indexReader.isEOF()) {\n auto key = indexReader.readString();\n offset += indexReader.readScalar();\n\n index->emplace(key, offset);\n }\n }\n\n DataReader(const DataReader& other)\n : FileReader(other.directory / \"data.bin\"), directory(other.directory), index(other.index) {}\n\n void seekToKey(const std::string& key) {\n seek(index->at(key));\n }\n\n std::shared_ptr> getIndex() const {\n return index;\n }\n\n void sortToIndex(std::vector& keys) const {\n std::sort(\n keys.begin(),\n keys.end(),\n [&](const std::string& a, const std::string& b) {\n return index->at(a) < index->at(b);\n });\n }\n};\n\ntemplate\nclass DataWriter : public FileWriter {\n FileWriter indexWriter;\n\n std::uint64_t lastPosition;\n\npublic:\n explicit DataWriter(const std::filesystem::path& directory)\n : FileWriter(directory / \"data.bin\"),\n indexWriter(directory / \"index.bin\"),\n lastPosition(0) {}\n\n void addKey(const std::string& key) {\n auto currentPosition = getPosition();\n auto deltaPosition = currentPosition - lastPosition;\n lastPosition = currentPosition;\n\n indexWriter.writeString(key);\n indexWriter.writeScalar(deltaPosition);\n }\n};\n\nstruct TemporaryDirectory {\n std::filesystem::path path;\n\n TemporaryDirectory() {\n auto rootDirectory = std::filesystem::temp_directory_path();\n\n for (int i = 1;; ++i) {\n path = rootDirectory / fmt::format(\"uspto-{}\", i);\n if (!std::filesystem::exists(path)) {\n std::filesystem::create_directories(path);\n break;\n }\n }\n }\n\n ~TemporaryDirectory() {\n std::filesystem::remove_all(path);\n }\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:10.957197Z","iopub.execute_input":"2024-07-24T22:02:10.957659Z","iopub.status.idle":"2024-07-24T22:02:10.973546Z","shell.execute_reply.started":"2024-07-24T22:02:10.957618Z","shell.execute_reply":"2024-07-24T22:02:10.972114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/generators.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\nextern \"C\" {\n#include \n}\n\nclass QueryGenerator {\npublic:\n virtual ~QueryGenerator() = default;\n\n virtual std::string getName() const = 0;\n\n virtual std::string generateQuery(\n const std::vector& targets,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher) const = 0;\n\n double getQueryScore(Searcher& searcher, const std::string& query, const std::vector& targets) const {\n if (query.empty()) {\n return 0;\n }\n\n auto results = searcher.search(query);\n\n // Adapted from https://www.kaggle.com/competitions/uspto-explainable-ai/discussion/499981#2791642\n double totalScore = 0.0;\n int found = 0;\n\n for (std::size_t i = 0; i < targets.size(); ++i) {\n if (results.contains(targets[i])) {\n ++found;\n }\n\n totalScore += static_cast(found) / static_cast(i + 1);\n }\n\n return totalScore / static_cast(targets.size());\n }\n\nprotected:\n std::string serializeTermGroups(\n const std::vector>& orGroups,\n const std::vector>& xorGroups) const {\n auto orQuery = joinTermGroups(\"OR\", orGroups);\n auto xorQuery = joinTermGroups(\"XOR\", xorGroups);\n\n if (!orGroups.empty() && !xorGroups.empty()) {\n return fmt::format(\"({}) XOR {}\", orQuery, xorQuery);\n }\n\n if (!orGroups.empty()) {\n return orQuery;\n }\n\n if (!xorGroups.empty()) {\n return xorQuery;\n }\n\n return \"\";\n }\n\n std::string joinTermGroups(const std::string& op, const std::vector>& groups) const {\n if (groups.empty()) {\n return \"\";\n }\n\n if (groups.size() == 1) {\n return serializeTermGroup(groups[0]);\n }\n\n std::string out = fmt::format(\"({} {} {})\", serializeTermGroup(groups[0]), op, serializeTermGroup(groups[1]));\n for (std::size_t i = 2; i < groups.size(); ++i) {\n out = fmt::format(\"({} {} {})\", out, op, serializeTermGroup(groups[i]));\n }\n\n return out;\n }\n\n std::string serializeTermGroup(const std::vector& group) const {\n if (group.size() == 1) {\n return group[0];\n }\n\n return fmt::format(\"({})\", fmt::join(group, \" \"));\n }\n};\n\nclass SingleTermQueryGenerator : public QueryGenerator {\n TermCategory::TermCategory category;\n\npublic:\n explicit SingleTermQueryGenerator(TermCategory::TermCategory category)\n : category(category) {}\n\n std::string getName() const override {\n return fmt::format(\"SingleTermQueryGenerator(category={})\", TermCategory::toString(category));\n }\n\n std::string generateQuery(\n const std::vector& targets,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher) const override {\n ankerl::unordered_dense::map counts;\n\n for (const auto& target : targets) {\n for (const auto& term : patentReader.readTerms(target, category)) {\n ++counts[term];\n }\n }\n\n std::string bestTerm = \"\";\n double bestScore = 0.0;\n\n for (const auto& [term, count] : counts) {\n double score = static_cast(count) / searchIndex.getTermSelectivity(term);\n if (score > bestScore) {\n bestTerm = term;\n bestScore = score;\n }\n }\n\n return bestTerm;\n }\n};\n\nclass FPGrowthQueryGenerator : public QueryGenerator {\n struct FPResult {\n std::vector terms;\n int support = std::numeric_limits::lowest();\n double score = std::numeric_limits::max();\n\n FPResult() = default;\n\n FPResult(const std::vector& terms, int support, double score)\n : terms(terms),\n support(support),\n score(score) {}\n\n bool operator<(const FPResult& other) const {\n if (score == other.score) {\n return support > other.support;\n }\n\n return score < other.score;\n }\n };\n\n struct FPState {\n ITEMBASE* ibase;\n SearchIndex& searchIndex;\n\n std::vector results;\n std::size_t resultsSize;\n\n FPState(ITEMBASE* ibase, SearchIndex& searchIndex)\n : ibase(ibase),\n searchIndex(searchIndex),\n results(1000),\n resultsSize(0) {}\n\n void report(const std::vector& terms, int support) {\n double selectivity = searchIndex.getTermSelectivity(terms[0]);\n for (std::size_t i = 1; i < terms.size(); ++i) {\n selectivity *= searchIndex.getTermSelectivity(terms[i]);\n }\n\n double score = static_cast(support) * selectivity;\n\n FPResult result(terms, support, score);\n mmheap::heap_insert_circular(result, results.data(), resultsSize, results.size());\n }\n };\n\n TermCategory::TermCategory categories;\n int minSupport;\n int minGroupSize;\n int maxGroupSize;\n int maxXorGroups;\n\npublic:\n FPGrowthQueryGenerator(\n TermCategory::TermCategory categories,\n int minSupport,\n int minGroupSize,\n int maxGroupSize,\n int maxXorGroups)\n : categories(categories),\n minSupport(minSupport),\n minGroupSize(minGroupSize),\n maxGroupSize(maxGroupSize),\n maxXorGroups(maxXorGroups) {}\n\n std::string getName() const override {\n return fmt::format(\n \"FPGrowthQueryGenerator(categories={}, minSupport={}, minGroupSize={}, maxGroupSize={}, maxXorGroups={})\",\n TermCategory::toString(categories),\n minSupport,\n minGroupSize,\n maxGroupSize,\n maxXorGroups);\n }\n\n std::string generateQuery(\n const std::vector& targets,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher) const override {\n std::vector> termGroups;\n termGroups.reserve(targets.size());\n\n ankerl::unordered_dense::map> termsByTarget;\n termsByTarget.reserve(targets.size());\n\n bool hasTerms = false;\n\n for (const auto& target : targets) {\n auto terms = patentReader.readTerms(target, categories);\n\n termGroups.emplace_back(terms);\n termsByTarget.emplace(target, ankerl::unordered_dense::set(terms.begin(), terms.end()));\n\n hasTerms = hasTerms || !terms.empty();\n }\n\n if (!hasTerms) {\n return \"\";\n }\n\n auto results = runFPGrowth(termGroups, searchIndex);\n\n std::vector>> groups;\n int querySize = 0;\n\n ankerl::unordered_dense::map targetCoverage;\n int uncoveredTargets = 50;\n\n for (const auto& result : results) {\n int newQuerySize = querySize + result.terms.size() + (querySize == 0 ? 0 : 1);\n if (newQuerySize > 50) {\n continue;\n }\n\n ankerl::unordered_dense::set coveredTargets;\n bool coversNewTarget = false;\n\n for (const auto& target : targets) {\n bool covered = true;\n const auto& targetTerms = termsByTarget[target];\n\n for (const auto& term : result.terms) {\n if (!targetTerms.contains(term)) {\n covered = false;\n break;\n }\n }\n\n if (!covered) {\n continue;\n }\n\n coveredTargets.emplace(target);\n coversNewTarget = coversNewTarget || targetCoverage[target] == 0;\n }\n\n if (uncoveredTargets > 0 && !coversNewTarget) {\n continue;\n }\n\n groups.emplace_back(result, coveredTargets);\n for (const auto& target : coveredTargets) {\n if (targetCoverage[target] == 0) {\n --uncoveredTargets;\n }\n\n ++targetCoverage[target];\n }\n\n querySize = newQuerySize;\n }\n\n std::vector> orGroups;\n std::vector> xorGroups;\n\n for (const auto& [result, coveredTargets] : groups) {\n // The Whoosh query parser becomes a lot slower when there are many XOR operators\n if (xorGroups.size() == maxXorGroups) {\n orGroups.emplace_back(result.terms);\n continue;\n }\n\n bool exclusive = true;\n for (const auto& target : coveredTargets) {\n if (targetCoverage[target] != 1) {\n exclusive = false;\n break;\n }\n }\n\n if (exclusive) {\n xorGroups.emplace_back(result.terms);\n } else {\n orGroups.emplace_back(result.terms);\n }\n }\n\n return serializeTermGroups(orGroups, xorGroups);\n }\n\nprivate:\n std::vector runFPGrowth(\n const std::vector>& termGroups,\n SearchIndex& searchIndex) const {\n auto* ibase = ib_create(0, 0);\n auto* tabag = tbg_create(ibase);\n\n for (std::size_t i = 0; i < 50; ++i) {\n for (const auto& term : termGroups[i]) {\n ib_add2ta(ibase, term.c_str());\n }\n\n ib_finta(ibase, 50 - i);\n tbg_add(tabag, nullptr);\n\n ib_clear(ibase);\n }\n\n auto* fpg = fpg_create(\n FPG_FREQUENT,\n minSupport,\n 100.0,\n 100.0,\n minGroupSize,\n maxGroupSize,\n FPG_LDRATIO,\n FPG_NONE,\n 1.0,\n FPG_COMPLEX,\n FPG_DEFAULT);\n\n fpg_data(fpg, tabag, 0, 2);\n auto* report = isr_create(ibase);\n\n FPState state(ibase, searchIndex);\n isr_setrepo(\n report,\n [](isreport* rep, void* data) {\n auto* state = static_cast(data);\n\n std::vector terms;\n terms.reserve(isr_cnt(rep));\n for (int i = 0; i < isr_cnt(rep); ++i) {\n terms.emplace_back(ib_name(state->ibase, isr_itemx(rep, i)));\n }\n\n state->report(terms, isr_supp(rep));\n },\n &state);\n\n fpg_report(fpg, report);\n fpg_mine(fpg, 0, 0);\n fpg_delete(fpg, 1);\n\n std::vector results;\n results.reserve(state.resultsSize);\n while (state.resultsSize > 0) {\n results.emplace_back(mmheap::heap_remove_min(state.results.data(), state.resultsSize));\n }\n\n return results;\n }\n};\n\nclass OptimizingQueryGenerator : public QueryGenerator {\n TermCategory::TermCategory categories;\n double timeout;\n int maxXorGroups;\n\n struct Group {\n std::size_t id;\n std::vector targets;\n std::vector> availableTerms;\n\n std::vector selectedTerms;\n double selectivity = 1.0;\n\n Group(\n std::size_t id,\n const std::vector& targets,\n const std::vector>& availableTerms)\n : id(id),\n targets(targets),\n availableTerms(availableTerms) {}\n };\n\n struct Action {\n virtual ~Action() = default;\n\n virtual void apply(std::vector>& targetGroups) = 0;\n };\n\n struct SplitToNewGroupAction : Action {\n std::size_t groupIdx;\n std::size_t targetIdx;\n\n SplitToNewGroupAction(std::size_t groupIdx, std::size_t targetIdx)\n : groupIdx(groupIdx),\n targetIdx(targetIdx) {}\n\n void apply(std::vector>& targetGroups) override {\n targetGroups.emplace_back(std::vector({targetGroups[groupIdx][targetIdx]}));\n auto& group = targetGroups[groupIdx];\n group.erase(group.begin() + targetIdx);\n }\n };\n\n struct MoveToNewGroupAction : Action {\n std::size_t groupIdx1;\n std::size_t groupIdx2;\n std::size_t targetIdx;\n\n MoveToNewGroupAction(std::size_t groupIdx1, std::size_t groupIdx2, std::size_t targetIdx)\n : groupIdx1(groupIdx1),\n groupIdx2(groupIdx2),\n targetIdx(targetIdx) {}\n\n void apply(std::vector>& targetGroups) override {\n auto& oldGroup = targetGroups[groupIdx1];\n auto& newGroup = targetGroups[groupIdx2];\n\n newGroup.emplace_back(oldGroup[targetIdx]);\n oldGroup.erase(oldGroup.begin() + targetIdx);\n }\n };\n\n struct MergeGroupsAction : Action {\n std::size_t groupIdx1;\n std::size_t groupIdx2;\n\n MergeGroupsAction(std::size_t groupIdx1, std::size_t groupIdx2)\n : groupIdx1(groupIdx1),\n groupIdx2(groupIdx2) {}\n\n void apply(std::vector>& targetGroups) override {\n auto& group1 = targetGroups[groupIdx1];\n const auto& group2 = targetGroups[groupIdx2];\n\n group1.insert(group1.end(), group2.begin(), group2.end());\n targetGroups.erase(targetGroups.begin() + groupIdx2);\n }\n };\n\npublic:\n OptimizingQueryGenerator(TermCategory::TermCategory categories, double timeout, int maxXorGroups)\n : categories(categories),\n timeout(timeout),\n maxXorGroups(maxXorGroups) {}\n\n std::string getName() const override {\n return fmt::format(\n \"OptimizingQueryGenerator(categories={}, timeout={}, maxXorGroups={})\",\n TermCategory::toString(categories),\n timeout,\n maxXorGroups);\n }\n\n std::string generateQuery(\n const std::vector& targets,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher) const override {\n std::vector> termsByTarget;\n termsByTarget.reserve(targets.size());\n for (const auto& target : targets) {\n auto terms = patentReader.readTerms(target, categories);\n termsByTarget.emplace_back(terms.begin(), terms.end());\n }\n\n std::vector> targetGroups(1);\n for (std::size_t i = 0; i < targets.size(); ++i) {\n targetGroups[0].emplace_back(i);\n }\n\n std::string bestQuery = createQuery(targetGroups, termsByTarget, searchIndex);\n auto maxScore = getQueryScore(searcher, bestQuery, targets);\n\n Timer timer;\n while (timer.elapsedSeconds() < timeout) {\n bool foundImprovement = false;\n\n for (const auto& action : getActions(targetGroups)) {\n auto newTargetGroups = targetGroups;\n action->apply(newTargetGroups);\n\n auto query = createQuery(newTargetGroups, termsByTarget, searchIndex);\n auto score = getQueryScore(searcher, query, targets);\n\n if (score > maxScore) {\n bestQuery = query;\n maxScore = score;\n targetGroups = newTargetGroups;\n foundImprovement = true;\n break;\n }\n }\n\n if (!foundImprovement) {\n break;\n }\n }\n\n return bestQuery;\n }\n\nprivate:\n std::vector> getActions(const std::vector>& targetGroups) const {\n std::vector> actions;\n\n for (std::size_t groupIdx1 = 0; groupIdx1 < targetGroups.size(); ++groupIdx1) {\n for (std::size_t groupIdx2 = groupIdx1 + 1; groupIdx2 < targetGroups.size(); ++groupIdx2) {\n actions.emplace_back(std::make_unique(groupIdx1, groupIdx2));\n }\n\n const auto& group1 = targetGroups[groupIdx1];\n if (group1.size() == 1) {\n continue;\n }\n\n for (std::size_t targetIdx = 0; targetIdx < group1.size(); ++targetIdx) {\n actions.emplace_back(std::make_unique(groupIdx1, targetIdx));\n\n for (std::size_t groupIdx2 = 0; groupIdx2 < targetGroups.size(); ++groupIdx2) {\n if (groupIdx1 != groupIdx2) {\n actions.emplace_back(\n std::make_unique(groupIdx1, groupIdx2, targetIdx));\n }\n }\n }\n }\n\n return actions;\n }\n\n std::string createQuery(\n const std::vector>& targetGroups,\n const std::vector>& termsByTarget,\n SearchIndex& searchIndex) const {\n std::vector groups;\n groups.reserve(targetGroups.size());\n\n for (const auto& targetGroup : targetGroups) {\n auto sharedTerms = termsByTarget[targetGroup[0]];\n\n for (std::size_t i = 1; i < targetGroup.size(); ++i) {\n const auto& targetTerms = termsByTarget[targetGroup[i]];\n\n std::set intersection;\n std::set_intersection(\n sharedTerms.begin(),\n sharedTerms.end(),\n targetTerms.begin(),\n targetTerms.end(),\n std::inserter(intersection, intersection.begin()));\n\n sharedTerms = intersection;\n }\n\n std::vector> availableTerms;\n\n for (const auto& term : sharedTerms) {\n double selectivity = searchIndex.getTermSelectivity(term);\n if (selectivity < 0.01) {\n availableTerms.emplace_back(term, selectivity);\n }\n }\n\n std::sort(\n availableTerms.begin(),\n availableTerms.end(),\n [](const std::pair& a, const std::pair& b) {\n return a.second > b.second;\n });\n\n groups.emplace_back(groups.size(), targetGroup, availableTerms);\n }\n\n int tokensRemaining = 50;\n while (tokensRemaining > 0) {\n int bestGroup = -1;\n std::size_t minTermCount = 100;\n double maxSelectivity = -1;\n int bestGroupRequiredTokens = -1;\n\n for (int i = 0; i < groups.size(); ++i) {\n const auto& group = groups[i];\n\n int requiredTokens = 1;\n if (tokensRemaining < 50 && group.selectedTerms.empty()) {\n requiredTokens = 2;\n }\n\n if (requiredTokens > tokensRemaining) {\n continue;\n }\n\n if (group.availableTerms.empty()) {\n continue;\n }\n\n std::size_t termCount = group.selectedTerms.size();\n double selectivity = group.selectivity;\n\n if (termCount < minTermCount || (termCount == minTermCount && selectivity > maxSelectivity)) {\n bestGroup = i;\n minTermCount = termCount;\n maxSelectivity = selectivity;\n bestGroupRequiredTokens = requiredTokens;\n }\n }\n\n if (bestGroup == -1) {\n break;\n }\n\n auto& group = groups[bestGroup];\n\n const auto& newTerm = group.availableTerms.back();\n group.selectedTerms.emplace_back(newTerm.first);\n group.selectivity *= newTerm.second;\n\n group.availableTerms.pop_back();\n tokensRemaining -= bestGroupRequiredTokens;\n }\n\n std::sort(\n groups.begin(),\n groups.end(),\n [](const Group& a, const Group& b) {\n return a.selectivity > b.selectivity;\n });\n\n std::vector> orGroups;\n std::vector> xorGroups;\n\n for (const auto& group : groups) {\n if (group.selectedTerms.empty()) {\n continue;\n }\n\n // The Whoosh query parser becomes a lot slower when there are many XOR operators\n if (xorGroups.size() == maxXorGroups) {\n orGroups.emplace_back(group.selectedTerms);\n continue;\n }\n\n bool exclusive = true;\n for (const auto& otherGroup : groups) {\n if (group.id == otherGroup.id) {\n continue;\n }\n\n for (const auto& target : otherGroup.targets) {\n bool containsAll = true;\n\n const auto& targetTerms = termsByTarget[target];\n for (const auto& term : group.selectedTerms) {\n if (targetTerms.find(term) == targetTerms.end()) {\n containsAll = false;\n break;\n }\n }\n\n if (containsAll) {\n exclusive = false;\n break;\n }\n }\n\n if (!exclusive) {\n break;\n }\n }\n\n if (exclusive) {\n xorGroups.emplace_back(group.selectedTerms);\n } else {\n orGroups.emplace_back(group.selectedTerms);\n }\n }\n\n return serializeTermGroups(orGroups, xorGroups);\n }\n};\n\nclass BestEffortQueryGenerator : public QueryGenerator {\n TermCategory::TermCategory categories;\n int maxXorGroups;\n\npublic:\n BestEffortQueryGenerator(TermCategory::TermCategory categories, int maxXorGroups)\n : categories(categories),\n maxXorGroups(maxXorGroups) {}\n\n std::string getName() const override {\n return fmt::format(\n \"BestEffortQueryGenerator(categories={}, maxXorGroups={})\",\n TermCategory::toString(categories),\n maxXorGroups);\n }\n\n std::string generateQuery(\n const std::vector& targets,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher) const override {\n ankerl::unordered_dense::map> termsByTarget;\n termsByTarget.reserve(targets.size());\n\n ankerl::unordered_dense::map> sortedTermsByTarget;\n sortedTermsByTarget.reserve(targets.size());\n\n for (const auto& target : targets) {\n auto terms = patentReader.readTerms(target, categories);\n\n std::sort(\n terms.begin(),\n terms.end(),\n [&](const std::string& a, const std::string& b) {\n return searchIndex.getTermCardinality(a) < searchIndex.getTermCardinality(b);\n });\n\n termsByTarget.emplace(target, ankerl::unordered_dense::set(terms.begin(), terms.end()));\n sortedTermsByTarget.emplace(target, terms);\n }\n\n std::string bestQuery;\n double maxScore = 0.0;\n\n std::vector> groups;\n int remainingTokens = 50;\n\n std::string previousTarget;\n ankerl::unordered_dense::set skippedTargets;\n\n while (remainingTokens > 0) {\n auto query = createQuery(groups, termsByTarget);\n\n auto score = getQueryScore(searcher, query, targets);\n if (score > maxScore) {\n bestQuery = query;\n maxScore = score;\n }\n\n std::string currentTarget;\n\n if (!groups.empty() && groups.back().size() < 2) {\n currentTarget = previousTarget;\n } else {\n auto results = !query.empty() ? searcher.search(query) : ankerl::unordered_dense::set();\n for (const auto& target : targets) {\n if (!skippedTargets.contains(target) && !results.contains(target)) {\n currentTarget = target;\n break;\n }\n }\n }\n\n if (currentTarget.empty()) {\n break;\n }\n\n int requiredTokens = 1;\n if (!groups.empty() && currentTarget != previousTarget) {\n ++requiredTokens;\n }\n\n if (requiredTokens > remainingTokens) {\n break;\n }\n\n const auto& availableTerms = sortedTermsByTarget[currentTarget];\n std::string bestTerm;\n\n if (currentTarget != previousTarget) {\n if (!availableTerms.empty()) {\n bestTerm = availableTerms[0];\n }\n } else {\n const auto& currentGroup = groups.back();\n std::vector currentGroupVec(currentGroup.begin(), currentGroup.end());\n\n auto bitset = searchIndex.getTermBitset(currentGroupVec[0]);\n for (std::size_t i = 1; i < currentGroup.size(); ++i) {\n bitset &= searchIndex.getTermBitset(currentGroupVec[i]);\n }\n\n std::size_t minCardinality = std::numeric_limits::max();\n\n for (const auto& term : availableTerms) {\n if (currentGroup.contains(term)) {\n continue;\n }\n\n std::size_t cardinality = bitset.and_cardinality(searchIndex.getTermBitset(term));\n if (cardinality < minCardinality) {\n bestTerm = term;\n minCardinality = cardinality;\n }\n }\n }\n\n if (bestTerm.empty()) {\n skippedTargets.emplace(currentTarget);\n continue;\n }\n\n if (currentTarget != previousTarget) {\n groups.emplace_back();\n }\n\n groups.back().emplace(bestTerm);\n\n remainingTokens -= requiredTokens;\n previousTarget = currentTarget;\n }\n\n return bestQuery;\n }\n\nprivate:\n std::string createQuery(\n const std::vector>& groups,\n const ankerl::unordered_dense::map<\n std::string,\n ankerl::unordered_dense::set>& termsByTarget) const {\n std::vector> targetsByGroup;\n ankerl::unordered_dense::map groupsByTarget;\n\n for (const auto& groupTerms : groups) {\n std::vector targets;\n\n for (const auto& [target, targetTerms] : termsByTarget) {\n bool matches = true;\n for (const auto& term : groupTerms) {\n if (!targetTerms.contains(term)) {\n matches = false;\n break;\n }\n }\n\n if (matches) {\n targets.emplace_back(target);\n ++groupsByTarget[target];\n }\n }\n\n targetsByGroup.emplace_back(targets);\n }\n\n std::vector> orGroups;\n std::vector> xorGroups;\n\n for (std::size_t i = 0; i < groups.size(); ++i) {\n const auto& group = groups[i];\n\n // The Whoosh query parser becomes a lot slower when there are many XOR operators\n if (xorGroups.size() == maxXorGroups) {\n orGroups.emplace_back(group.begin(), group.end());\n continue;\n }\n\n bool isExclusive = true;\n for (const auto& target : targetsByGroup[i]) {\n if (groupsByTarget[target] > 1) {\n isExclusive = false;\n break;\n }\n }\n\n if (isExclusive) {\n xorGroups.emplace_back(group.begin(), group.end());\n } else {\n orGroups.emplace_back(group.begin(), group.end());\n }\n }\n\n return serializeTermGroups(orGroups, xorGroups);\n }\n};\n\ninline std::vector> createQueryGenerators() {\n std::vector> out;\n\n int maxXorGroups = 5;\n\n out.emplace_back(\n std::make_unique(\n TermCategory::Cpc | TermCategory::Title | TermCategory::Abstract | TermCategory::Claims,\n maxXorGroups));\n\n out.emplace_back(\n std::make_unique(\n TermCategory::Cpc | TermCategory::Title | TermCategory::Abstract | TermCategory::Claims,\n 20,\n maxXorGroups));\n\n out.emplace_back(\n std::make_unique(\n TermCategory::Cpc | TermCategory::Title | TermCategory::Abstract | TermCategory::Claims,\n 10,\n 0));\n\n for (auto categories : std::vector{\n TermCategory::Cpc,\n TermCategory::Title,\n TermCategory::Abstract,\n TermCategory::Cpc | TermCategory::Title,\n TermCategory::Cpc | TermCategory::Abstract,\n TermCategory::Title | TermCategory::Abstract,\n TermCategory::Cpc | TermCategory::Title | TermCategory::Abstract,\n }) {\n out.emplace_back(std::make_unique(categories, 2, 2, 2, maxXorGroups));\n }\n\n for (auto category : std::vector{\n TermCategory::Cpc,\n TermCategory::Title,\n TermCategory::Abstract,\n TermCategory::Claims,\n }) {\n out.emplace_back(std::make_unique(category));\n }\n\n return out;\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:10.976116Z","iopub.execute_input":"2024-07-24T22:02:10.977169Z","iopub.status.idle":"2024-07-24T22:02:11.013820Z","shell.execute_reply.started":"2024-07-24T22:02:10.977105Z","shell.execute_reply":"2024-07-24T22:02:11.012433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/grafana.h\n#pragma once\n\n#include \n#include \n#include \n#include \n\n#include \n\n#include \n#include \n\n#ifdef GRAFANA_ENABLED\n\n#include \n\nclass GrafanaReporter {\n pqxx::connection connection;\n\npublic:\n explicit GrafanaReporter(bool initDatabase = false)\n : connection(\"postgres://postgres:postgres@localhost:5432/postgres\") {\n if (initDatabase) {\n pqxx::work work(connection);\n\n work.exec0(\n R\"(\nCREATE TABLE IF NOT EXISTS properties (\n key TEXT PRIMARY KEY,\n value INTEGER NOT NULL\n)\n)\");\n work.exec0(\n R\"(\nCREATE TABLE IF NOT EXISTS generators (\n task INTEGER NOT NULL,\n generator TEXT NOT NULL,\n score DOUBLE PRECISION,\n seconds DOUBLE PRECISION,\n PRIMARY KEY (task, generator)\n)\n)\");\n work.exec0(\n R\"(\nCREATE TABLE IF NOT EXISTS tasks (\n id INTEGER PRIMARY KEY,\n generator TEXT,\n score DOUBLE PRECISION,\n seconds DOUBLE PRECISION\n)\n)\");\n\n work.exec0(\"TRUNCATE properties\");\n work.exec0(\"TRUNCATE generators\");\n work.exec0(\"TRUNCATE tasks\");\n\n work.commit();\n }\n\n connection.prepare(\"insert_property\", \"INSERT INTO properties VALUES ($1, $2)\");\n connection.prepare(\"insert_generator\", \"INSERT INTO generators VALUES ($1, $2, $3, $4)\");\n connection.prepare(\"insert_task\", \"INSERT INTO tasks VALUES ($1, $2, $3, $4)\");\n\n connection.prepare(\n \"update_generator\",\n \"UPDATE generators SET score = $3, seconds = $4 WHERE task = $1 AND generator = $2\");\n connection.prepare(\"update_task\", \"UPDATE tasks SET generator = $2, score = $3, seconds = $4 WHERE id = $1\");\n }\n\n void init(\n std::size_t threadCount,\n std::size_t taskCount,\n const std::vector>& generators) {\n try {\n pqxx::work work(connection);\n\n work.exec_prepared(\"insert_property\", \"threadCount\", threadCount);\n\n for (std::size_t i = 0; i < taskCount; ++i) {\n work.exec_prepared(\"insert_task\", i, nullptr, nullptr, nullptr);\n\n for (const auto& generator : generators) {\n work.exec_prepared(\"insert_generator\", i, generator->getName(), nullptr, nullptr);\n }\n }\n\n work.commit();\n } catch (const pqxx::sql_error& exception) {\n spdlog::warn(R\"(init({}, #{} generators): {})\", taskCount, generators.size(), exception.what());\n }\n }\n\n void reportGenerator(std::size_t task, const std::string& generator, double score, double seconds) {\n try {\n pqxx::work work(connection);\n work.exec_prepared(\"update_generator\", task, generator, score, seconds);\n work.commit();\n } catch (const pqxx::sql_error& exception) {\n spdlog::warn(R\"(reportGenerator({}, \"{}\", {}, {}): {})\", task, generator, score, seconds, exception.what());\n }\n }\n\n void reportTask(std::size_t task, const std::string& generator, double score, double seconds) {\n try {\n pqxx::work work(connection);\n work.exec_prepared(\"update_task\", task, generator, score, seconds);\n work.commit();\n } catch (const pqxx::sql_error& exception) {\n spdlog::warn(R\"(reportTask({}, \"{}\", {}, {}): {})\", task, generator, score, seconds, exception.what());\n }\n }\n};\n\n#else\n\nclass GrafanaReporter {\npublic:\n explicit GrafanaReporter(bool initDatabase = false) {}\n\n void init(\n std::size_t threadCount,\n std::size_t taskCount,\n const std::vector>& generators) {}\n\n void reportGenerator(std::size_t task, const std::string& generator, double score, double seconds) {}\n\n void reportTask(std::size_t task, const std::string& generator, double score, double seconds) {}\n};\n\n#endif","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.015842Z","iopub.execute_input":"2024-07-24T22:02:11.016409Z","iopub.status.idle":"2024-07-24T22:02:11.037456Z","shell.execute_reply.started":"2024-07-24T22:02:11.016354Z","shell.execute_reply":"2024-07-24T22:02:11.035390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/index.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\nclass SearchIndexReader : public DataReader {\npublic:\n using DataReader::DataReader;\n\n std::uint32_t readPatentCount() {\n seekToKey(\"ids\");\n return readScalar();\n }\n\n std::vector readPatentIdsReversed() {\n seekToKey(\"ids\");\n\n auto size = readScalar();\n std::vector publicationNumbers(size);\n\n for (std::uint32_t i = 0; i < size; ++i) {\n auto publicationNumber = readString();\n auto id = readScalar();\n\n publicationNumbers[id] = publicationNumber;\n }\n\n return publicationNumbers;\n }\n\n roaring::Roaring readTermBitset(const std::string& term) {\n seekToKey(term);\n\n auto size = readScalar();\n auto buffer = readRaw(size);\n\n return roaring::Roaring::read(buffer.data(), false);\n }\n\n ankerl::unordered_dense::map readTermCounts(const std::string& term) {\n seekToKey(\" \" + term);\n\n ankerl::unordered_dense::map counts;\n\n auto size = readScalar();\n counts.reserve(size);\n\n for (std::uint32_t i = 0; i < size; ++i) {\n auto patentId = readScalar();\n auto count = readScalar();\n\n counts.emplace(patentId, count);\n }\n\n return counts;\n }\n\n std::uint32_t readTermCardinality(const std::string& term) {\n seekToKey(\" \" + term);\n return readScalar();\n }\n};\n\nclass SearchIndexWriter : public DataWriter {\npublic:\n using DataWriter::DataWriter;\n\n void writeIds(const ankerl::unordered_dense::map& ids) {\n addKey(\"ids\");\n\n writeScalar(ids.size());\n for (const auto& [publicationNumber, id] : ids) {\n writeString(publicationNumber);\n writeScalar(id);\n }\n }\n\n void writeCounts(\n const std::string& term,\n const ankerl::unordered_dense::map& counts) {\n writeCountsAsBitset(term, counts);\n writeCountsAsMap(term, counts);\n }\n\nprivate:\n void writeCountsAsBitset(\n const std::string& term,\n const ankerl::unordered_dense::map& counts) {\n roaring::Roaring bitset;\n roaring::BulkContext bulkContext;\n\n for (const auto& [patentId, _] : counts) {\n bitset.addBulk(bulkContext, patentId);\n }\n\n bitset.runOptimize();\n bitset.shrinkToFit();\n\n addKey(term);\n\n auto size = bitset.getSizeInBytes(false);\n std::vector buffer(size);\n\n bitset.write(buffer.data(), false);\n\n writeScalar(size);\n writeRaw(buffer.data(), size);\n }\n\n void writeCountsAsMap(\n const std::string& term,\n const ankerl::unordered_dense::map& counts) {\n addKey(\" \" + term);\n\n writeScalar(counts.size());\n for (const auto& [patentId, count] : counts) {\n writeScalar(patentId);\n writeScalar(count);\n }\n }\n};\n\nclass SearchIndex {\n SearchIndexReader reader;\n\n std::uint32_t patentCount;\n\n ankerl::unordered_dense::map bitsets;\n ankerl::unordered_dense::map> counts;\n ankerl::unordered_dense::map cardinalities;\n\n std::vector tfIdfScores;\n\npublic:\n explicit SearchIndex(const SearchIndexReader& reader)\n : reader(reader), patentCount(this->reader.readPatentCount()), tfIdfScores(patentCount) {}\n\n void clearCache() {\n bitsets.clear();\n counts.clear();\n cardinalities.clear();\n }\n\n std::uint32_t getPatentCount() const {\n return patentCount;\n }\n\n const roaring::Roaring& getTermBitset(const std::string& term) {\n auto it = bitsets.find(term);\n if (it != bitsets.end()) {\n return it->second;\n }\n\n bitsets.emplace(term, reader.readTermBitset(term));\n return bitsets[term];\n }\n\n const ankerl::unordered_dense::map& getTermCounts(const std::string& term) {\n auto it = counts.find(term);\n if (it != counts.end()) {\n return it->second;\n }\n\n counts.emplace(term, reader.readTermCounts(term));\n return counts[term];\n }\n\n std::uint32_t getTermCardinality(const std::string& term) {\n auto it = cardinalities.find(term);\n if (it != cardinalities.end()) {\n return it->second;\n }\n\n cardinalities.emplace(term, reader.readTermCardinality(term));\n return cardinalities[term];\n }\n\n double getTermSelectivity(const std::string& term) {\n return static_cast(getTermCardinality(term)) / static_cast(patentCount);\n }\n};\n\ninline void processTermGroup(\n SearchIndexWriter& writer,\n const PatentReader& patentReader,\n BS::thread_pool& threadPool,\n const std::vector& publicationNumbers,\n const ankerl::unordered_dense::map& patentIds,\n TermCategory::TermCategory category,\n const ankerl::unordered_dense::set& terms,\n const std::string& description) {\n ankerl::unordered_dense::map> termCounts;\n std::mutex termCountsMutex;\n\n ProgressBar progressBar(publicationNumbers.size(), description);\n bool acceptAllTerms = terms.empty();\n\n threadPool.detach_blocks(\n static_cast(0),\n publicationNumbers.size(),\n [&](std::size_t start, std::size_t end) {\n PatentReader localPatentReader(patentReader);\n ankerl::unordered_dense::map<\n std::string,\n ankerl::unordered_dense::map> localTermCounts;\n\n for (std::size_t i = start; i < end; ++i) {\n const auto& publicationNumber = publicationNumbers[i];\n auto patentId = patentIds.at(publicationNumber);\n\n auto patentCounts = localPatentReader.readTermsWithCounts(publicationNumber, category);\n for (const auto& [term, count] : patentCounts) {\n if (acceptAllTerms || terms.contains(term)) {\n localTermCounts[term].emplace(patentId, count);\n }\n }\n }\n\n progressBar.update(end - start);\n\n std::lock_guard lock(termCountsMutex);\n for (const auto& [term, counts] : localTermCounts) {\n termCounts[term].insert(counts.begin(), counts.end());\n }\n },\n threadPool.get_thread_count() * 5);\n\n threadPool.wait();\n\n for (const auto& [term, counts] : termCounts) {\n writer.writeCounts(term, counts);\n }\n}\n\ninline void processTerms(\n SearchIndexWriter& writer,\n const PatentReader& patentReader,\n BS::thread_pool& threadPool,\n const std::vector& publicationNumbers,\n const ankerl::unordered_dense::map& patentIds,\n TermCategory::TermCategory category) {\n if (category != TermCategory::Claims && category != TermCategory::Description) {\n processTermGroup(\n writer,\n patentReader,\n threadPool,\n publicationNumbers,\n patentIds,\n category,\n {},\n fmt::format(\"Processing {} terms\", TermCategory::toString(category)));\n return;\n }\n\n ankerl::unordered_dense::map termCounts;\n std::mutex termCountsMutex;\n\n ProgressBar progressBar(\n publicationNumbers.size(),\n fmt::format(\"Collecting {} terms\", TermCategory::toString(category)));\n\n threadPool.detach_blocks(\n static_cast(0),\n publicationNumbers.size(),\n [&](std::size_t start, std::size_t end) {\n PatentReader localPatentReader(patentReader);\n ankerl::unordered_dense::map localTermCounts;\n\n for (std::size_t i = start; i < end; ++i) {\n auto patentTermCounts = localPatentReader.readTermsWithCounts(publicationNumbers[i], category);\n for (const auto& [term, count] : patentTermCounts) {\n localTermCounts[term] += count;\n }\n }\n\n progressBar.update(end - start);\n\n std::lock_guard lock(termCountsMutex);\n for (const auto& [term, count] : localTermCounts) {\n termCounts[term] += count;\n }\n },\n threadPool.get_thread_count() * 5);\n\n threadPool.wait();\n\n int groupCount = category == TermCategory::Claims ? 5 : 20;\n spdlog::info(\n \"Processing {} {} terms in {} groups\",\n termCounts.size(),\n TermCategory::toString(category),\n groupCount);\n\n std::vector> termCountsSorted(termCounts.begin(), termCounts.end());\n std::sort(\n termCountsSorted.begin(),\n termCountsSorted.end(),\n [](const std::pair& a, const std::pair& b) {\n return a.second < b.second;\n });\n\n termCounts.clear();\n\n for (int i = 0; i < groupCount; ++i) {\n ankerl::unordered_dense::set terms;\n terms.reserve(termCountsSorted.size() / groupCount);\n for (std::size_t j = i; j < termCountsSorted.size(); j += groupCount) {\n terms.emplace(termCountsSorted[j].first);\n }\n\n processTermGroup(\n writer,\n patentReader,\n threadPool,\n publicationNumbers,\n patentIds,\n category,\n terms,\n fmt::format(\"Processing {} terms (group {}/{})\", TermCategory::toString(category), i + 1, groupCount));\n }\n}\n\ninline void createSearchIndex(\n const ankerl::unordered_dense::set& publicationNumbers,\n const std::filesystem::path& outputDirectory,\n const PatentReader& patentReader,\n bool includeDescription) {\n spdlog::info(\n \"Building search index containing {} patents in {}\",\n publicationNumbers.size(),\n outputDirectory.c_str());\n\n spdlog::info(\"Sorting publication numbers\");\n std::vector sortedPublicationNumbers(publicationNumbers.begin(), publicationNumbers.end());\n patentReader.sortToIndex(sortedPublicationNumbers);\n\n SearchIndexWriter searchIndexWriter(outputDirectory);\n\n ankerl::unordered_dense::map patentIds;\n patentIds.reserve(sortedPublicationNumbers.size());\n\n for (const auto& publicationNumber : sortedPublicationNumbers) {\n patentIds.emplace(publicationNumber, patentIds.size());\n }\n\n spdlog::info(\"Saving ids\");\n searchIndexWriter.writeIds(patentIds);\n\n BS::thread_pool threadPool;\n\n for (const auto category : std::vector{\n TermCategory::Cpc,\n TermCategory::Title,\n TermCategory::Abstract,\n TermCategory::Claims,\n TermCategory::Description,\n }) {\n if (!includeDescription && category == TermCategory::Description) {\n continue;\n }\n\n processTerms(searchIndexWriter, patentReader, threadPool, sortedPublicationNumbers, patentIds, category);\n }\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.040047Z","iopub.execute_input":"2024-07-24T22:02:11.040636Z","iopub.status.idle":"2024-07-24T22:02:11.065519Z","shell.execute_reply.started":"2024-07-24T22:02:11.040585Z","shell.execute_reply":"2024-07-24T22:02:11.064050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/patents.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#include \n#include \n#include \n\nclass PatentReader : public DataReader {\npublic:\n using DataReader::DataReader;\n\n PatentReader() : PatentReader(getReformattedPatentDataDirectory()) {}\n\n std::vector readTerms(const std::string& publicationNumber, TermCategory::TermCategory categories) {\n seekToKey(publicationNumber);\n\n auto cpcSize = readScalar();\n auto titleSize = readScalar();\n auto abstractSize = readScalar();\n auto claimsSize = readScalar();\n\n auto cpcOffset = getPosition();\n auto titleOffset = cpcOffset + cpcSize;\n auto abstractOffset = titleOffset + titleSize;\n auto claimsOffset = abstractOffset + abstractSize;\n auto descriptionOffset = claimsOffset + claimsSize;\n\n std::vector out;\n\n if ((categories & TermCategory::Cpc) != 0) {\n readKeywordTokens(out, cpcOffset, TermCategory::Cpc);\n }\n\n if ((categories & TermCategory::Title) != 0) {\n readTextTokens(out, titleOffset, TermCategory::Title);\n }\n\n if ((categories & TermCategory::Abstract) != 0) {\n readTextTokens(out, abstractOffset, TermCategory::Abstract);\n }\n\n if ((categories & TermCategory::Claims) != 0) {\n readTextTokens(out, claimsOffset, TermCategory::Claims);\n }\n\n if ((categories & TermCategory::Description) != 0) {\n readTextTokens(out, descriptionOffset, TermCategory::Description);\n }\n\n return out;\n }\n\n ankerl::unordered_dense::map readTermsWithCounts(\n const std::string& publicationNumber,\n TermCategory::TermCategory categories) {\n seekToKey(publicationNumber);\n\n auto cpcSize = readScalar();\n auto titleSize = readScalar();\n auto abstractSize = readScalar();\n auto claimsSize = readScalar();\n\n auto cpcOffset = getPosition();\n auto titleOffset = cpcOffset + cpcSize;\n auto abstractOffset = titleOffset + titleSize;\n auto claimsOffset = abstractOffset + abstractSize;\n auto descriptionOffset = claimsOffset + claimsSize;\n\n ankerl::unordered_dense::map out;\n\n if ((categories & TermCategory::Cpc) != 0) {\n readKeywordTokensWithCounts(out, cpcOffset, TermCategory::Cpc);\n }\n\n if ((categories & TermCategory::Title) != 0) {\n readTextTokensWithCounts(out, titleOffset, TermCategory::Title);\n }\n\n if ((categories & TermCategory::Abstract) != 0) {\n readTextTokensWithCounts(out, abstractOffset, TermCategory::Abstract);\n }\n\n if ((categories & TermCategory::Claims) != 0) {\n readTextTokensWithCounts(out, claimsOffset, TermCategory::Claims);\n }\n\n if ((categories & TermCategory::Description) != 0) {\n readTextTokensWithCounts(out, descriptionOffset, TermCategory::Description);\n }\n\n return out;\n }\n\nprivate:\n void readKeywordTokens(std::vector& out, std::uint64_t offset, TermCategory::TermCategory category) {\n auto prefix = TermCategory::toString(category) + \":\";\n seek(offset);\n\n auto noTokens = readScalar();\n out.reserve(out.size() + noTokens);\n\n // ankerl::unordered_dense::set wildcardTokens;\n\n for (std::uint16_t i = 0; i < noTokens; ++i) {\n auto term = prefix + readString();\n out.emplace_back(term);\n\n /*auto slashIndex = term.find('/');\n if (slashIndex != std::string::npos) {\n wildcardTokens.insert(term.substr(0, slashIndex) + \"/*\");\n }*/\n }\n\n // out.insert(out.end(), wildcardTokens.begin(), wildcardTokens.end());\n }\n\n void readTextTokens(std::vector& out, std::uint64_t offset, TermCategory::TermCategory category) {\n auto prefix = TermCategory::toString(category) + \":\";\n seek(offset);\n\n auto noTokens = readScalar();\n out.reserve(out.size() + noTokens);\n\n for (std::uint32_t i = 0; i < noTokens; ++i) {\n out.emplace_back(prefix + readString());\n readScalar();\n }\n }\n\n void readKeywordTokensWithCounts(\n ankerl::unordered_dense::map& out,\n std::uint64_t offset,\n TermCategory::TermCategory category) {\n auto prefix = TermCategory::toString(category) + \":\";\n seek(offset);\n\n auto noTokens = readScalar();\n out.reserve(out.size() + noTokens);\n\n for (std::uint16_t i = 0; i < noTokens; ++i) {\n auto term = prefix + readString();\n out.emplace(term, 1);\n\n /*auto slashIndex = term.find('/');\n if (slashIndex != std::string::npos) {\n ++out[term.substr(0, slashIndex) + \"/*\"];\n }*/\n }\n }\n\n void readTextTokensWithCounts(\n ankerl::unordered_dense::map& out,\n std::uint64_t offset,\n TermCategory::TermCategory category) {\n auto prefix = TermCategory::toString(category) + \":\";\n seek(offset);\n\n auto noTokens = readScalar();\n out.reserve(out.size() + noTokens);\n\n for (std::uint32_t i = 0; i < noTokens; ++i) {\n auto token = readString();\n auto count = readScalar();\n\n out.emplace(prefix + token, count);\n }\n }\n};\n\nclass PatentWriter : public DataWriter {\n std::array tokenBlock{};\n\npublic:\n PatentWriter() : PatentWriter(getReformattedPatentDataDirectory()) {}\n\n explicit PatentWriter(const std::filesystem::path& directory)\n : DataWriter(directory) {}\n\n void writePatent(\n const std::string& publicationNumber,\n const std::vector& cpcCodes,\n const std::string& title,\n const std::string& abstract,\n const std::string& claims,\n const std::string& description) {\n auto titleTokens = compressTextTokens(extractTextTokens(title));\n auto abstractTokens = compressTextTokens(extractTextTokens(abstract));\n auto claimsTokens = compressTextTokens(extractTextTokens(claims));\n auto descriptionTokens = compressTextTokens(extractTextTokens(description));\n\n auto cpcSize = getKeywordChunkSize(cpcCodes);\n auto titleSize = getTextChunkSize(titleTokens);\n auto abstractSize = getTextChunkSize(abstractTokens);\n auto claimsSize = getTextChunkSize(claimsTokens);\n\n addKey(publicationNumber);\n\n writeScalar(cpcSize);\n writeScalar(titleSize);\n writeScalar(abstractSize);\n writeScalar(claimsSize);\n\n writeKeywordTokens(cpcCodes);\n writeTextTokens(titleTokens);\n writeTextTokens(abstractTokens);\n writeTextTokens(claimsTokens);\n writeTextTokens(descriptionTokens);\n }\n\n // Only used internally, but publicly exposed for unit tests\n std::vector extractTextTokens(const std::string& text) {\n std::vector tokens;\n\n std::size_t tokenLength = 0;\n\n std::size_t textLength = text.length();\n auto* textChars = text.data();\n\n for (std::size_t i = 0; i < textLength; ++i) {\n char ch = textChars[i];\n\n if (!isWordCharacter(ch)\n && !(tokenLength > 0 && ch == '.' && i < textLength - 1 && isWordCharacter(textChars[i + 1]))) {\n if (tokenLength > 1) {\n std::string token(tokenBlock.data(), tokenBlock.data() + tokenLength);\n\n if (!isForbiddenToken(token)) {\n tokens.emplace_back(std::move(token));\n }\n }\n\n tokenLength = 0;\n } else {\n tokenBlock[tokenLength++] = toLower(ch);\n }\n }\n\n if (tokenLength > 1) {\n std::string token(tokenBlock.data(), tokenBlock.data() + tokenLength);\n\n if (!isForbiddenToken(token)) {\n tokens.emplace_back(std::move(token));\n }\n }\n\n return tokens;\n }\n\nprivate:\n bool isWordCharacter(char ch) const {\n return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_';\n }\n\n char toLower(char ch) const {\n if (ch >= 'A' && ch <= 'Z') {\n return static_cast(ch + 32);\n }\n\n return ch;\n }\n\n bool isForbiddenToken(const std::string& token) const {\n bool hasNonDecimal = false;\n int periodCount = 0;\n\n for (char ch : token) {\n if (ch == '.') {\n ++periodCount;\n } else if (ch < '0' || ch > '9') {\n hasNonDecimal = true;\n break;\n }\n }\n\n if (!hasNonDecimal && periodCount < 2) {\n return true;\n }\n\n auto tokenLength = token.length();\n return (tokenLength == 2 && (token == \"an\" || token == \"by\" || token == \"if\"\n || token == \"is\" || token == \"no\" || token == \"of\"\n || token == \"on\" || token == \"to\"))\n || (tokenLength == 3 && (token == \"are\" || token == \"for\" || token == \"not\"\n || token == \"the\" || token == \"was\"))\n || (tokenLength == 4 && (token == \"into\" || token == \"such\" || token == \"that\"\n || token == \"then\" || token == \"they\" || token == \"this\"\n || token == \"will\"))\n || (tokenLength == 5 && (token == \"their\" || token == \"there\" || token == \"these\"));\n }\n\n ankerl::unordered_dense::map compressTextTokens(\n const std::vector& tokens) const {\n ankerl::unordered_dense::map count;\n\n for (const auto& token : tokens) {\n ++count[token];\n }\n\n return count;\n }\n\n std::uint32_t getKeywordChunkSize(const std::vector& tokens) const {\n // Number of tokens (uint16)\n std::uint32_t size = 2;\n\n for (const auto& token : tokens) {\n // Length of token (uint8) + token (byte per character)\n size += 1 + token.size();\n }\n\n return size;\n }\n\n std::uint32_t getTextChunkSize(const ankerl::unordered_dense::map& tokens) const {\n // Number of tokens (uint32)\n std::uint32_t size = 4;\n\n for (const auto& pair : tokens) {\n // Length of token (uint16) + token (byte per character) + token count (uint16)\n size += 4 + pair.first.size();\n }\n\n return size;\n }\n\n void writeKeywordTokens(const std::vector& tokens) {\n writeScalar(tokens.size());\n\n for (const auto& token : tokens) {\n writeString(token);\n }\n }\n\n void writeTextTokens(const ankerl::unordered_dense::map& tokens) {\n writeScalar(tokens.size());\n\n for (const auto& [token, count] : tokens) {\n writeString(token);\n writeScalar(count);\n }\n }\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.068223Z","iopub.execute_input":"2024-07-24T22:02:11.069135Z","iopub.status.idle":"2024-07-24T22:02:11.092026Z","shell.execute_reply.started":"2024-07-24T22:02:11.069090Z","shell.execute_reply":"2024-07-24T22:02:11.090533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/progress.h\n#pragma once\n\n#include \n#include \n#include \n\n#include \n#include \n\nclass ProgressBar {\n std::size_t total;\n std::string description;\n\n std::size_t counter;\n std::mutex mutex;\n\n indicators::ProgressBar bar{\n indicators::option::BarWidth{102},\n indicators::option::Start{\"[\"},\n indicators::option::Fill{\"=\"},\n indicators::option::Lead{\">\"},\n indicators::option::Remainder{\" \"},\n indicators::option::End{\"]\"},\n indicators::option::ShowPercentage{true},\n indicators::option::ShowElapsedTime{true},\n indicators::option::ShowRemainingTime{true},\n };\n\npublic:\n ProgressBar(std::size_t total, const std::string& description)\n : total(total), description(description), counter(0) {\n print();\n }\n\n void setDescription(const std::string& newDescription) {\n std::lock_guard lock(mutex);\n description = newDescription;\n print();\n }\n\n void update(std::size_t delta = 1) {\n std::lock_guard lock(mutex);\n counter += delta;\n print();\n }\n\nprivate:\n void print() {\n bar.set_option(indicators::option::PostfixText{fmt::format(\"[{}/{}] {}\", counter, total, description)});\n bar.set_progress(static_cast(counter) / static_cast(total) * 100);\n }\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.094473Z","iopub.execute_input":"2024-07-24T22:02:11.095056Z","iopub.status.idle":"2024-07-24T22:02:11.112886Z","shell.execute_reply.started":"2024-07-24T22:02:11.094999Z","shell.execute_reply":"2024-07-24T22:02:11.111268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/queries.h\n#pragma once\n\n#include \n#include \n\n#include \n\nnamespace TermCategory {\nenum TermCategory {\n Cpc = 1,\n Title = 2,\n Abstract = 4,\n Claims = 8,\n Description = 16,\n};\n\ninline TermCategory operator|(TermCategory a, TermCategory b) {\n return static_cast(static_cast(a) | static_cast(b));\n}\n\ninline std::string toString(TermCategory categories) {\n switch (categories) {\n case Cpc:\n return \"cpc\";\n case Title:\n return \"ti\";\n case Abstract:\n return \"ab\";\n case Claims:\n return \"clm\";\n case Description:\n return \"detd\";\n default:\n std::vector matches;\n\n if ((categories & Cpc) != 0) matches.emplace_back(\"cpc\");\n if ((categories & Title) != 0) matches.emplace_back(\"ti\");\n if ((categories & Abstract) != 0) matches.emplace_back(\"ab\");\n if ((categories & Claims) != 0) matches.emplace_back(\"clm\");\n if ((categories & Description) != 0) matches.emplace_back(\"detd\");\n\n return fmt::format(\"[{}]\", fmt::join(matches, \", \"));\n }\n}\n}\n\nstruct Term {\n TermCategory::TermCategory category;\n std::string token;\n\n Term(const char* term) : Term(std::string(term)) {}\n\n Term(const std::string& term) {\n switch (term[0]) {\n case 'c':\n if (term[1] == 'p') {\n category = TermCategory::Cpc;\n token = term.substr(4);\n } else {\n category = TermCategory::Claims;\n token = term.substr(4);\n }\n break;\n case 't':\n category = TermCategory::Title;\n token = term.substr(3);\n break;\n case 'a':\n category = TermCategory::Abstract;\n token = term.substr(3);\n break;\n case 'd':\n category = TermCategory::Description;\n token = term.substr(5);\n break;\n default:\n __builtin_unreachable();\n }\n }\n\n Term(TermCategory::TermCategory category, const std::string& token)\n : category(category),\n token(token) {}\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.115048Z","iopub.execute_input":"2024-07-24T22:02:11.115634Z","iopub.status.idle":"2024-07-24T22:02:11.134897Z","shell.execute_reply.started":"2024-07-24T22:02:11.115583Z","shell.execute_reply":"2024-07-24T22:02:11.133453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/searcher.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n\nclass Searcher {\n SearchIndex& searchIndex;\n\n const std::vector& patentIdsReversed;\n\n ankerl::unordered_dense::map> resultsCache;\n\n struct MatchCollector : whoosh::WhooshVisitor {\n Searcher& searcher;\n\n explicit MatchCollector(Searcher& searcher)\n : searcher(searcher) {}\n\n std::any visitTerm(whoosh::WhooshParser::TermContext* ctx) override {\n return searcher.searchIndex.getTermBitset(ctx->TOKEN(0)->toString() + \":\" + ctx->TOKEN(1)->toString());\n }\n\n std::any visitTermExpr(whoosh::WhooshParser::TermExprContext* ctx) override {\n return visit(ctx->term());\n }\n\n std::any visitWrappedExpr(whoosh::WhooshParser::WrappedExprContext* ctx) override {\n return visit(ctx->expr());\n }\n\n std::any visitOrExpr(whoosh::WhooshParser::OrExprContext* ctx) override {\n return std::any_cast(visit(ctx->left))\n | std::any_cast(visit(ctx->right));\n }\n\n std::any visitAndExpr(whoosh::WhooshParser::AndExprContext* ctx) override {\n return std::any_cast(visit(ctx->left))\n & std::any_cast(visit(ctx->right));\n }\n\n std::any visitXorExpr(whoosh::WhooshParser::XorExprContext* ctx) override {\n return std::any_cast(visit(ctx->left))\n ^ std::any_cast(visit(ctx->right));\n }\n\n std::any visitNotExpr(whoosh::WhooshParser::NotExprContext* ctx) override {\n auto bits = std::any_cast(visit(ctx->right));\n bits.flip(0, searcher.searchIndex.getPatentCount());\n return bits;\n }\n };\n\n struct TermCollector : whoosh::WhooshBaseListener {\n const Searcher& searcher;\n\n ankerl::unordered_dense::map terms;\n\n explicit TermCollector(const Searcher& searcher)\n : searcher(searcher) {}\n\n void enterTerm(whoosh::WhooshParser::TermContext* ctx) override {\n auto term = ctx->TOKEN(0)->toString() + \":\" + ctx->TOKEN(1)->toString();\n\n double patentCount = searcher.searchIndex.getPatentCount();\n double termFrequency = searcher.searchIndex.getTermCardinality(term);\n double idf = std::log(patentCount / (termFrequency + 1)) + 1;\n\n terms[term] += idf;\n }\n };\n\n struct SearchResult {\n std::uint32_t patentId = std::numeric_limits::max();\n double tfIdf = std::numeric_limits::lowest();\n\n SearchResult() = default;\n\n SearchResult(std::uint32_t patentId, double tfIdf)\n : patentId(patentId),\n tfIdf(tfIdf) {}\n\n bool operator<(const SearchResult& other) const {\n if (tfIdf == other.tfIdf) {\n return patentId < other.patentId;\n }\n\n return tfIdf > other.tfIdf;\n }\n };\n\npublic:\n Searcher(SearchIndex& searchIndex, const std::vector& patentIdsReversed)\n : searchIndex(searchIndex),\n patentIdsReversed(patentIdsReversed) {}\n\n void clearCache() {\n resultsCache.clear();\n }\n\n ankerl::unordered_dense::set search(const std::string& query) {\n auto cachedResults = resultsCache.find(query);\n if (cachedResults != resultsCache.end()) {\n return cachedResults->second;\n }\n\n antlr4::ANTLRInputStream input(query);\n whoosh::WhooshLexer lexer(&input);\n antlr4::CommonTokenStream tokens(&lexer);\n whoosh::WhooshParser parser(&tokens);\n\n antlr4::tree::ParseTree* tree = parser.expr();\n\n MatchCollector matchCollector(*this);\n auto bits = std::any_cast(matchCollector.visit(tree));\n auto bitsCardinality = bits.cardinality();\n\n // Queries with too many results are unlikely to be winners, and are expensive to sort\n if (bitsCardinality > 5000) {\n ankerl::unordered_dense::set out;\n resultsCache.emplace(query, out);\n return out;\n }\n\n std::vector matchingPatentIds;\n matchingPatentIds.reserve(bitsCardinality);\n for (auto id : bits) {\n matchingPatentIds.emplace_back(id);\n }\n\n if (matchingPatentIds.size() <= 50) {\n auto out = idsToPublicationNumbers(matchingPatentIds);\n resultsCache.emplace(query, out);\n return out;\n }\n\n TermCollector termCollector(*this);\n antlr4::tree::ParseTreeWalker::DEFAULT.walk(&termCollector, tree);\n\n std::vector results(50);\n std::size_t resultsSize = 0;\n\n for (const auto& id : matchingPatentIds) {\n double tfIdf = 0;\n\n for (const auto& [term, idf] : termCollector.terms) {\n const auto& counts = searchIndex.getTermCounts(term);\n\n auto countsIt = counts.find(id);\n if (countsIt != counts.end()) {\n tfIdf += static_cast(countsIt->second) * idf;\n }\n }\n\n SearchResult result(id, tfIdf);\n mmheap::heap_insert_circular(result, results.data(), resultsSize, results.size());\n }\n\n std::vector sortedPatentIds;\n sortedPatentIds.reserve(resultsSize);\n while (resultsSize > 0) {\n sortedPatentIds.emplace_back(mmheap::heap_remove_min(results.data(), resultsSize).patentId);\n }\n\n auto out = idsToPublicationNumbers(sortedPatentIds);\n resultsCache.emplace(query, out);\n return out;\n }\n\nprivate:\n ankerl::unordered_dense::set idsToPublicationNumbers(const std::vector& ids) const {\n std::size_t size = std::min(static_cast(50), ids.size());\n\n ankerl::unordered_dense::set out;\n out.reserve(size);\n for (std::size_t i = 0; i < size; ++i) {\n out.emplace(patentIdsReversed[ids[i]]);\n }\n\n return out;\n }\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.137266Z","iopub.execute_input":"2024-07-24T22:02:11.137729Z","iopub.status.idle":"2024-07-24T22:02:11.157778Z","shell.execute_reply.started":"2024-07-24T22:02:11.137679Z","shell.execute_reply":"2024-07-24T22:02:11.156462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/submission.h\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstruct Task {\n std::size_t id;\n std::string publicationNumber;\n std::vector targets;\n\n std::string bestQuery = \"ti:device\";\n double bestScore = 0.0;\n std::string bestQueryGenerator = \"null\";\n\n Task(std::size_t id, const std::string& publicationNumber, const std::vector& targets)\n : id(id),\n publicationNumber(publicationNumber),\n targets(targets) {}\n\n void tryGenerator(\n const std::unique_ptr& queryGenerator,\n PatentReader& patentReader,\n SearchIndex& searchIndex,\n Searcher& searcher,\n GrafanaReporter& reporter) {\n Timer timer;\n auto query = queryGenerator->generateQuery(targets, patentReader, searchIndex, searcher);\n double seconds = timer.elapsedSeconds();\n\n double score = queryGenerator->getQueryScore(searcher, query, targets);\n reporter.reportGenerator(id, queryGenerator->getName(), score, seconds);\n\n if (score > bestScore) {\n bestScore = score;\n bestQuery = query;\n bestQueryGenerator = queryGenerator->getName();\n }\n }\n};\n\ninline ankerl::unordered_dense::set getPublicationNumbers(\n const std::filesystem::path& testDataFile,\n int maxRows = -1) {\n ankerl::unordered_dense::set publicationNumbers;\n\n spdlog::info(\"Reading test data\");\n readNeighbors(\n testDataFile,\n [&](const std::string& publicationNumber, const std::vector& neighbors) {\n publicationNumbers.emplace(publicationNumber);\n publicationNumbers.insert(neighbors.begin(), neighbors.end());\n },\n maxRows);\n\n ankerl::unordered_dense::map relevantNeighborScores;\n std::vector neighborPercentages{\n 0.08998410676658847,\n 0.06998410676658849,\n 0.059984106766588476,\n 0.05331744009992183,\n 0.04831744009992184,\n 0.04431744009992184,\n 0.04098410676658849,\n 0.038126963909445644,\n 0.03562696390944564,\n 0.03340474168722341,\n 0.03140474168722341,\n 0.02958655986904159,\n 0.02791989320237492,\n 0.02638143166391339,\n 0.02495286023534196,\n 0.023619526902008633,\n 0.02236952690200863,\n 0.021193056313773346,\n 0.020081945202662235,\n 0.019029313623714867,\n 0.018029313623714866,\n 0.01707693267133391,\n 0.016167841762243004,\n 0.015298276544851703,\n 0.014464943211518368,\n 0.01366494321151837,\n 0.012895712442287599,\n 0.012154971701546858,\n 0.011440685987261143,\n 0.010751030814847346,\n 0.01008436414818068,\n 0.009439202857858098,\n 0.008814202857858097,\n 0.00820814225179749,\n 0.007619906957679844,\n 0.007048478386251274,\n 0.006492922830695719,\n 0.005952382290155179,\n 0.005426066500681496,\n 0.0049132459878609815,\n 0.004413245987860982,\n 0.0039254411098122,\n 0.003449250633621724,\n 0.002984134354551957,\n 0.0025295889000065026,\n 0.0020851444555620583,\n 0.0016503618468664061,\n 0.001224829931972789,\n 0.0008081632653061224,\n 0.0004\n };\n\n spdlog::info(\"Reading nearest neighbors\");\n readNeighbors(\n getCompetitionDataDirectory() / \"nearest_neighbors.csv\",\n [&](const std::string& publicationNumber, const std::vector& neighbors) {\n if (!publicationNumbers.contains(publicationNumber)) {\n return;\n }\n\n for (std::size_t i = 0; i < neighbors.size() && i < neighborPercentages.size(); ++i) {\n relevantNeighborScores[neighbors[i]] += neighborPercentages[i];\n }\n });\n\n std::vector relevantNeighbors;\n relevantNeighbors.reserve(relevantNeighborScores.size());\n for (const auto& [neighbor, _] : relevantNeighborScores) {\n relevantNeighbors.emplace_back(neighbor);\n }\n\n std::sort(\n relevantNeighbors.begin(),\n relevantNeighbors.end(),\n [&](const std::string& a, const std::string& b) {\n return relevantNeighborScores[a] > relevantNeighborScores[b];\n });\n\n for (std::size_t i = 0; i < relevantNeighbors.size() && publicationNumbers.size() < 5'000'000; ++i) {\n publicationNumbers.emplace(relevantNeighbors[i]);\n }\n\n return publicationNumbers;\n}\n\ninline std::vector generateQueries(const std::filesystem::path& testDataFile, int maxRows = -1) {\n Timer globalTimer;\n\n spdlog::info(\"Creating patent reader\");\n PatentReader patentReader;\n\n auto searchIndexDirectory = getSubmissionIndexDirectory();\n\n spdlog::info(\"Creating search index\");\n auto publicationNumbers = getPublicationNumbers(testDataFile, maxRows);\n createSearchIndex(publicationNumbers, searchIndexDirectory, patentReader, false);\n\n spdlog::info(\"Creating search index reader\");\n SearchIndexReader searchIndexReader(searchIndexDirectory);\n\n spdlog::info(\"Reading reversed patent ids\");\n auto patentIdsReversed = searchIndexReader.readPatentIdsReversed();\n\n std::vector tasks;\n std::vector targets;\n\n spdlog::info(\"Reading test data\");\n readNeighbors(\n testDataFile,\n [&](const std::string& publicationNumber, const std::vector& neighbors) {\n tasks.emplace_back(tasks.size(), publicationNumber, neighbors);\n\n targets.reserve(targets.size() + neighbors.size());\n targets.insert(targets.end(), neighbors.begin(), neighbors.end());\n },\n maxRows);\n\n spdlog::info(\"Creating query generators\");\n auto queryGenerators = createQueryGenerators();\n\n BS::thread_pool threadPool;\n std::mutex mutex;\n\n spdlog::info(\"Initializing reporter\");\n GrafanaReporter initReporter(true);\n initReporter.init(threadPool.get_thread_count(), tasks.size(), queryGenerators);\n\n spdlog::info(\"Finding best queries for {} test data rows\", tasks.size());\n ProgressBar progressBar(tasks.size(), \"Processing tasks\");\n\n double totalScore = 0.0;\n double tasksProcessed = 0.0;\n\n threadPool.detach_blocks(\n static_cast(0),\n tasks.size(),\n [&](std::size_t start, std::size_t end) {\n PatentReader localPatentReader(patentReader);\n\n SearchIndex searchIndex(searchIndexReader);\n Searcher searcher(searchIndex, patentIdsReversed);\n\n GrafanaReporter reporter;\n\n for (std::size_t i = start; i < end; ++i) {\n Timer localTimer;\n auto& task = tasks[i];\n\n for (const auto& queryGenerator : queryGenerators) {\n task.tryGenerator(queryGenerator, localPatentReader, searchIndex, searcher, reporter);\n\n // The submission notebook may run for up to 9 hours\n // It's ran in an environment with 4 CPUs and there are 2,500 tasks to process\n // This means there are 51.84 seconds per task when nothing else has to happen\n // Limiting to 40 seconds per task should prevent any timeouts\n if (localTimer.elapsedSeconds() >= 40) {\n spdlog::warn(\"Task {} timed out after {}\", task.id, queryGenerator->getName());\n break;\n }\n\n searchIndex.clearCache();\n searcher.clearCache();\n }\n\n reporter.reportTask(task.id, task.bestQueryGenerator, task.bestScore, localTimer.elapsedSeconds());\n\n std::lock_guard lock(mutex);\n totalScore += task.bestScore;\n ++tasksProcessed;\n\n progressBar.setDescription(\n fmt::format(\"Processing tasks (mean score: {:.3f})\", totalScore / tasksProcessed));\n progressBar.update(1);\n }\n },\n threadPool.get_thread_count() * 5);\n\n threadPool.wait();\n\n spdlog::info(\"Mean score: {:.3f}\", totalScore / tasksProcessed);\n spdlog::info(\"Total time taken: {:.3f} seconds\", globalTimer.elapsedSeconds());\n return tasks;\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.164467Z","iopub.execute_input":"2024-07-24T22:02:11.165030Z","iopub.status.idle":"2024-07-24T22:02:11.180526Z","shell.execute_reply.started":"2024-07-24T22:02:11.164954Z","shell.execute_reply":"2024-07-24T22:02:11.179256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/timer.h\n#pragma once\n\n#include \n\nclass Timer {\n std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();\n\npublic:\n double elapsedSeconds() const {\n return std::chrono::duration(std::chrono::high_resolution_clock::now() - start).count();\n }\n};","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.182681Z","iopub.execute_input":"2024-07-24T22:02:11.183207Z","iopub.status.idle":"2024-07-24T22:02:11.201108Z","shell.execute_reply.started":"2024-07-24T22:02:11.183159Z","shell.execute_reply":"2024-07-24T22:02:11.199429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/tools/run-submission.cpp\n#include \n\n#include \n\n#include \n#include \n\nint main() {\n auto tasks = generateQueries(getCompetitionDataDirectory() / \"test.csv\");\n\n auto submissionFile = getOutputDirectory() / \"submission.csv\";\n spdlog::info(\"Writing best queries to {}\", submissionFile.c_str());\n\n std::ofstream out(submissionFile);\n\n out << \"publication_number,query\\n\";\n for (const auto& task : tasks) {\n out << task.publicationNumber << ',' << task.bestQuery << '\\n';\n }\n\n out.flush();\n return 0;\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.202939Z","iopub.execute_input":"2024-07-24T22:02:11.203518Z","iopub.status.idle":"2024-07-24T22:02:11.216421Z","shell.execute_reply.started":"2024-07-24T22:02:11.203463Z","shell.execute_reply":"2024-07-24T22:02:11.214720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/Whoosh.g4\ngrammar Whoosh;\n\n// This ANTLR4 grammar only supports the subset of Whoosh query syntax that this project needs\n\nWS: [ ]+ -> skip ;\n\nTOKEN: [a-zA-Z0-9_./*]+ ;\nterm: TOKEN ':' TOKEN ;\n\nexpr: term #termExpr\n | '(' expr ')' #wrappedExpr\n | left=expr 'OR' right=expr #orExpr\n | left=expr 'AND' right=expr #andExpr\n | left=expr 'XOR' right=expr #xorExpr\n | 'NOT' right=expr #notExpr\n | left=expr right=expr #andExpr ;","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.218017Z","iopub.execute_input":"2024-07-24T22:02:11.218409Z","iopub.status.idle":"2024-07-24T22:02:11.235697Z","shell.execute_reply.started":"2024-07-24T22:02:11.218375Z","shell.execute_reply":"2024-07-24T22:02:11.234077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/Whoosh.interp\ntoken literal names:\nnull\n':'\n'('\n')'\n'OR'\n'AND'\n'XOR'\n'NOT'\nnull\nnull\n\ntoken symbolic names:\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nWS\nTOKEN\n\nrule names:\nterm\nexpr\n\n\natn:\n[4, 1, 9, 35, 2, 0, 7, 0, 2, 1, 7, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 17, 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 30, 8, 1, 10, 1, 12, 1, 33, 9, 1, 1, 1, 0, 1, 2, 2, 0, 2, 0, 0, 38, 0, 4, 1, 0, 0, 0, 2, 16, 1, 0, 0, 0, 4, 5, 5, 9, 0, 0, 5, 6, 5, 1, 0, 0, 6, 7, 5, 9, 0, 0, 7, 1, 1, 0, 0, 0, 8, 9, 6, 1, -1, 0, 9, 17, 3, 0, 0, 0, 10, 11, 5, 2, 0, 0, 11, 12, 3, 2, 1, 0, 12, 13, 5, 3, 0, 0, 13, 17, 1, 0, 0, 0, 14, 15, 5, 7, 0, 0, 15, 17, 3, 2, 1, 2, 16, 8, 1, 0, 0, 0, 16, 10, 1, 0, 0, 0, 16, 14, 1, 0, 0, 0, 17, 31, 1, 0, 0, 0, 18, 19, 10, 5, 0, 0, 19, 20, 5, 4, 0, 0, 20, 30, 3, 2, 1, 6, 21, 22, 10, 4, 0, 0, 22, 23, 5, 5, 0, 0, 23, 30, 3, 2, 1, 5, 24, 25, 10, 3, 0, 0, 25, 26, 5, 6, 0, 0, 26, 30, 3, 2, 1, 4, 27, 28, 10, 1, 0, 0, 28, 30, 3, 2, 1, 2, 29, 18, 1, 0, 0, 0, 29, 21, 1, 0, 0, 0, 29, 24, 1, 0, 0, 0, 29, 27, 1, 0, 0, 0, 30, 33, 1, 0, 0, 0, 31, 29, 1, 0, 0, 0, 31, 32, 1, 0, 0, 0, 32, 3, 1, 0, 0, 0, 33, 31, 1, 0, 0, 0, 3, 16, 29, 31]","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.237022Z","iopub.execute_input":"2024-07-24T22:02:11.237479Z","iopub.status.idle":"2024-07-24T22:02:11.253179Z","shell.execute_reply.started":"2024-07-24T22:02:11.237438Z","shell.execute_reply":"2024-07-24T22:02:11.251720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/Whoosh.tokens\nT__0=1\nT__1=2\nT__2=3\nT__3=4\nT__4=5\nT__5=6\nT__6=7\nWS=8\nTOKEN=9\n':'=1\n'('=2\n')'=3\n'OR'=4\n'AND'=5\n'XOR'=6\n'NOT'=7","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.254922Z","iopub.execute_input":"2024-07-24T22:02:11.256086Z","iopub.status.idle":"2024-07-24T22:02:11.285437Z","shell.execute_reply.started":"2024-07-24T22:02:11.256036Z","shell.execute_reply":"2024-07-24T22:02:11.284063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshBaseListener.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshBaseListener.h\"\n\n\nusing namespace whoosh;","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.286981Z","iopub.execute_input":"2024-07-24T22:02:11.287470Z","iopub.status.idle":"2024-07-24T22:02:11.306127Z","shell.execute_reply.started":"2024-07-24T22:02:11.287429Z","shell.execute_reply":"2024-07-24T22:02:11.304036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshBaseListener.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"WhooshListener.h\"\n\n\nnamespace whoosh {\n\n/**\n * This class provides an empty implementation of WhooshListener,\n * which can be extended to create a listener which only needs to handle a subset\n * of the available methods.\n */\nclass WhooshBaseListener : public WhooshListener {\npublic:\n\n virtual void enterTerm(WhooshParser::TermContext * /*ctx*/) override { }\n virtual void exitTerm(WhooshParser::TermContext * /*ctx*/) override { }\n\n virtual void enterWrappedExpr(WhooshParser::WrappedExprContext * /*ctx*/) override { }\n virtual void exitWrappedExpr(WhooshParser::WrappedExprContext * /*ctx*/) override { }\n\n virtual void enterNotExpr(WhooshParser::NotExprContext * /*ctx*/) override { }\n virtual void exitNotExpr(WhooshParser::NotExprContext * /*ctx*/) override { }\n\n virtual void enterTermExpr(WhooshParser::TermExprContext * /*ctx*/) override { }\n virtual void exitTermExpr(WhooshParser::TermExprContext * /*ctx*/) override { }\n\n virtual void enterOrExpr(WhooshParser::OrExprContext * /*ctx*/) override { }\n virtual void exitOrExpr(WhooshParser::OrExprContext * /*ctx*/) override { }\n\n virtual void enterXorExpr(WhooshParser::XorExprContext * /*ctx*/) override { }\n virtual void exitXorExpr(WhooshParser::XorExprContext * /*ctx*/) override { }\n\n virtual void enterAndExpr(WhooshParser::AndExprContext * /*ctx*/) override { }\n virtual void exitAndExpr(WhooshParser::AndExprContext * /*ctx*/) override { }\n\n\n virtual void enterEveryRule(antlr4::ParserRuleContext * /*ctx*/) override { }\n virtual void exitEveryRule(antlr4::ParserRuleContext * /*ctx*/) override { }\n virtual void visitTerminal(antlr4::tree::TerminalNode * /*node*/) override { }\n virtual void visitErrorNode(antlr4::tree::ErrorNode * /*node*/) override { }\n\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.308174Z","iopub.execute_input":"2024-07-24T22:02:11.308622Z","iopub.status.idle":"2024-07-24T22:02:11.323216Z","shell.execute_reply.started":"2024-07-24T22:02:11.308585Z","shell.execute_reply":"2024-07-24T22:02:11.321711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshBaseVisitor.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshBaseVisitor.h\"\n\n\nusing namespace whoosh;","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.325697Z","iopub.execute_input":"2024-07-24T22:02:11.326221Z","iopub.status.idle":"2024-07-24T22:02:11.346852Z","shell.execute_reply.started":"2024-07-24T22:02:11.326164Z","shell.execute_reply":"2024-07-24T22:02:11.345477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshBaseVisitor.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"WhooshVisitor.h\"\n\n\nnamespace whoosh {\n\n/**\n * This class provides an empty implementation of WhooshVisitor, which can be\n * extended to create a visitor which only needs to handle a subset of the available methods.\n */\nclass WhooshBaseVisitor : public WhooshVisitor {\npublic:\n\n virtual std::any visitTerm(WhooshParser::TermContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitWrappedExpr(WhooshParser::WrappedExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitNotExpr(WhooshParser::NotExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitTermExpr(WhooshParser::TermExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitOrExpr(WhooshParser::OrExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitXorExpr(WhooshParser::XorExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n virtual std::any visitAndExpr(WhooshParser::AndExprContext *ctx) override {\n return visitChildren(ctx);\n }\n\n\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.349105Z","iopub.execute_input":"2024-07-24T22:02:11.349658Z","iopub.status.idle":"2024-07-24T22:02:11.368056Z","shell.execute_reply.started":"2024-07-24T22:02:11.349609Z","shell.execute_reply":"2024-07-24T22:02:11.366467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshLexer.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshLexer.h\"\n\n\nusing namespace antlr4;\n\nusing namespace whoosh;\n\n\nusing namespace antlr4;\n\nnamespace {\n\nstruct WhooshLexerStaticData final {\n WhooshLexerStaticData(std::vector ruleNames,\n std::vector channelNames,\n std::vector modeNames,\n std::vector literalNames,\n std::vector symbolicNames)\n : ruleNames(std::move(ruleNames)), channelNames(std::move(channelNames)),\n modeNames(std::move(modeNames)), literalNames(std::move(literalNames)),\n symbolicNames(std::move(symbolicNames)),\n vocabulary(this->literalNames, this->symbolicNames) {}\n\n WhooshLexerStaticData(const WhooshLexerStaticData&) = delete;\n WhooshLexerStaticData(WhooshLexerStaticData&&) = delete;\n WhooshLexerStaticData& operator=(const WhooshLexerStaticData&) = delete;\n WhooshLexerStaticData& operator=(WhooshLexerStaticData&&) = delete;\n\n std::vector decisionToDFA;\n antlr4::atn::PredictionContextCache sharedContextCache;\n const std::vector ruleNames;\n const std::vector channelNames;\n const std::vector modeNames;\n const std::vector literalNames;\n const std::vector symbolicNames;\n const antlr4::dfa::Vocabulary vocabulary;\n antlr4::atn::SerializedATNView serializedATN;\n std::unique_ptr atn;\n};\n\n::antlr4::internal::OnceFlag whooshlexerLexerOnceFlag;\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\nstatic thread_local\n#endif\nWhooshLexerStaticData *whooshlexerLexerStaticData = nullptr;\n\nvoid whooshlexerLexerInitialize() {\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\n if (whooshlexerLexerStaticData != nullptr) {\n return;\n }\n#else\n assert(whooshlexerLexerStaticData == nullptr);\n#endif\n auto staticData = std::make_unique(\n std::vector{\n \"T__0\", \"T__1\", \"T__2\", \"T__3\", \"T__4\", \"T__5\", \"T__6\", \"WS\", \"TOKEN\"\n },\n std::vector{\n \"DEFAULT_TOKEN_CHANNEL\", \"HIDDEN\"\n },\n std::vector{\n \"DEFAULT_MODE\"\n },\n std::vector{\n \"\", \"':'\", \"'('\", \"')'\", \"'OR'\", \"'AND'\", \"'XOR'\", \"'NOT'\"\n },\n std::vector{\n \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"WS\", \"TOKEN\"\n }\n );\n static const int32_t serializedATNSegment[] = {\n \t4,0,9,52,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,6,\n \t2,7,7,7,2,8,7,8,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,3,1,4,1,4,1,4,1,4,1,\n \t5,1,5,1,5,1,5,1,6,1,6,1,6,1,6,1,7,4,7,42,8,7,11,7,12,7,43,1,7,1,7,1,8,\n \t4,8,49,8,8,11,8,12,8,50,0,0,9,1,1,3,2,5,3,7,4,9,5,11,6,13,7,15,8,17,9,\n \t1,0,2,1,0,32,32,5,0,42,42,46,57,65,90,95,95,97,122,53,0,1,1,0,0,0,0,3,\n \t1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,9,1,0,0,0,0,11,1,0,0,0,0,13,1,0,0,0,\n \t0,15,1,0,0,0,0,17,1,0,0,0,1,19,1,0,0,0,3,21,1,0,0,0,5,23,1,0,0,0,7,25,\n \t1,0,0,0,9,28,1,0,0,0,11,32,1,0,0,0,13,36,1,0,0,0,15,41,1,0,0,0,17,48,\n \t1,0,0,0,19,20,5,58,0,0,20,2,1,0,0,0,21,22,5,40,0,0,22,4,1,0,0,0,23,24,\n \t5,41,0,0,24,6,1,0,0,0,25,26,5,79,0,0,26,27,5,82,0,0,27,8,1,0,0,0,28,29,\n \t5,65,0,0,29,30,5,78,0,0,30,31,5,68,0,0,31,10,1,0,0,0,32,33,5,88,0,0,33,\n \t34,5,79,0,0,34,35,5,82,0,0,35,12,1,0,0,0,36,37,5,78,0,0,37,38,5,79,0,\n \t0,38,39,5,84,0,0,39,14,1,0,0,0,40,42,7,0,0,0,41,40,1,0,0,0,42,43,1,0,\n \t0,0,43,41,1,0,0,0,43,44,1,0,0,0,44,45,1,0,0,0,45,46,6,7,0,0,46,16,1,0,\n \t0,0,47,49,7,1,0,0,48,47,1,0,0,0,49,50,1,0,0,0,50,48,1,0,0,0,50,51,1,0,\n \t0,0,51,18,1,0,0,0,3,0,43,50,1,6,0,0\n };\n staticData->serializedATN = antlr4::atn::SerializedATNView(serializedATNSegment, sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0]));\n\n antlr4::atn::ATNDeserializer deserializer;\n staticData->atn = deserializer.deserialize(staticData->serializedATN);\n\n const size_t count = staticData->atn->getNumberOfDecisions();\n staticData->decisionToDFA.reserve(count);\n for (size_t i = 0; i < count; i++) { \n staticData->decisionToDFA.emplace_back(staticData->atn->getDecisionState(i), i);\n }\n whooshlexerLexerStaticData = staticData.release();\n}\n\n}\n\nWhooshLexer::WhooshLexer(CharStream *input) : Lexer(input) {\n WhooshLexer::initialize();\n _interpreter = new atn::LexerATNSimulator(this, *whooshlexerLexerStaticData->atn, whooshlexerLexerStaticData->decisionToDFA, whooshlexerLexerStaticData->sharedContextCache);\n}\n\nWhooshLexer::~WhooshLexer() {\n delete _interpreter;\n}\n\nstd::string WhooshLexer::getGrammarFileName() const {\n return \"Whoosh.g4\";\n}\n\nconst std::vector& WhooshLexer::getRuleNames() const {\n return whooshlexerLexerStaticData->ruleNames;\n}\n\nconst std::vector& WhooshLexer::getChannelNames() const {\n return whooshlexerLexerStaticData->channelNames;\n}\n\nconst std::vector& WhooshLexer::getModeNames() const {\n return whooshlexerLexerStaticData->modeNames;\n}\n\nconst dfa::Vocabulary& WhooshLexer::getVocabulary() const {\n return whooshlexerLexerStaticData->vocabulary;\n}\n\nantlr4::atn::SerializedATNView WhooshLexer::getSerializedATN() const {\n return whooshlexerLexerStaticData->serializedATN;\n}\n\nconst atn::ATN& WhooshLexer::getATN() const {\n return *whooshlexerLexerStaticData->atn;\n}\n\n\n\n\nvoid WhooshLexer::initialize() {\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\n whooshlexerLexerInitialize();\n#else\n ::antlr4::internal::call_once(whooshlexerLexerOnceFlag, whooshlexerLexerInitialize);\n#endif\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.370681Z","iopub.execute_input":"2024-07-24T22:02:11.371239Z","iopub.status.idle":"2024-07-24T22:02:11.386716Z","shell.execute_reply.started":"2024-07-24T22:02:11.371181Z","shell.execute_reply":"2024-07-24T22:02:11.385380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshLexer.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n\n\nnamespace whoosh {\n\n\nclass WhooshLexer : public antlr4::Lexer {\npublic:\n enum {\n T__0 = 1, T__1 = 2, T__2 = 3, T__3 = 4, T__4 = 5, T__5 = 6, T__6 = 7, \n WS = 8, TOKEN = 9\n };\n\n explicit WhooshLexer(antlr4::CharStream *input);\n\n ~WhooshLexer() override;\n\n\n std::string getGrammarFileName() const override;\n\n const std::vector& getRuleNames() const override;\n\n const std::vector& getChannelNames() const override;\n\n const std::vector& getModeNames() const override;\n\n const antlr4::dfa::Vocabulary& getVocabulary() const override;\n\n antlr4::atn::SerializedATNView getSerializedATN() const override;\n\n const antlr4::atn::ATN& getATN() const override;\n\n // By default the static state used to implement the lexer is lazily initialized during the first\n // call to the constructor. You can call this function if you wish to initialize the static state\n // ahead of time.\n static void initialize();\n\nprivate:\n\n // Individual action functions triggered by action() above.\n\n // Individual semantic predicate functions triggered by sempred() above.\n\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.388986Z","iopub.execute_input":"2024-07-24T22:02:11.389769Z","iopub.status.idle":"2024-07-24T22:02:11.406294Z","shell.execute_reply.started":"2024-07-24T22:02:11.389722Z","shell.execute_reply":"2024-07-24T22:02:11.404801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshLexer.interp\ntoken literal names:\nnull\n':'\n'('\n')'\n'OR'\n'AND'\n'XOR'\n'NOT'\nnull\nnull\n\ntoken symbolic names:\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nWS\nTOKEN\n\nrule names:\nT__0\nT__1\nT__2\nT__3\nT__4\nT__5\nT__6\nWS\nTOKEN\n\nchannel names:\nDEFAULT_TOKEN_CHANNEL\nHIDDEN\n\nmode names:\nDEFAULT_MODE\n\natn:\n[4, 0, 9, 52, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 4, 7, 42, 8, 7, 11, 7, 12, 7, 43, 1, 7, 1, 7, 1, 8, 4, 8, 49, 8, 8, 11, 8, 12, 8, 50, 0, 0, 9, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 1, 0, 2, 1, 0, 32, 32, 5, 0, 42, 42, 46, 57, 65, 90, 95, 95, 97, 122, 53, 0, 1, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 1, 19, 1, 0, 0, 0, 3, 21, 1, 0, 0, 0, 5, 23, 1, 0, 0, 0, 7, 25, 1, 0, 0, 0, 9, 28, 1, 0, 0, 0, 11, 32, 1, 0, 0, 0, 13, 36, 1, 0, 0, 0, 15, 41, 1, 0, 0, 0, 17, 48, 1, 0, 0, 0, 19, 20, 5, 58, 0, 0, 20, 2, 1, 0, 0, 0, 21, 22, 5, 40, 0, 0, 22, 4, 1, 0, 0, 0, 23, 24, 5, 41, 0, 0, 24, 6, 1, 0, 0, 0, 25, 26, 5, 79, 0, 0, 26, 27, 5, 82, 0, 0, 27, 8, 1, 0, 0, 0, 28, 29, 5, 65, 0, 0, 29, 30, 5, 78, 0, 0, 30, 31, 5, 68, 0, 0, 31, 10, 1, 0, 0, 0, 32, 33, 5, 88, 0, 0, 33, 34, 5, 79, 0, 0, 34, 35, 5, 82, 0, 0, 35, 12, 1, 0, 0, 0, 36, 37, 5, 78, 0, 0, 37, 38, 5, 79, 0, 0, 38, 39, 5, 84, 0, 0, 39, 14, 1, 0, 0, 0, 40, 42, 7, 0, 0, 0, 41, 40, 1, 0, 0, 0, 42, 43, 1, 0, 0, 0, 43, 41, 1, 0, 0, 0, 43, 44, 1, 0, 0, 0, 44, 45, 1, 0, 0, 0, 45, 46, 6, 7, 0, 0, 46, 16, 1, 0, 0, 0, 47, 49, 7, 1, 0, 0, 48, 47, 1, 0, 0, 0, 49, 50, 1, 0, 0, 0, 50, 48, 1, 0, 0, 0, 50, 51, 1, 0, 0, 0, 51, 18, 1, 0, 0, 0, 3, 0, 43, 50, 1, 6, 0, 0]","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.408462Z","iopub.execute_input":"2024-07-24T22:02:11.409035Z","iopub.status.idle":"2024-07-24T22:02:11.427709Z","shell.execute_reply.started":"2024-07-24T22:02:11.408977Z","shell.execute_reply":"2024-07-24T22:02:11.425815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshLexer.tokens\nT__0=1\nT__1=2\nT__2=3\nT__3=4\nT__4=5\nT__5=6\nT__6=7\nWS=8\nTOKEN=9\n':'=1\n'('=2\n')'=3\n'OR'=4\n'AND'=5\n'XOR'=6\n'NOT'=7","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.429942Z","iopub.execute_input":"2024-07-24T22:02:11.430613Z","iopub.status.idle":"2024-07-24T22:02:11.446668Z","shell.execute_reply.started":"2024-07-24T22:02:11.430460Z","shell.execute_reply":"2024-07-24T22:02:11.445194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshListener.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshListener.h\"\n\n\nusing namespace whoosh;","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.448674Z","iopub.execute_input":"2024-07-24T22:02:11.449124Z","iopub.status.idle":"2024-07-24T22:02:11.466482Z","shell.execute_reply.started":"2024-07-24T22:02:11.449073Z","shell.execute_reply":"2024-07-24T22:02:11.465090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshListener.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"WhooshParser.h\"\n\n\nnamespace whoosh {\n\n/**\n * This interface defines an abstract listener for a parse tree produced by WhooshParser.\n */\nclass WhooshListener : public antlr4::tree::ParseTreeListener {\npublic:\n\n virtual void enterTerm(WhooshParser::TermContext *ctx) = 0;\n virtual void exitTerm(WhooshParser::TermContext *ctx) = 0;\n\n virtual void enterWrappedExpr(WhooshParser::WrappedExprContext *ctx) = 0;\n virtual void exitWrappedExpr(WhooshParser::WrappedExprContext *ctx) = 0;\n\n virtual void enterNotExpr(WhooshParser::NotExprContext *ctx) = 0;\n virtual void exitNotExpr(WhooshParser::NotExprContext *ctx) = 0;\n\n virtual void enterTermExpr(WhooshParser::TermExprContext *ctx) = 0;\n virtual void exitTermExpr(WhooshParser::TermExprContext *ctx) = 0;\n\n virtual void enterOrExpr(WhooshParser::OrExprContext *ctx) = 0;\n virtual void exitOrExpr(WhooshParser::OrExprContext *ctx) = 0;\n\n virtual void enterXorExpr(WhooshParser::XorExprContext *ctx) = 0;\n virtual void exitXorExpr(WhooshParser::XorExprContext *ctx) = 0;\n\n virtual void enterAndExpr(WhooshParser::AndExprContext *ctx) = 0;\n virtual void exitAndExpr(WhooshParser::AndExprContext *ctx) = 0;\n\n\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.468283Z","iopub.execute_input":"2024-07-24T22:02:11.468758Z","iopub.status.idle":"2024-07-24T22:02:11.481603Z","shell.execute_reply.started":"2024-07-24T22:02:11.468709Z","shell.execute_reply":"2024-07-24T22:02:11.480222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshParser.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshListener.h\"\n#include \"WhooshVisitor.h\"\n\n#include \"WhooshParser.h\"\n\n\nusing namespace antlrcpp;\nusing namespace whoosh;\n\nusing namespace antlr4;\n\nnamespace {\n\nstruct WhooshParserStaticData final {\n WhooshParserStaticData(std::vector ruleNames,\n std::vector literalNames,\n std::vector symbolicNames)\n : ruleNames(std::move(ruleNames)), literalNames(std::move(literalNames)),\n symbolicNames(std::move(symbolicNames)),\n vocabulary(this->literalNames, this->symbolicNames) {}\n\n WhooshParserStaticData(const WhooshParserStaticData&) = delete;\n WhooshParserStaticData(WhooshParserStaticData&&) = delete;\n WhooshParserStaticData& operator=(const WhooshParserStaticData&) = delete;\n WhooshParserStaticData& operator=(WhooshParserStaticData&&) = delete;\n\n std::vector decisionToDFA;\n antlr4::atn::PredictionContextCache sharedContextCache;\n const std::vector ruleNames;\n const std::vector literalNames;\n const std::vector symbolicNames;\n const antlr4::dfa::Vocabulary vocabulary;\n antlr4::atn::SerializedATNView serializedATN;\n std::unique_ptr atn;\n};\n\n::antlr4::internal::OnceFlag whooshParserOnceFlag;\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\nstatic thread_local\n#endif\nWhooshParserStaticData *whooshParserStaticData = nullptr;\n\nvoid whooshParserInitialize() {\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\n if (whooshParserStaticData != nullptr) {\n return;\n }\n#else\n assert(whooshParserStaticData == nullptr);\n#endif\n auto staticData = std::make_unique(\n std::vector{\n \"term\", \"expr\"\n },\n std::vector{\n \"\", \"':'\", \"'('\", \"')'\", \"'OR'\", \"'AND'\", \"'XOR'\", \"'NOT'\"\n },\n std::vector{\n \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"WS\", \"TOKEN\"\n }\n );\n static const int32_t serializedATNSegment[] = {\n \t4,1,9,35,2,0,7,0,2,1,7,1,1,0,1,0,1,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\n \t1,1,3,1,17,8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,1,30,8,1,\n \t10,1,12,1,33,9,1,1,1,0,1,2,2,0,2,0,0,38,0,4,1,0,0,0,2,16,1,0,0,0,4,5,\n \t5,9,0,0,5,6,5,1,0,0,6,7,5,9,0,0,7,1,1,0,0,0,8,9,6,1,-1,0,9,17,3,0,0,0,\n \t10,11,5,2,0,0,11,12,3,2,1,0,12,13,5,3,0,0,13,17,1,0,0,0,14,15,5,7,0,0,\n \t15,17,3,2,1,2,16,8,1,0,0,0,16,10,1,0,0,0,16,14,1,0,0,0,17,31,1,0,0,0,\n \t18,19,10,5,0,0,19,20,5,4,0,0,20,30,3,2,1,6,21,22,10,4,0,0,22,23,5,5,0,\n \t0,23,30,3,2,1,5,24,25,10,3,0,0,25,26,5,6,0,0,26,30,3,2,1,4,27,28,10,1,\n \t0,0,28,30,3,2,1,2,29,18,1,0,0,0,29,21,1,0,0,0,29,24,1,0,0,0,29,27,1,0,\n \t0,0,30,33,1,0,0,0,31,29,1,0,0,0,31,32,1,0,0,0,32,3,1,0,0,0,33,31,1,0,\n \t0,0,3,16,29,31\n };\n staticData->serializedATN = antlr4::atn::SerializedATNView(serializedATNSegment, sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0]));\n\n antlr4::atn::ATNDeserializer deserializer;\n staticData->atn = deserializer.deserialize(staticData->serializedATN);\n\n const size_t count = staticData->atn->getNumberOfDecisions();\n staticData->decisionToDFA.reserve(count);\n for (size_t i = 0; i < count; i++) { \n staticData->decisionToDFA.emplace_back(staticData->atn->getDecisionState(i), i);\n }\n whooshParserStaticData = staticData.release();\n}\n\n}\n\nWhooshParser::WhooshParser(TokenStream *input) : WhooshParser(input, antlr4::atn::ParserATNSimulatorOptions()) {}\n\nWhooshParser::WhooshParser(TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options) : Parser(input) {\n WhooshParser::initialize();\n _interpreter = new atn::ParserATNSimulator(this, *whooshParserStaticData->atn, whooshParserStaticData->decisionToDFA, whooshParserStaticData->sharedContextCache, options);\n}\n\nWhooshParser::~WhooshParser() {\n delete _interpreter;\n}\n\nconst atn::ATN& WhooshParser::getATN() const {\n return *whooshParserStaticData->atn;\n}\n\nstd::string WhooshParser::getGrammarFileName() const {\n return \"Whoosh.g4\";\n}\n\nconst std::vector& WhooshParser::getRuleNames() const {\n return whooshParserStaticData->ruleNames;\n}\n\nconst dfa::Vocabulary& WhooshParser::getVocabulary() const {\n return whooshParserStaticData->vocabulary;\n}\n\nantlr4::atn::SerializedATNView WhooshParser::getSerializedATN() const {\n return whooshParserStaticData->serializedATN;\n}\n\n\n//----------------- TermContext ------------------------------------------------------------------\n\nWhooshParser::TermContext::TermContext(ParserRuleContext *parent, size_t invokingState)\n : ParserRuleContext(parent, invokingState) {\n}\n\nstd::vector WhooshParser::TermContext::TOKEN() {\n return getTokens(WhooshParser::TOKEN);\n}\n\ntree::TerminalNode* WhooshParser::TermContext::TOKEN(size_t i) {\n return getToken(WhooshParser::TOKEN, i);\n}\n\n\nsize_t WhooshParser::TermContext::getRuleIndex() const {\n return WhooshParser::RuleTerm;\n}\n\nvoid WhooshParser::TermContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterTerm(this);\n}\n\nvoid WhooshParser::TermContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitTerm(this);\n}\n\n\nstd::any WhooshParser::TermContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitTerm(this);\n else\n return visitor->visitChildren(this);\n}\n\nWhooshParser::TermContext* WhooshParser::term() {\n TermContext *_localctx = _tracker.createInstance(_ctx, getState());\n enterRule(_localctx, 0, WhooshParser::RuleTerm);\n\n#if __cplusplus > 201703L\n auto onExit = finally([=, this] {\n#else\n auto onExit = finally([=] {\n#endif\n exitRule();\n });\n try {\n enterOuterAlt(_localctx, 1);\n setState(4);\n match(WhooshParser::TOKEN);\n setState(5);\n match(WhooshParser::T__0);\n setState(6);\n match(WhooshParser::TOKEN);\n \n }\n catch (RecognitionException &e) {\n _errHandler->reportError(this, e);\n _localctx->exception = std::current_exception();\n _errHandler->recover(this, _localctx->exception);\n }\n\n return _localctx;\n}\n\n//----------------- ExprContext ------------------------------------------------------------------\n\nWhooshParser::ExprContext::ExprContext(ParserRuleContext *parent, size_t invokingState)\n : ParserRuleContext(parent, invokingState) {\n}\n\n\nsize_t WhooshParser::ExprContext::getRuleIndex() const {\n return WhooshParser::RuleExpr;\n}\n\nvoid WhooshParser::ExprContext::copyFrom(ExprContext *ctx) {\n ParserRuleContext::copyFrom(ctx);\n}\n\n//----------------- WrappedExprContext ------------------------------------------------------------------\n\nWhooshParser::ExprContext* WhooshParser::WrappedExprContext::expr() {\n return getRuleContext(0);\n}\n\nWhooshParser::WrappedExprContext::WrappedExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::WrappedExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterWrappedExpr(this);\n}\nvoid WhooshParser::WrappedExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitWrappedExpr(this);\n}\n\nstd::any WhooshParser::WrappedExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitWrappedExpr(this);\n else\n return visitor->visitChildren(this);\n}\n//----------------- NotExprContext ------------------------------------------------------------------\n\nWhooshParser::ExprContext* WhooshParser::NotExprContext::expr() {\n return getRuleContext(0);\n}\n\nWhooshParser::NotExprContext::NotExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::NotExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterNotExpr(this);\n}\nvoid WhooshParser::NotExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitNotExpr(this);\n}\n\nstd::any WhooshParser::NotExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitNotExpr(this);\n else\n return visitor->visitChildren(this);\n}\n//----------------- TermExprContext ------------------------------------------------------------------\n\nWhooshParser::TermContext* WhooshParser::TermExprContext::term() {\n return getRuleContext(0);\n}\n\nWhooshParser::TermExprContext::TermExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::TermExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterTermExpr(this);\n}\nvoid WhooshParser::TermExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitTermExpr(this);\n}\n\nstd::any WhooshParser::TermExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitTermExpr(this);\n else\n return visitor->visitChildren(this);\n}\n//----------------- OrExprContext ------------------------------------------------------------------\n\nstd::vector WhooshParser::OrExprContext::expr() {\n return getRuleContexts();\n}\n\nWhooshParser::ExprContext* WhooshParser::OrExprContext::expr(size_t i) {\n return getRuleContext(i);\n}\n\nWhooshParser::OrExprContext::OrExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::OrExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterOrExpr(this);\n}\nvoid WhooshParser::OrExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitOrExpr(this);\n}\n\nstd::any WhooshParser::OrExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitOrExpr(this);\n else\n return visitor->visitChildren(this);\n}\n//----------------- XorExprContext ------------------------------------------------------------------\n\nstd::vector WhooshParser::XorExprContext::expr() {\n return getRuleContexts();\n}\n\nWhooshParser::ExprContext* WhooshParser::XorExprContext::expr(size_t i) {\n return getRuleContext(i);\n}\n\nWhooshParser::XorExprContext::XorExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::XorExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterXorExpr(this);\n}\nvoid WhooshParser::XorExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitXorExpr(this);\n}\n\nstd::any WhooshParser::XorExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitXorExpr(this);\n else\n return visitor->visitChildren(this);\n}\n//----------------- AndExprContext ------------------------------------------------------------------\n\nstd::vector WhooshParser::AndExprContext::expr() {\n return getRuleContexts();\n}\n\nWhooshParser::ExprContext* WhooshParser::AndExprContext::expr(size_t i) {\n return getRuleContext(i);\n}\n\nWhooshParser::AndExprContext::AndExprContext(ExprContext *ctx) { copyFrom(ctx); }\n\nvoid WhooshParser::AndExprContext::enterRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->enterAndExpr(this);\n}\nvoid WhooshParser::AndExprContext::exitRule(tree::ParseTreeListener *listener) {\n auto parserListener = dynamic_cast(listener);\n if (parserListener != nullptr)\n parserListener->exitAndExpr(this);\n}\n\nstd::any WhooshParser::AndExprContext::accept(tree::ParseTreeVisitor *visitor) {\n if (auto parserVisitor = dynamic_cast(visitor))\n return parserVisitor->visitAndExpr(this);\n else\n return visitor->visitChildren(this);\n}\n\nWhooshParser::ExprContext* WhooshParser::expr() {\n return expr(0);\n}\n\nWhooshParser::ExprContext* WhooshParser::expr(int precedence) {\n ParserRuleContext *parentContext = _ctx;\n size_t parentState = getState();\n WhooshParser::ExprContext *_localctx = _tracker.createInstance(_ctx, parentState);\n WhooshParser::ExprContext *previousContext = _localctx;\n (void)previousContext; // Silence compiler, in case the context is not used by generated code.\n size_t startState = 2;\n enterRecursionRule(_localctx, 2, WhooshParser::RuleExpr, precedence);\n\n \n\n#if __cplusplus > 201703L\n auto onExit = finally([=, this] {\n#else\n auto onExit = finally([=] {\n#endif\n unrollRecursionContexts(parentContext);\n });\n try {\n size_t alt;\n enterOuterAlt(_localctx, 1);\n setState(16);\n _errHandler->sync(this);\n switch (_input->LA(1)) {\n case WhooshParser::TOKEN: {\n _localctx = _tracker.createInstance(_localctx);\n _ctx = _localctx;\n previousContext = _localctx;\n\n setState(9);\n term();\n break;\n }\n\n case WhooshParser::T__1: {\n _localctx = _tracker.createInstance(_localctx);\n _ctx = _localctx;\n previousContext = _localctx;\n setState(10);\n match(WhooshParser::T__1);\n setState(11);\n expr(0);\n setState(12);\n match(WhooshParser::T__2);\n break;\n }\n\n case WhooshParser::T__6: {\n _localctx = _tracker.createInstance(_localctx);\n _ctx = _localctx;\n previousContext = _localctx;\n setState(14);\n match(WhooshParser::T__6);\n setState(15);\n antlrcpp::downCast(_localctx)->right = expr(2);\n break;\n }\n\n default:\n throw NoViableAltException(this);\n }\n _ctx->stop = _input->LT(-1);\n setState(31);\n _errHandler->sync(this);\n alt = getInterpreter()->adaptivePredict(_input, 2, _ctx);\n while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) {\n if (alt == 1) {\n if (!_parseListeners.empty())\n triggerExitRuleEvent();\n previousContext = _localctx;\n setState(29);\n _errHandler->sync(this);\n switch (getInterpreter()->adaptivePredict(_input, 1, _ctx)) {\n case 1: {\n auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState));\n _localctx = newContext;\n newContext->left = previousContext;\n pushNewRecursionContext(newContext, startState, RuleExpr);\n setState(18);\n\n if (!(precpred(_ctx, 5))) throw FailedPredicateException(this, \"precpred(_ctx, 5)\");\n setState(19);\n match(WhooshParser::T__3);\n setState(20);\n antlrcpp::downCast(_localctx)->right = expr(6);\n break;\n }\n\n case 2: {\n auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState));\n _localctx = newContext;\n newContext->left = previousContext;\n pushNewRecursionContext(newContext, startState, RuleExpr);\n setState(21);\n\n if (!(precpred(_ctx, 4))) throw FailedPredicateException(this, \"precpred(_ctx, 4)\");\n setState(22);\n match(WhooshParser::T__4);\n setState(23);\n antlrcpp::downCast(_localctx)->right = expr(5);\n break;\n }\n\n case 3: {\n auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState));\n _localctx = newContext;\n newContext->left = previousContext;\n pushNewRecursionContext(newContext, startState, RuleExpr);\n setState(24);\n\n if (!(precpred(_ctx, 3))) throw FailedPredicateException(this, \"precpred(_ctx, 3)\");\n setState(25);\n match(WhooshParser::T__5);\n setState(26);\n antlrcpp::downCast(_localctx)->right = expr(4);\n break;\n }\n\n case 4: {\n auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState));\n _localctx = newContext;\n newContext->left = previousContext;\n pushNewRecursionContext(newContext, startState, RuleExpr);\n setState(27);\n\n if (!(precpred(_ctx, 1))) throw FailedPredicateException(this, \"precpred(_ctx, 1)\");\n setState(28);\n antlrcpp::downCast(_localctx)->right = expr(2);\n break;\n }\n\n default:\n break;\n } \n }\n setState(33);\n _errHandler->sync(this);\n alt = getInterpreter()->adaptivePredict(_input, 2, _ctx);\n }\n }\n catch (RecognitionException &e) {\n _errHandler->reportError(this, e);\n _localctx->exception = std::current_exception();\n _errHandler->recover(this, _localctx->exception);\n }\n return _localctx;\n}\n\nbool WhooshParser::sempred(RuleContext *context, size_t ruleIndex, size_t predicateIndex) {\n switch (ruleIndex) {\n case 1: return exprSempred(antlrcpp::downCast(context), predicateIndex);\n\n default:\n break;\n }\n return true;\n}\n\nbool WhooshParser::exprSempred(ExprContext *_localctx, size_t predicateIndex) {\n switch (predicateIndex) {\n case 0: return precpred(_ctx, 5);\n case 1: return precpred(_ctx, 4);\n case 2: return precpred(_ctx, 3);\n case 3: return precpred(_ctx, 1);\n\n default:\n break;\n }\n return true;\n}\n\nvoid WhooshParser::initialize() {\n#if ANTLR4_USE_THREAD_LOCAL_CACHE\n whooshParserInitialize();\n#else\n ::antlr4::internal::call_once(whooshParserOnceFlag, whooshParserInitialize);\n#endif\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.484220Z","iopub.execute_input":"2024-07-24T22:02:11.484818Z","iopub.status.idle":"2024-07-24T22:02:11.509770Z","shell.execute_reply.started":"2024-07-24T22:02:11.484762Z","shell.execute_reply":"2024-07-24T22:02:11.508284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshParser.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n\n\nnamespace whoosh {\n\n\nclass WhooshParser : public antlr4::Parser {\npublic:\n enum {\n T__0 = 1, T__1 = 2, T__2 = 3, T__3 = 4, T__4 = 5, T__5 = 6, T__6 = 7, \n WS = 8, TOKEN = 9\n };\n\n enum {\n RuleTerm = 0, RuleExpr = 1\n };\n\n explicit WhooshParser(antlr4::TokenStream *input);\n\n WhooshParser(antlr4::TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options);\n\n ~WhooshParser() override;\n\n std::string getGrammarFileName() const override;\n\n const antlr4::atn::ATN& getATN() const override;\n\n const std::vector& getRuleNames() const override;\n\n const antlr4::dfa::Vocabulary& getVocabulary() const override;\n\n antlr4::atn::SerializedATNView getSerializedATN() const override;\n\n\n class TermContext;\n class ExprContext; \n\n class TermContext : public antlr4::ParserRuleContext {\n public:\n TermContext(antlr4::ParserRuleContext *parent, size_t invokingState);\n virtual size_t getRuleIndex() const override;\n std::vector TOKEN();\n antlr4::tree::TerminalNode* TOKEN(size_t i);\n\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n \n };\n\n TermContext* term();\n\n class ExprContext : public antlr4::ParserRuleContext {\n public:\n ExprContext(antlr4::ParserRuleContext *parent, size_t invokingState);\n \n ExprContext() = default;\n void copyFrom(ExprContext *context);\n using antlr4::ParserRuleContext::copyFrom;\n\n virtual size_t getRuleIndex() const override;\n\n \n };\n\n class WrappedExprContext : public ExprContext {\n public:\n WrappedExprContext(ExprContext *ctx);\n\n ExprContext *expr();\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n class NotExprContext : public ExprContext {\n public:\n NotExprContext(ExprContext *ctx);\n\n WhooshParser::ExprContext *right = nullptr;\n ExprContext *expr();\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n class TermExprContext : public ExprContext {\n public:\n TermExprContext(ExprContext *ctx);\n\n TermContext *term();\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n class OrExprContext : public ExprContext {\n public:\n OrExprContext(ExprContext *ctx);\n\n WhooshParser::ExprContext *left = nullptr;\n WhooshParser::ExprContext *right = nullptr;\n std::vector expr();\n ExprContext* expr(size_t i);\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n class XorExprContext : public ExprContext {\n public:\n XorExprContext(ExprContext *ctx);\n\n WhooshParser::ExprContext *left = nullptr;\n WhooshParser::ExprContext *right = nullptr;\n std::vector expr();\n ExprContext* expr(size_t i);\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n class AndExprContext : public ExprContext {\n public:\n AndExprContext(ExprContext *ctx);\n\n WhooshParser::ExprContext *left = nullptr;\n WhooshParser::ExprContext *right = nullptr;\n std::vector expr();\n ExprContext* expr(size_t i);\n virtual void enterRule(antlr4::tree::ParseTreeListener *listener) override;\n virtual void exitRule(antlr4::tree::ParseTreeListener *listener) override;\n\n virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;\n };\n\n ExprContext* expr();\n ExprContext* expr(int precedence);\n\n bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override;\n\n bool exprSempred(ExprContext *_localctx, size_t predicateIndex);\n\n // By default the static state used to implement the parser is lazily initialized during the first\n // call to the constructor. You can call this function if you wish to initialize the static state\n // ahead of time.\n static void initialize();\n\nprivate:\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.511710Z","iopub.execute_input":"2024-07-24T22:02:11.512192Z","iopub.status.idle":"2024-07-24T22:02:11.532161Z","shell.execute_reply.started":"2024-07-24T22:02:11.512152Z","shell.execute_reply":"2024-07-24T22:02:11.530606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshVisitor.cpp\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n\n#include \"WhooshVisitor.h\"\n\n\nusing namespace whoosh;","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.533934Z","iopub.execute_input":"2024-07-24T22:02:11.534416Z","iopub.status.idle":"2024-07-24T22:02:11.552509Z","shell.execute_reply.started":"2024-07-24T22:02:11.534372Z","shell.execute_reply":"2024-07-24T22:02:11.551171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/uspto/whoosh/WhooshVisitor.h\n// Generated from src/uspto/whoosh/Whoosh.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"WhooshParser.h\"\n\n\nnamespace whoosh {\n\n/**\n * This class defines an abstract visitor for a parse tree\n * produced by WhooshParser.\n */\nclass WhooshVisitor : public antlr4::tree::AbstractParseTreeVisitor {\npublic:\n\n /**\n * Visit parse trees produced by WhooshParser.\n */\n virtual std::any visitTerm(WhooshParser::TermContext *context) = 0;\n\n virtual std::any visitWrappedExpr(WhooshParser::WrappedExprContext *context) = 0;\n\n virtual std::any visitNotExpr(WhooshParser::NotExprContext *context) = 0;\n\n virtual std::any visitTermExpr(WhooshParser::TermExprContext *context) = 0;\n\n virtual std::any visitOrExpr(WhooshParser::OrExprContext *context) = 0;\n\n virtual std::any visitXorExpr(WhooshParser::XorExprContext *context) = 0;\n\n virtual std::any visitAndExpr(WhooshParser::AndExprContext *context) = 0;\n\n\n};\n\n} // namespace whoosh","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.554381Z","iopub.execute_input":"2024-07-24T22:02:11.554931Z","iopub.status.idle":"2024-07-24T22:02:11.568378Z","shell.execute_reply.started":"2024-07-24T22:02:11.554885Z","shell.execute_reply":"2024-07-24T22:02:11.566926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -f /opt/conda/lib/libcurl*","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:11.570127Z","iopub.execute_input":"2024-07-24T22:02:11.570655Z","iopub.status.idle":"2024-07-24T22:02:12.761029Z","shell.execute_reply.started":"2024-07-24T22:02:11.570594Z","shell.execute_reply":"2024-07-24T22:02:12.758853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/uspto-explainable-ai-ensemble-dependencies/.conan2 /root/.conan2","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:02:12.763597Z","iopub.execute_input":"2024-07-24T22:02:12.764272Z","iopub.status.idle":"2024-07-24T22:03:36.547896Z","shell.execute_reply.started":"2024-07-24T22:02:12.764208Z","shell.execute_reply":"2024-07-24T22:03:36.546065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p vendor","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:03:36.550429Z","iopub.execute_input":"2024-07-24T22:03:36.551084Z","iopub.status.idle":"2024-07-24T22:03:37.727788Z","shell.execute_reply.started":"2024-07-24T22:03:36.551010Z","shell.execute_reply":"2024-07-24T22:03:37.725889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/christian-borgelt-fp-growth-6-21 vendor/borgelt","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:03:37.729950Z","iopub.execute_input":"2024-07-24T22:03:37.730476Z","iopub.status.idle":"2024-07-24T22:03:39.176193Z","shell.execute_reply.started":"2024-07-24T22:03:37.730414Z","shell.execute_reply":"2024-07-24T22:03:39.174383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/jason-l-causey-min-max-heap vendor/min-max_heap","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:03:39.178517Z","iopub.execute_input":"2024-07-24T22:03:39.179026Z","iopub.status.idle":"2024-07-24T22:03:40.376245Z","shell.execute_reply.started":"2024-07-24T22:03:39.178979Z","shell.execute_reply":"2024-07-24T22:03:40.374412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=/kaggle/input/uspto-explainable-ai-ensemble-dependencies/build/Release/generators/conan_toolchain.cmake","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:03:40.378547Z","iopub.execute_input":"2024-07-24T22:03:40.379058Z","iopub.status.idle":"2024-07-24T22:03:43.950649Z","shell.execute_reply.started":"2024-07-24T22:03:40.379010Z","shell.execute_reply":"2024-07-24T22:03:43.949328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!make -j","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:03:43.952883Z","iopub.execute_input":"2024-07-24T22:03:43.953366Z","iopub.status.idle":"2024-07-24T22:04:18.173716Z","shell.execute_reply.started":"2024-07-24T22:03:43.953322Z","shell.execute_reply":"2024-07-24T22:04:18.171998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!./run-submission","metadata":{"execution":{"iopub.status.busy":"2024-07-24T22:04:18.182725Z","iopub.execute_input":"2024-07-24T22:04:18.183218Z","iopub.status.idle":"2024-07-24T22:06:48.268899Z","shell.execute_reply.started":"2024-07-24T22:04:18.183177Z","shell.execute_reply":"2024-07-24T22:06:48.266292Z"},"trusted":true},"execution_count":null,"outputs":[]}]}