{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":177382875,"sourceType":"kernelVersion"},{"sourceId":177932770,"sourceType":"kernelVersion"},{"sourceId":185015269,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\n\nimport whoosh_utils","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:06.574769Z","iopub.execute_input":"2024-07-11T00:05:06.575034Z","iopub.status.idle":"2024-07-11T00:05:13.130982Z","shell.execute_reply.started":"2024-07-11T00:05:06.575005Z","shell.execute_reply":"2024-07-11T00:05:13.13027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容\n全ての年でclaimやり直し  \nchunkのマージに失敗したのでやり直し","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:13.132238Z","iopub.execute_input":"2024-07-11T00:05:13.132588Z","iopub.status.idle":"2024-07-11T00:05:18.865695Z","shell.execute_reply.started":"2024-07-11T00:05:13.132562Z","shell.execute_reply":"2024-07-11T00:05:18.864732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:18.866993Z","iopub.execute_input":"2024-07-11T00:05:18.867269Z","iopub.status.idle":"2024-07-11T00:05:18.873342Z","shell.execute_reply.started":"2024-07-11T00:05:18.867241Z","shell.execute_reply":"2024-07-11T00:05:18.872771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_count = multiprocessing.cpu_count()\nprint('cpu_count', cpu_count)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:18.874957Z","iopub.execute_input":"2024-07-11T00:05:18.875219Z","iopub.status.idle":"2024-07-11T00:05:18.881794Z","shell.execute_reply.started":"2024-07-11T00:05:18.875192Z","shell.execute_reply":"2024-07-11T00:05:18.881189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:18.882635Z","iopub.execute_input":"2024-07-11T00:05:18.882879Z","iopub.status.idle":"2024-07-11T00:05:18.88984Z","shell.execute_reply.started":"2024-07-11T00:05:18.882856Z","shell.execute_reply":"2024-07-11T00:05:18.889243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search_patent_paths():\n    patent_all_paths = glob.glob('/kaggle/input/uspto-explainable-ai/patent_data/*.parquet')\n\n    patent_paths = []\n    for path in tqdm(patent_all_paths):\n        try:\n            year = int(path.split('/')[-1].split('_')[0])\n        except Exception as e:\n            print(path)\n            continue\n        \"\"\"\n        if year >= 1975:\n            patent_paths.append(path)\n        \"\"\"\n        patent_paths.append(path)\n    patent_paths = sorted(patent_paths)\n    \n    print(len(patent_all_paths), len(patent_paths))\n    \n    return patent_paths\n","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:18.890628Z","iopub.execute_input":"2024-07-11T00:05:18.890867Z","iopub.status.idle":"2024-07-11T00:05:18.898709Z","shell.execute_reply.started":"2024-07-11T00:05:18.890843Z","shell.execute_reply":"2024-07-11T00:05:18.898121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = search_patent_paths()","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:18.89958Z","iopub.execute_input":"2024-07-11T00:05:18.899842Z","iopub.status.idle":"2024-07-11T00:05:19.004049Z","shell.execute_reply.started":"2024-07-11T00:05:18.899818Z","shell.execute_reply":"2024-07-11T00:05:19.003431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh, re\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:19.004975Z","iopub.execute_input":"2024-07-11T00:05:19.005229Z","iopub.status.idle":"2024-07-11T00:05:19.009139Z","shell.execute_reply.started":"2024-07-11T00:05:19.005202Z","shell.execute_reply":"2024-07-11T00:05:19.008546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Todo  \ntpuマシン使って、並列化で早く終わらせる  \ntpu使うときは、Internet Onにする","metadata":{}},{"cell_type":"code","source":"def read_pub_desc(path):\n    df = pd.read_parquet(path, columns=['publication_number', 'claims'])\n    \n    pub_list = df['publication_number'].tolist()\n    desc_list = df['claims'].tolist()\n    \n    return pub_list, desc_list","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:19.00989Z","iopub.execute_input":"2024-07-11T00:05:19.010099Z","iopub.status.idle":"2024-07-11T00:05:19.069172Z","shell.execute_reply.started":"2024-07-11T00:05:19.010077Z","shell.execute_reply":"2024-07-11T00:05:19.068535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _make_word_to_pub_set(path):\n    \n\n    BRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n            'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\n\n    custom_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()\n    \n    MAX_NUM = 100000 # 10000\n\n    word_to_pub_set = defaultdict(list)\n    remove_word_set = set()\n\n    pub_list, desc_list = read_pub_desc(path)\n\n    for pub, claims in zip(pub_list, desc_list): \n        \n        claims = ' '.join(set(claims.split()))\n\n        claims = [token.text for token in custom_analyzer(claims)]\n        claims = list(set(claims))\n\n        for word in claims:\n            word = 'clm:' + word\n\n            if word in remove_word_set:\n                continue\n\n            word_to_pub_set[word].append(pub)\n\n            # 個数が多すぎるものは使わない\n            if len(word_to_pub_set[word]) == MAX_NUM:\n                word_to_pub_set.pop(word)\n                remove_word_set.add(word)\n\n    gc.collect()\n\n    return word_to_pub_set, remove_word_set","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:19.071249Z","iopub.execute_input":"2024-07-11T00:05:19.071497Z","iopub.status.idle":"2024-07-11T00:05:19.172883Z","shell.execute_reply.started":"2024-07-11T00:05:19.071472Z","shell.execute_reply":"2024-07-11T00:05:19.172207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_word_to_pub_set(paths_chunk, processes=None):\n    if processes is None:\n        processes = multiprocessing.cpu_count()\n\n    print('processes:', processes)\n    with multiprocessing.Pool(processes=processes) as pool:\n        results = pool.imap_unordered(_make_word_to_pub_set, paths_chunk)\n        results = tqdm(results)\n        results = list(results)\n    # df = pd.concat(dfs)\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:19.173777Z","iopub.execute_input":"2024-07-11T00:05:19.17401Z","iopub.status.idle":"2024-07-11T00:05:19.470331Z","shell.execute_reply.started":"2024-07-11T00:05:19.173987Z","shell.execute_reply":"2024-07-11T00:05:19.469518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set_list = make_word_to_pub_set(paths, processes=48)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:05:19.471407Z","iopub.execute_input":"2024-07-11T00:05:19.471722Z","iopub.status.idle":"2024-07-11T00:08:27.42373Z","shell.execute_reply.started":"2024-07-11T00:05:19.471666Z","shell.execute_reply":"2024-07-11T00:08:27.422808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = 0\nfor word_to_pub_set_chunk, remove_word_set in word_to_pub_set_list:\n    if len(word_to_pub_set_chunk) == 0:\n        cnt += 1\ncnt","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:08:27.424972Z","iopub.execute_input":"2024-07-11T00:08:27.425243Z","iopub.status.idle":"2024-07-11T00:08:27.435251Z","shell.execute_reply.started":"2024-07-11T00:08:27.425215Z","shell.execute_reply":"2024-07-11T00:08:27.434599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set = defaultdict(list)\nremove_word_set = set()\n\nMAX_NUM = 100000\n\n# remove_word_setをchunkので初期化\nfor _, _remove_word_set in word_to_pub_set_list:\n    remove_word_set |= _remove_word_set\n\nprint('len(remove_word_set)', len(remove_word_set))\n\nfor word_to_pub_set_chunk, _ in word_to_pub_set_list:\n    for word, pub_set in tqdm(word_to_pub_set_chunk.items()):\n        if word in remove_word_set:\n            continue\n        \n        word_to_pub_set[word].extend(pub_set)\n        \n        if len(word_to_pub_set[word]) > MAX_NUM:\n            word_to_pub_set.pop(word)\n            remove_word_set.add(word)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:08:27.436217Z","iopub.execute_input":"2024-07-11T00:08:27.436517Z","iopub.status.idle":"2024-07-11T00:09:25.567463Z","shell.execute_reply.started":"2024-07-11T00:08:27.436489Z","shell.execute_reply":"2024-07-11T00:09:25.56679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(word_to_pub_set), len(remove_word_set)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:11:34.682197Z","iopub.execute_input":"2024-07-11T00:11:34.68255Z","iopub.status.idle":"2024-07-11T00:11:34.687574Z","shell.execute_reply.started":"2024-07-11T00:11:34.68252Z","shell.execute_reply":"2024-07-11T00:11:34.686894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_pub_set, open(f'word_to_pub_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:09:25.57424Z","iopub.execute_input":"2024-07-11T00:09:25.574523Z","iopub.status.idle":"2024-07-11T00:09:49.811969Z","shell.execute_reply.started":"2024-07-11T00:09:25.574495Z","shell.execute_reply":"2024-07-11T00:09:49.81121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(remove_word_set, open(f'remove_word_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:11:32.045988Z","iopub.execute_input":"2024-07-11T00:11:32.046336Z","iopub.status.idle":"2024-07-11T00:11:32.05508Z","shell.execute_reply.started":"2024-07-11T00:11:32.046294Z","shell.execute_reply":"2024-07-11T00:11:32.054377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}