{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":177382875,"sourceType":"kernelVersion"},{"sourceId":177932770,"sourceType":"kernelVersion"},{"sourceId":185015269,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\n\nimport whoosh_utils","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:05.192021Z","iopub.execute_input":"2024-07-11T00:14:05.192298Z","iopub.status.idle":"2024-07-11T00:14:12.018871Z","shell.execute_reply.started":"2024-07-11T00:14:05.192267Z","shell.execute_reply":"2024-07-11T00:14:12.018182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容\ndescription用のdatasetを分割して作る  \nchunk4はOOMしたので、さらに1/2  \nこれでもOOMしたので、tpuでやる","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:12.020173Z","iopub.execute_input":"2024-07-11T00:14:12.020531Z","iopub.status.idle":"2024-07-11T00:14:18.284008Z","shell.execute_reply.started":"2024-07-11T00:14:12.020504Z","shell.execute_reply":"2024-07-11T00:14:18.283076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.285332Z","iopub.execute_input":"2024-07-11T00:14:18.285624Z","iopub.status.idle":"2024-07-11T00:14:18.2923Z","shell.execute_reply.started":"2024-07-11T00:14:18.285594Z","shell.execute_reply":"2024-07-11T00:14:18.291694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_count = multiprocessing.cpu_count()\nprint('cpu_count', cpu_count)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.294004Z","iopub.execute_input":"2024-07-11T00:14:18.294279Z","iopub.status.idle":"2024-07-11T00:14:18.302029Z","shell.execute_reply.started":"2024-07-11T00:14:18.294252Z","shell.execute_reply":"2024-07-11T00:14:18.301439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.302757Z","iopub.execute_input":"2024-07-11T00:14:18.303Z","iopub.status.idle":"2024-07-11T00:14:18.311787Z","shell.execute_reply.started":"2024-07-11T00:14:18.302974Z","shell.execute_reply":"2024-07-11T00:14:18.311228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search_patent_paths():\n    patent_all_paths = glob.glob('/kaggle/input/uspto-explainable-ai/patent_data/*.parquet')\n\n    patent_paths = []\n    for path in tqdm(patent_all_paths):\n        try:\n            year = int(path.split('/')[-1].split('_')[0])\n        except Exception as e:\n            print(path)\n            continue\n        if year >= 1975:\n            patent_paths.append(path)\n    \n    patent_paths = sorted(patent_paths)\n    \n    print(len(patent_all_paths), len(patent_paths))\n    \n    return patent_paths\n","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.312613Z","iopub.execute_input":"2024-07-11T00:14:18.312836Z","iopub.status.idle":"2024-07-11T00:14:18.322292Z","shell.execute_reply.started":"2024-07-11T00:14:18.312814Z","shell.execute_reply":"2024-07-11T00:14:18.321697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = search_patent_paths()","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.323092Z","iopub.execute_input":"2024-07-11T00:14:18.323336Z","iopub.status.idle":"2024-07-11T00:14:18.531159Z","shell.execute_reply.started":"2024-07-11T00:14:18.323311Z","shell.execute_reply":"2024-07-11T00:14:18.530568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\n\nis_first_half = False\n\nif is_first_half:\n    start_idx = i*145\n    end_idx = start_idx + 90\nelse:\n    start_idx = i*145 + 90\n    end_idx = len(paths)\n\nprint(start_idx, end_idx ,len(paths[start_idx:end_idx]))\n\npaths = paths[start_idx:end_idx]","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.531972Z","iopub.execute_input":"2024-07-11T00:14:18.532198Z","iopub.status.idle":"2024-07-11T00:14:18.537003Z","shell.execute_reply.started":"2024-07-11T00:14:18.532177Z","shell.execute_reply":"2024-07-11T00:14:18.536398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh, re\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.537736Z","iopub.execute_input":"2024-07-11T00:14:18.537977Z","iopub.status.idle":"2024-07-11T00:14:18.547995Z","shell.execute_reply.started":"2024-07-11T00:14:18.537953Z","shell.execute_reply":"2024-07-11T00:14:18.547377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Todo  \ntpuマシン使って、並列化で早く終わらせる  \ntpu使うときは、Internet Onにする","metadata":{}},{"cell_type":"code","source":"def read_pub_desc(path):\n    df = pd.read_parquet(path, columns=['publication_number', 'description'])\n    \n    pub_list = df['publication_number'].tolist()\n    desc_list = df['description'].tolist()\n    \n    return pub_list, desc_list","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.55008Z","iopub.execute_input":"2024-07-11T00:14:18.550308Z","iopub.status.idle":"2024-07-11T00:14:18.558306Z","shell.execute_reply.started":"2024-07-11T00:14:18.550286Z","shell.execute_reply":"2024-07-11T00:14:18.557731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _make_word_to_pub_set(path):\n    \n\n    BRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n            'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\n\n    custom_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()\n    \n    MAX_NUM = 100000 # 10000\n\n    word_to_pub_set = defaultdict(list)\n    remove_word_set = set()\n\n    pub_list, desc_list = read_pub_desc(path)\n\n    for pub, description in zip(pub_list, desc_list): \n        \n        description = ' '.join(set(description.split()))\n\n        description = [token.text for token in custom_analyzer(description)]\n        description = list(set(description))\n\n        for word in description:\n            word = 'detd:' + word\n\n            if word in remove_word_set:\n                continue\n\n            word_to_pub_set[word].append(pub)\n\n            # 個数が多すぎるものは使わない\n            if len(word_to_pub_set[word]) == MAX_NUM:\n                word_to_pub_set.pop(word)\n                remove_word_set.add(word)\n\n    gc.collect()\n\n    return word_to_pub_set, remove_word_set","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.559034Z","iopub.execute_input":"2024-07-11T00:14:18.559253Z","iopub.status.idle":"2024-07-11T00:14:18.568399Z","shell.execute_reply.started":"2024-07-11T00:14:18.55923Z","shell.execute_reply":"2024-07-11T00:14:18.567753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_word_to_pub_set(paths_chunk, processes=None):\n    if processes is None:\n        processes = multiprocessing.cpu_count()\n\n    print('processes:', processes)\n    with multiprocessing.Pool(processes=processes) as pool:\n        results = pool.imap_unordered(_make_word_to_pub_set, paths_chunk)\n        results = tqdm(results)\n        results = list(results)\n    # df = pd.concat(dfs)\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.569193Z","iopub.execute_input":"2024-07-11T00:14:18.569402Z","iopub.status.idle":"2024-07-11T00:14:18.582364Z","shell.execute_reply.started":"2024-07-11T00:14:18.56938Z","shell.execute_reply":"2024-07-11T00:14:18.581784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set_list = make_word_to_pub_set(paths, processes=20)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:14:18.58318Z","iopub.execute_input":"2024-07-11T00:14:18.583434Z","iopub.status.idle":"2024-07-11T00:27:44.089775Z","shell.execute_reply.started":"2024-07-11T00:14:18.583409Z","shell.execute_reply":"2024-07-11T00:27:44.088326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set = defaultdict(list)\nremove_word_set = set()\n\nMAX_NUM = 100000\n\ncounts = []\n# remove_word_setをchunkので初期化\nfor _, _remove_word_set in word_to_pub_set_list:\n    remove_word_set |= _remove_word_set\n    counts.append(len(_remove_word_set))\nplt.hist(counts)\nplt.title('len(_remove_word_set)')\nplt.show()\n\nfor word_to_pub_set_chunk, _ in word_to_pub_set_list:\n    for word, pub_set in tqdm(word_to_pub_set_chunk.items()):\n        if word in remove_word_set:\n            continue\n        \n        word_to_pub_set[word].extend(pub_set)\n        \n        if len(word_to_pub_set[word]) > MAX_NUM:\n            word_to_pub_set.pop(word)\n            remove_word_set.add(word)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:30:36.517988Z","iopub.execute_input":"2024-07-11T00:30:36.518526Z","iopub.status.idle":"2024-07-11T00:32:13.393795Z","shell.execute_reply.started":"2024-07-11T00:30:36.518483Z","shell.execute_reply":"2024-07-11T00:32:13.392759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(word_to_pub_set), len(remove_word_set)","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:32:13.395336Z","iopub.execute_input":"2024-07-11T00:32:13.395607Z","iopub.status.idle":"2024-07-11T00:32:13.401004Z","shell.execute_reply.started":"2024-07-11T00:32:13.395582Z","shell.execute_reply":"2024-07-11T00:32:13.400281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_pub_set, open(f'word_to_pub_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:32:13.401891Z","iopub.execute_input":"2024-07-11T00:32:13.402155Z","iopub.status.idle":"2024-07-11T00:32:43.204351Z","shell.execute_reply.started":"2024-07-11T00:32:13.402127Z","shell.execute_reply":"2024-07-11T00:32:43.203468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(remove_word_set, open(f'remove_word_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-11T00:32:43.206149Z","iopub.execute_input":"2024-07-11T00:32:43.206456Z","iopub.status.idle":"2024-07-11T00:32:43.233849Z","shell.execute_reply.started":"2024-07-11T00:32:43.206426Z","shell.execute_reply":"2024-07-11T00:32:43.233068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}