{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":177382875,"sourceType":"kernelVersion"},{"sourceId":177932770,"sourceType":"kernelVersion"},{"sourceId":185015269,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\n\nimport whoosh_utils","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容\nchunkのマージのミスを修正","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:12.352361Z","iopub.execute_input":"2024-07-10T11:43:12.353079Z","iopub.status.idle":"2024-07-10T11:43:18.662179Z","shell.execute_reply.started":"2024-07-10T11:43:12.353035Z","shell.execute_reply":"2024-07-10T11:43:18.66114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.664241Z","iopub.execute_input":"2024-07-10T11:43:18.664542Z","iopub.status.idle":"2024-07-10T11:43:18.672964Z","shell.execute_reply.started":"2024-07-10T11:43:18.664514Z","shell.execute_reply":"2024-07-10T11:43:18.672274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_count = multiprocessing.cpu_count()\nprint('cpu_count', cpu_count)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.673933Z","iopub.execute_input":"2024-07-10T11:43:18.674198Z","iopub.status.idle":"2024-07-10T11:43:18.681936Z","shell.execute_reply.started":"2024-07-10T11:43:18.674171Z","shell.execute_reply":"2024-07-10T11:43:18.681307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.68353Z","iopub.execute_input":"2024-07-10T11:43:18.683809Z","iopub.status.idle":"2024-07-10T11:43:18.692623Z","shell.execute_reply.started":"2024-07-10T11:43:18.683782Z","shell.execute_reply":"2024-07-10T11:43:18.691921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search_patent_paths():\n    patent_all_paths = glob.glob('/kaggle/input/uspto-explainable-ai/patent_data/*.parquet')\n\n    patent_paths = []\n    for path in tqdm(patent_all_paths):\n        try:\n            year = int(path.split('/')[-1].split('_')[0])\n        except Exception as e:\n            print(path)\n            continue\n        # if year >= 1975:\n        if year < 1975:\n            patent_paths.append(path)\n    \n    patent_paths = sorted(patent_paths)\n    \n    print(len(patent_all_paths), len(patent_paths))\n    \n    return patent_paths\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.693543Z","iopub.execute_input":"2024-07-10T11:43:18.693805Z","iopub.status.idle":"2024-07-10T11:43:18.703226Z","shell.execute_reply.started":"2024-07-10T11:43:18.693779Z","shell.execute_reply":"2024-07-10T11:43:18.702502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = search_patent_paths()","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.704101Z","iopub.execute_input":"2024-07-10T11:43:18.704357Z","iopub.status.idle":"2024-07-10T11:43:18.782944Z","shell.execute_reply.started":"2024-07-10T11:43:18.704322Z","shell.execute_reply":"2024-07-10T11:43:18.782115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import whoosh, re\n\nNUMBER_REGEX = re.compile(r'^(\\d+|\\d{1,3}(,\\d{3})*)(\\.\\d+)?$')\nclass NumberFilter(whoosh.analysis.Filter):\n    def __call__(self, tokens):\n        for t in tokens:\n            if not NUMBER_REGEX.match(t.text):\n                yield t","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.783929Z","iopub.execute_input":"2024-07-10T11:43:18.784198Z","iopub.status.idle":"2024-07-10T11:43:18.788596Z","shell.execute_reply.started":"2024-07-10T11:43:18.78417Z","shell.execute_reply":"2024-07-10T11:43:18.787978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Todo  \ntpuマシン使って、並列化で早く終わらせる  \ntpu使うときは、Internet Onにする","metadata":{}},{"cell_type":"code","source":"def read_pub_desc(path):\n    df = pd.read_parquet(path, columns=['publication_number', 'description'])\n    \n    pub_list = df['publication_number'].tolist()\n    desc_list = df['description'].tolist()\n    \n    return pub_list, desc_list","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.789388Z","iopub.execute_input":"2024-07-10T11:43:18.789609Z","iopub.status.idle":"2024-07-10T11:43:18.799753Z","shell.execute_reply.started":"2024-07-10T11:43:18.789585Z","shell.execute_reply":"2024-07-10T11:43:18.799115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _make_word_to_pub_set(path):\n    \n\n    BRS_STOPWORDS = ['an', 'are', 'by', 'for', 'if', 'into', 'is', 'no', 'not', 'of', 'on', 'such',\n            'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will']\n\n    custom_analyzer = whoosh.analysis.StandardAnalyzer(stoplist=BRS_STOPWORDS) | NumberFilter()\n    \n    MAX_NUM = 100000 # 10000\n\n    word_to_pub_set = defaultdict(list)\n    remove_word_set = set()\n\n    pub_list, desc_list = read_pub_desc(path)\n\n    for pub, description in zip(pub_list, desc_list): \n        \n        description = ' '.join(set(description.split()))\n\n        description = [token.text for token in custom_analyzer(description)]\n        description = list(set(description))\n\n        for word in description:\n            word = 'detd:' + word\n\n            if word in remove_word_set:\n                continue\n\n            word_to_pub_set[word].append(pub)\n\n            # 個数が多すぎるものは使わない\n            if len(word_to_pub_set[word]) == MAX_NUM:\n                word_to_pub_set.pop(word)\n                remove_word_set.add(word)\n\n    gc.collect()\n\n    return word_to_pub_set, remove_word_set","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:18.800574Z","iopub.execute_input":"2024-07-10T11:43:18.800829Z","iopub.status.idle":"2024-07-10T11:43:18.812195Z","shell.execute_reply.started":"2024-07-10T11:43:18.800805Z","shell.execute_reply":"2024-07-10T11:43:18.811512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_word_to_pub_set(paths_chunk, processes=None):\n    if processes is None:\n        processes = multiprocessing.cpu_count()\n\n    print('processes:', processes)\n    with multiprocessing.Pool(processes=processes) as pool:\n        results = pool.imap_unordered(_make_word_to_pub_set, paths_chunk)\n        results = tqdm(results)\n        results = list(results)\n    # df = pd.concat(dfs)\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:25.371533Z","iopub.execute_input":"2024-07-10T11:43:25.371931Z","iopub.status.idle":"2024-07-10T11:43:25.376772Z","shell.execute_reply.started":"2024-07-10T11:43:25.371901Z","shell.execute_reply":"2024-07-10T11:43:25.376084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set_list = make_word_to_pub_set(paths, processes=48)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:43:25.559007Z","iopub.execute_input":"2024-07-10T11:43:25.559332Z","iopub.status.idle":"2024-07-10T11:49:41.479926Z","shell.execute_reply.started":"2024-07-10T11:43:25.559304Z","shell.execute_reply":"2024-07-10T11:49:41.478346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = 0\n\nfor word_to_pub_set_chunk in word_to_pub_set_list:\n    if len(word_to_pub_set_chunk) == 0:\n        cnt += 1\n\ncnt","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:55:43.652096Z","iopub.execute_input":"2024-07-10T11:55:43.652345Z","iopub.status.idle":"2024-07-10T11:55:43.658614Z","shell.execute_reply.started":"2024-07-10T11:55:43.652319Z","shell.execute_reply":"2024-07-10T11:55:43.658051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set = defaultdict(list)\nremove_word_set = set()\n\nMAX_NUM = 100000\n\ncounts = []\n# remove_word_setをchunkので初期化\nfor _, _remove_word_set in word_to_pub_set_list:\n    remove_word_set |= _remove_word_set\n    counts.append(len(_remove_word_set))\nplt.hist(counts)\nplt.title('len(_remove_word_set)')\nplt.show()\n\nfor word_to_pub_set_chunk, _ in word_to_pub_set_list:\n    for word, pub_set in tqdm(word_to_pub_set_chunk.items()):\n        if word in remove_word_set:\n            continue\n        \n        word_to_pub_set[word].extend(pub_set)\n        \n        if len(word_to_pub_set[word]) > MAX_NUM:\n            word_to_pub_set.pop(word)\n            remove_word_set.add(word)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:49:41.48193Z","iopub.execute_input":"2024-07-10T11:49:41.482264Z","iopub.status.idle":"2024-07-10T11:54:43.847193Z","shell.execute_reply.started":"2024-07-10T11:49:41.48223Z","shell.execute_reply":"2024-07-10T11:54:43.846462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(word_to_pub_set), len(remove_word_set)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:54:43.848447Z","iopub.execute_input":"2024-07-10T11:54:43.848855Z","iopub.status.idle":"2024-07-10T11:54:43.856411Z","shell.execute_reply.started":"2024-07-10T11:54:43.848823Z","shell.execute_reply":"2024-07-10T11:54:43.855829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_pub_set, open(f'word_to_pub_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-10T11:54:43.857904Z","iopub.execute_input":"2024-07-10T11:54:43.858164Z","iopub.status.idle":"2024-07-10T11:55:43.651009Z","shell.execute_reply.started":"2024-07-10T11:54:43.858137Z","shell.execute_reply":"2024-07-10T11:55:43.650193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(remove_word_set, open(f'remove_word_set.pkl', 'wb'))","metadata":{},"execution_count":null,"outputs":[]}]}