{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8841782,"sourceType":"datasetVersion","datasetId":5321429},{"sourceId":8895545,"sourceType":"datasetVersion","datasetId":5348899},{"sourceId":8938499,"sourceType":"datasetVersion","datasetId":5378087},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":184564185,"sourceType":"kernelVersion"},{"sourceId":184888693,"sourceType":"kernelVersion"},{"sourceId":184888863,"sourceType":"kernelVersion"},{"sourceId":184888979,"sourceType":"kernelVersion"},{"sourceId":184889107,"sourceType":"kernelVersion"},{"sourceId":184889163,"sourceType":"kernelVersion"},{"sourceId":184889258,"sourceType":"kernelVersion"},{"sourceId":184889300,"sourceType":"kernelVersion"},{"sourceId":184889341,"sourceType":"kernelVersion"},{"sourceId":184896183,"sourceType":"kernelVersion"},{"sourceId":184896412,"sourceType":"kernelVersion"},{"sourceId":184896601,"sourceType":"kernelVersion"},{"sourceId":184953693,"sourceType":"kernelVersion"},{"sourceId":184953782,"sourceType":"kernelVersion"},{"sourceId":184953835,"sourceType":"kernelVersion"},{"sourceId":184956014,"sourceType":"kernelVersion"},{"sourceId":184957521,"sourceType":"kernelVersion"},{"sourceId":184964970,"sourceType":"kernelVersion"},{"sourceId":185053572,"sourceType":"kernelVersion"},{"sourceId":185113928,"sourceType":"kernelVersion"},{"sourceId":185127580,"sourceType":"kernelVersion"},{"sourceId":185224792,"sourceType":"kernelVersion"},{"sourceId":185224924,"sourceType":"kernelVersion"},{"sourceId":185225035,"sourceType":"kernelVersion"},{"sourceId":185225107,"sourceType":"kernelVersion"},{"sourceId":185225335,"sourceType":"kernelVersion"},{"sourceId":185276734,"sourceType":"kernelVersion"},{"sourceId":185276963,"sourceType":"kernelVersion"},{"sourceId":185276996,"sourceType":"kernelVersion"},{"sourceId":185372249,"sourceType":"kernelVersion"},{"sourceId":185375760,"sourceType":"kernelVersion"},{"sourceId":185375937,"sourceType":"kernelVersion"},{"sourceId":185748995,"sourceType":"kernelVersion"},{"sourceId":185749137,"sourceType":"kernelVersion"},{"sourceId":186224823,"sourceType":"kernelVersion"},{"sourceId":187073074,"sourceType":"kernelVersion"},{"sourceId":187074438,"sourceType":"kernelVersion"},{"sourceId":187075126,"sourceType":"kernelVersion"},{"sourceId":187075212,"sourceType":"kernelVersion"},{"sourceId":187075290,"sourceType":"kernelVersion"},{"sourceId":187075349,"sourceType":"kernelVersion"},{"sourceId":187075405,"sourceType":"kernelVersion"},{"sourceId":187075446,"sourceType":"kernelVersion"},{"sourceId":187075518,"sourceType":"kernelVersion"},{"sourceId":187075621,"sourceType":"kernelVersion"},{"sourceId":187080002,"sourceType":"kernelVersion"},{"sourceId":187184469,"sourceType":"kernelVersion"},{"sourceId":187184512,"sourceType":"kernelVersion"},{"sourceId":187184562,"sourceType":"kernelVersion"},{"sourceId":187184593,"sourceType":"kernelVersion"},{"sourceId":187184651,"sourceType":"kernelVersion"},{"sourceId":187184780,"sourceType":"kernelVersion"}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import itertools\nimport os\nimport gc\nimport pickle\nimport random\nimport re\nfrom collections import Counter, defaultdict\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-12T12:34:06.598259Z","iopub.execute_input":"2024-07-12T12:34:06.598671Z","iopub.status.idle":"2024-07-12T12:34:07.945288Z","shell.execute_reply.started":"2024-07-12T12:34:06.598624Z","shell.execute_reply":"2024-07-12T12:34:07.944224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIMIT = 31","metadata":{"execution":{"iopub.status.busy":"2024-07-12T12:36:59.600651Z","iopub.execute_input":"2024-07-12T12:36:59.601056Z","iopub.status.idle":"2024-07-12T12:36:59.606319Z","shell.execute_reply.started":"2024-07-12T12:36:59.601024Z","shell.execute_reply":"2024-07-12T12:36:59.604938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cpc\nglobal_counter = pickle.load(open(\"/kaggle/input/uspto-global-cpc-counter/global_cpc_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_cpc_counter.pkl\", \"wb\"))\n\nglobal_counter = pickle.load(open(\"/kaggle/input/uspto-global-cpc-counter/global_comb_cpc_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_comb_cpc_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-22T10:49:07.752819Z","iopub.execute_input":"2024-06-22T10:49:07.753164Z","iopub.status.idle":"2024-06-22T10:51:10.514587Z","shell.execute_reply.started":"2024-06-22T10:49:07.753133Z","shell.execute_reply":"2024-06-22T10:51:10.513433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# title word\nglobal_counter =  pickle.load(open(\"/kaggle/input/uspto-global-title-word-counter/global_title_word_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_title_word_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abstract word\nglobal_counter =  pickle.load(open(\"/kaggle/input/uspto-global-abstract-word-counter/global_abstract_word_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_abstract_word_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# claims word\nfmt_path = \"/kaggle/input/uspto-global-claims-word-counter-part{part}/global_claims_word_counter_part{part}.pkl\"\n\n\nglobal_counter = Counter()\nfor part in range(8):\n    path = fmt_path.format(part=part)\n    if Path(path).exists():\n        _counter = pickle.load(open(path, \"rb\"))\n        global_counter.update(_counter)\n\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(f\"global_claims_word_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-22T10:49:01.064526Z","iopub.execute_input":"2024-06-22T10:49:01.06509Z","iopub.status.idle":"2024-06-22T10:49:06.780439Z","shell.execute_reply.started":"2024-06-22T10:49:01.065053Z","shell.execute_reply":"2024-06-22T10:49:06.778942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# description word\nfiles = [\n    \"/kaggle/input/uspto-global-description-word-counter-part0/global_description_word_counter_part0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part1/global_description_word_counter_part1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part2/global_description_word_counter_part2.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part3/global_description_word_counter_part3.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part4/global_description_word_counter_part4.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part5/global_description_word_counter_part5.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part6/global_description_word_counter_part6.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-0/global_description_word_counter_part7_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-1/global_description_word_counter_part7_1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-2-0/global_description_word_counter_part7_2_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-2-1/global_description_word_counter_part7_2_1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-2-2/global_description_word_counter_part7_2_2.pkl\",\n    \"/kaggle/input/uspto-global-description-word-counter-part7-2-3/global_description_word_counter_part7_2_3.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-0-0/global_description_word_counter_part7_3_0_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-0-1/global_description_word_counter_part7_3_0_1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-1-0/global_description_word_counter_part7_3_1_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-1-1/global_description_word_counter_part7_3_1_1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-2-0/global_description_word_counter_part7_3_2_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-2-1/global_description_word_counter_part7_3_2_1.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-3-0/global_description_word_counter_part7_3_3_0.pkl\",\n    \"/kaggle/input/uspto-global-description-word-couter-part7-3-3-1/global_description_word_counter_part7_3_3_1.pkl\",\n]\nglobal_counter = Counter()\nfor file in tqdm(files):\n    global_counter.update(pickle.load(open(file, \"rb\")))\n\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(f\"global_description_word_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-25T12:52:34.058671Z","iopub.execute_input":"2024-06-25T12:52:34.0592Z","iopub.status.idle":"2024-06-25T12:57:35.604596Z","shell.execute_reply.started":"2024-06-25T12:52:34.059166Z","shell.execute_reply":"2024-06-25T12:57:35.602663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# title bigram\nglobal_counter =  pickle.load(open(\"/kaggle/input/uspto-global-title-bigram-counter-4/global_title_bigram_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_title_bigram_counter.pkl\", \"wb\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abstract bigram\nglobal_counter =  pickle.load(open(\"/kaggle/input/uspto-global-abstract-bigram-counter-4/global_abstract_bigram_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_abstract_bigram_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# メモリで落ちる\n# # claims bigram\n# files = [\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part1/global_claims_bigram_counter_part1.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part2/global_claims_bigram_counter_part2.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part3/global_claims_bigram_counter_part3.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part4/global_claims_bigram_counter_part4.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part5/global_claims_bigram_counter_part5.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part6/global_claims_bigram_counter_part6.pkl\",\n#     \"/kaggle/input/uspto-global-claims-bigram-counter-part7/global_claims_bigram_counter_part7.pkl\",\n# ]\n\n# global_counter = Counter()\n# for file in tqdm(files):\n#     global_counter.update(pickle.load(open(file, \"rb\")))\n# global_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\n# pickle.dump(global_counter, open(f\"global_claims_bigram_counter.pkl\", \"wb\"))\n\n\n# del global_counter\n# gc.collect()\n\n# ローカルで作ったやつをセーブする\nglobal_counter = pickle.load(open(\"/kaggle/input/uspto-global-claims-bigram-counter/global_claims_bigram_counter.pkl\", \"rb\"))\npickle.dump(global_counter, open(f\"global_claims_bigram_counter.pkl\", \"wb\"))\ndel global_counter\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-02T13:47:40.459921Z","iopub.execute_input":"2024-07-02T13:47:40.460582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# title trigram\nglobal_counter = pickle.load(open(\"/kaggle/input/uspto-global-title-trigram-counter/global_title_trigram_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(\"global_title_trigram_counter.pkl\", \"wb\"))\n\n\ndel global_counter\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# abstract trigram\n\n# ローカルで作ったものをセーブする\nglobal_counter = pickle.load(open(\"/kaggle/input/global-abstract-trigram-counter/global_abstract_trigram_counter.pkl\", \"rb\"))\nglobal_counter = Counter({k: v for k, v in global_counter.items() if v < LIMIT})\npickle.dump(global_counter, open(f\"global_abstract_trigram_counter.pkl\", \"wb\"))\n\ndel global_counter\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-12T12:37:02.943667Z","iopub.execute_input":"2024-07-12T12:37:02.944463Z","iopub.status.idle":"2024-07-12T12:39:12.452599Z","shell.execute_reply.started":"2024-07-12T12:37:02.944426Z","shell.execute_reply":"2024-07-12T12:39:12.451452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}