{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":177382875,"sourceType":"kernelVersion"},{"sourceId":187731620,"sourceType":"kernelVersion"},{"sourceId":188662347,"sourceType":"kernelVersion"},{"sourceId":189059712,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\n\nimport whoosh_utils","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:06:31.196035Z","iopub.execute_input":"2024-07-10T12:06:31.196566Z","iopub.status.idle":"2024-07-10T12:06:31.202221Z","shell.execute_reply.started":"2024-07-10T12:06:31.196532Z","shell.execute_reply":"2024-07-10T12:06:31.200759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容  \n\ncpc,title,abstをこっちに  \nhttps://www.kaggle.com/code/tanakar/precompute-cpc-title-desc-max-inf\nclaimをこっちに変更  \nhttps://www.kaggle.com/code/tanakar/preprocess-claim-max-100000?scriptVersionId=188662347","metadata":{}},{"cell_type":"markdown","source":"# OOMしないように、先にlistにする","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport pickle\nimport os\n\n\noutput_dir = 'tmp1/'\nos.makedirs(output_dir)\n\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-word-to-int/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-allyear/'\nbase_path = '/kaggle/input/precompute-cpc-title-desc-max-inf/'\nword_to_pub_set = pickle.load(open(base_path + 'cpc_to_pub_set.pkl', 'rb'))\n\nfor word, pub_set in word_to_pub_set.items():\n    word_to_pub_set[word] = list(pub_set)\n\npickle.dump(word_to_pub_set, open(output_dir + 'word_to_pub_set.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:06:31.203703Z","iopub.execute_input":"2024-07-10T12:06:31.204273Z","iopub.status.idle":"2024-07-10T12:11:07.120258Z","shell.execute_reply.started":"2024-07-10T12:06:31.204234Z","shell.execute_reply":"2024-07-10T12:11:07.118886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport os\nimport pickle\n\noutput_dir = 'tmp2/'\nos.makedirs(output_dir, exist_ok=True)\n\n# base_path = '/kaggle/input/precompute-claim-set-max-10000/'\n# base_path = '/kaggle/input/preprocess-claim-tpu-year-all/'\n# base_path = '/kaggle/input/preprocess-claim-tpu-year-all-fix-chunk-merge/'\nbase_path = '/kaggle/input/preprocess-claim-max-100000/'\n\nword_to_pub_set = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))\n\n# for word, pub_set in word_to_pub_set.items():\n#     word_to_pub_set[word] = list(pub_set)\n\n# pickle.dump(word_to_pub_set, open(output_dir + 'word_to_pub_set.pkl', 'wb'))\n\nword_to_number = dict()\nnumber_to_word = dict()\n\nuse_words = sorted(word_to_pub_set.keys())\n\n# base_path = '/kaggle/input/precompute-claim-set-max-10000-add-claims-lowmem/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-allyear/'\nbase_path = '/kaggle/input/precompute-cpc-title-desc-max-inf/'\nword_to_number_base = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\nfor i, word in enumerate(use_words):\n    i = len(word_to_number_base) + i # 後で、base_path内のword_to_numberと結合するため\n    word_to_number[word] = i\n    number_to_word[i] = word\n\nmin(number_to_word.keys()), max(number_to_word.keys())\n\nword_to_pub_set_new = defaultdict(list)\nfor word, pub_set in word_to_pub_set.items():\n    word_to_pub_set_new[word_to_number[word]] = pub_set\nword_to_pub_set = word_to_pub_set_new\n\nword_to_pubcount = defaultdict(int)\nfor word, pub_set in word_to_pub_set.items():\n    word_to_pubcount[word] = len(pub_set)\n    \npublication_to_word = defaultdict(list)\nfor word, pub_set in tqdm(word_to_pub_set.items()):\n    for pub in pub_set:\n        publication_to_word[pub].append(word)\n    \n\nprint('len(publication_to_word)', len(publication_to_word))\n        \n\npickle.dump(word_to_number, open(output_dir+f'word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(output_dir+f'number_to_word.pkl', 'wb'))\npickle.dump(word_to_pub_set, open(output_dir+f'word_to_pub_set.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(output_dir+f'word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(output_dir+f'publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:14:11.834104Z","iopub.execute_input":"2024-07-10T12:14:11.834514Z","iopub.status.idle":"2024-07-10T12:18:24.701307Z","shell.execute_reply.started":"2024-07-10T12:14:11.834483Z","shell.execute_reply":"2024-07-10T12:18:24.700021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-word-to-int/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-allyear/'\nbase_path = '/kaggle/input/precompute-cpc-title-desc-max-inf/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set = pickle.load(open('tmp1/' + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount = pickle.load(open(base_path + 'cpc_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word = pickle.load(open(base_path + 'publication_to_cpc.pkl', 'rb'))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:18:24.703891Z","iopub.execute_input":"2024-07-10T12:18:24.7044Z","iopub.status.idle":"2024-07-10T12:21:46.560721Z","shell.execute_reply.started":"2024-07-10T12:18:24.704357Z","shell.execute_reply":"2024-07-10T12:21:46.559609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\n# base_path = '/kaggle/input/precompute-claim-set-max-10000/'\nbase_path = 'tmp2/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number_claim_only = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word_claim_only = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set_claim_only = pickle.load(open('tmp2/' + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount_claim_only = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word_claim_only = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n\n# pub_to_number = pickle.load(open(base_path + 'pub_to_number.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:21:46.562532Z","iopub.execute_input":"2024-07-10T12:21:46.562957Z","iopub.status.idle":"2024-07-10T12:22:56.648423Z","shell.execute_reply.started":"2024-07-10T12:21:46.562907Z","shell.execute_reply":"2024-07-10T12:22:56.647267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(publication_to_word), len(publication_to_word_claim_only)","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:22:56.65163Z","iopub.execute_input":"2024-07-10T12:22:56.652131Z","iopub.status.idle":"2024-07-10T12:22:56.660855Z","shell.execute_reply.started":"2024-07-10T12:22:56.652085Z","shell.execute_reply":"2024-07-10T12:22:56.659603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(number_to_word_claim_only.keys())","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:22:56.662568Z","iopub.execute_input":"2024-07-10T12:22:56.663028Z","iopub.status.idle":"2024-07-10T12:22:56.727498Z","shell.execute_reply.started":"2024-07-10T12:22:56.662987Z","shell.execute_reply":"2024-07-10T12:22:56.725858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# claimを追加","metadata":{}},{"cell_type":"code","source":"for k, v in word_to_number_claim_only.items():\n    word_to_number[k] = v\nfor k, v in number_to_word_claim_only.items():\n    number_to_word[k] = v\nfor k, v in word_to_pub_set_claim_only.items():\n    word_to_pub_set[k] = v\nfor k, v in word_to_pubcount_claim_only.items():\n    word_to_pubcount[k] = v","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:22:56.728791Z","iopub.execute_input":"2024-07-10T12:22:56.729272Z","iopub.status.idle":"2024-07-10T12:23:00.951521Z","shell.execute_reply.started":"2024-07-10T12:22:56.729228Z","shell.execute_reply":"2024-07-10T12:23:00.950143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, v in tqdm(publication_to_word_claim_only.items()):\n    publication_to_word[k] += publication_to_word_claim_only[k]","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:23:00.953284Z","iopub.execute_input":"2024-07-10T12:23:00.953756Z","iopub.status.idle":"2024-07-10T12:23:24.650907Z","shell.execute_reply.started":"2024-07-10T12:23:00.953713Z","shell.execute_reply":"2024-07-10T12:23:24.649714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_number, open(f'word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(f'number_to_word.pkl', 'wb'))\npickle.dump(word_to_pub_set, open(f'word_to_pub_set.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(f'word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(f'publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:23:24.652631Z","iopub.execute_input":"2024-07-10T12:23:24.653144Z","iopub.status.idle":"2024-07-10T12:25:05.277987Z","shell.execute_reply.started":"2024-07-10T12:23:24.653101Z","shell.execute_reply":"2024-07-10T12:25:05.275398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n\nshutil.rmtree('tmp1')\nshutil.rmtree('tmp2')","metadata":{"execution":{"iopub.status.busy":"2024-07-10T12:31:39.258172Z","iopub.execute_input":"2024-07-10T12:31:39.259369Z","iopub.status.idle":"2024-07-10T12:31:39.530701Z","shell.execute_reply.started":"2024-07-10T12:31:39.259311Z","shell.execute_reply":"2024-07-10T12:31:39.529533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}