{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":177382875,"sourceType":"kernelVersion"},{"sourceId":177932770,"sourceType":"kernelVersion"},{"sourceId":185015269,"sourceType":"kernelVersion"},{"sourceId":187064915,"sourceType":"kernelVersion"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\n\nimport whoosh_utils","metadata":{"execution":{"iopub.status.busy":"2024-07-06T07:20:31.378123Z","iopub.execute_input":"2024-07-06T07:20:31.378540Z","iopub.status.idle":"2024-07-06T07:21:05.342590Z","shell.execute_reply.started":"2024-07-06T07:20:31.378496Z","shell.execute_reply":"2024-07-06T07:21:05.341539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle","metadata":{"execution":{"iopub.status.busy":"2024-07-06T07:21:05.344671Z","iopub.execute_input":"2024-07-06T07:21:05.345231Z","iopub.status.idle":"2024-07-06T07:21:05.350529Z","shell.execute_reply.started":"2024-07-06T07:21:05.345191Z","shell.execute_reply":"2024-07-06T07:21:05.349347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\nbase_path = '/kaggle/input/precompute-cpc-title-desc-max-10000-word-to-int/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set = pickle.load(open(base_path + 'cpc_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount = pickle.load(open(base_path + 'cpc_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word = pickle.load(open(base_path + 'publication_to_cpc.pkl', 'rb'))\n","metadata":{"execution":{"iopub.status.busy":"2024-07-06T07:21:05.351801Z","iopub.execute_input":"2024-07-06T07:21:05.352119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\nbase_path = '/kaggle/input/precompute-claim-set-max-1000/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number_claim_only = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word_claim_only = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set_claim_only = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount_claim_only = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word_claim_only = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n\npub_to_number = pickle.load(open(base_path + 'pub_to_number.pkl', 'rb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(publication_to_word), len(publication_to_word_claim_only)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(number_to_word_claim_only.keys())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# publicationを数値に変換","metadata":{}},{"cell_type":"code","source":"\"\"\"\nfor word, pub_set in word_to_pub_set.items():\n    pub_set = set([pub_to_number[pub] for pub in pub_set])\n    word_to_pub_set[word] = pub_set\npublication_to_word_new = defaultdict(list)\nfor pub, word in publication_to_word.items():\n    pub = pub_to_number[pub]\n    publication_to_word_new[pub] = word\npublication_to_word = publication_to_word_new\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数値を元のpublicationに変換","metadata":{}},{"cell_type":"code","source":"number_to_pub = dict()\nfor pub, number in pub_to_number.items():\n    number_to_pub[number] = pub\n\nfor word, pub_set in word_to_pub_set_claim_only.items():\n    pub_set = set([number_to_pub[pub] for pub in pub_set])\n    word_to_pub_set_claim_only[word] = pub_set\npublication_to_word_claim_only_new = defaultdict(list)\nfor pub, word in publication_to_word_claim_only.items():\n    pub = number_to_pub[pub]\n    publication_to_word_claim_only_new[pub] = word\npublication_to_word_claim_only = publication_to_word_claim_only_new","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub, publication_to_word_claim_only_new[pub]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# claimを追加","metadata":{}},{"cell_type":"code","source":"for k, v in word_to_number_claim_only.items():\n    word_to_number[k] = v\nfor k, v in number_to_word_claim_only.items():\n    number_to_word[k] = v\nfor k, v in word_to_pub_set_claim_only.items():\n    word_to_pub_set[k] = v\nfor k, v in word_to_pubcount_claim_only.items():\n    word_to_pubcount[k] = v","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, v in tqdm(publication_to_word_claim_only.items()):\n    publication_to_word[k] += publication_to_word_claim_only[k]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_number, open(f'word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(f'number_to_word.pkl', 'wb'))\npickle.dump(word_to_pub_set, open(f'word_to_pub_set.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(f'word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(f'publication_to_word.pkl', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}