{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":8479599,"sourceType":"datasetVersion","datasetId":4517815},{"sourceId":8723242,"sourceType":"datasetVersion","datasetId":5234797},{"sourceId":8792952,"sourceType":"datasetVersion","datasetId":5286840},{"sourceId":174185912,"sourceType":"kernelVersion"},{"sourceId":189238016,"sourceType":"kernelVersion"},{"sourceId":189247894,"sourceType":"kernelVersion"}],"dockerImageVersionId":30666,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install kaggle","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nimport os\n\n\"\"\"\nhttps://www.kaggle.com/code/tanakar/calc-sub-time\n\"\"\"\n\nKAGGLE_USERNAME = 'tanakar' # Change to your username\nuser_secrets = UserSecretsClient()\nos.environ[\"KAGGLE_USERNAME\"] = KAGGLE_USERNAME\nos.environ[\"KAGGLE_KEY\"] = user_secrets.get_secret(\"kaggle_api\") # Change to the key listed in kaggle.json","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:30:41.677019Z","iopub.execute_input":"2024-07-22T07:30:41.677428Z","iopub.status.idle":"2024-07-22T07:30:41.912157Z","shell.execute_reply.started":"2024-07-22T07:30:41.677380Z","shell.execute_reply":"2024-07-22T07:30:41.910972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle.api.kaggle_api_extended import KaggleApi\nimport json\n\ndef make_kaggle_dataset(title, output_dir):\n    # baseline_name = 'feedback-3-starter'\n\n\n    ID = 'tanakar' # '<自分のID>'\n    DATASET_DIR = title # '<datasetのpath>'\n    DATASET_TITLE  = title # '<datasetのタイトル>'\n    FOLDER_DIR = output_dir # '<データセットにアップロードするフォルダ>'\n\n    # shutil.copy(f'{exp_name}/myutils/config.py', FOLDER_DIR)\n    \n    dataset_metadata = {}\n    dataset_metadata['id'] = f'{ID}/{DATASET_DIR}'\n    dataset_metadata['licenses'] = [{'name': 'CC0-1.0'}]\n    dataset_metadata['title'] = DATASET_TITLE\n\n    with open(FOLDER_DIR + '/dataset-metadata.json', 'w') as f:\n        json.dump(dataset_metadata, f, indent=4)\n\n    api = KaggleApi()\n    api.authenticate()\n    api.dataset_create_new(folder=FOLDER_DIR, convert_to_csv=False, dir_mode='zip')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:30:41.914088Z","iopub.execute_input":"2024-07-22T07:30:41.914430Z","iopub.status.idle":"2024-07-22T07:30:42.138285Z","shell.execute_reply.started":"2024-07-22T07:30:41.914402Z","shell.execute_reply":"2024-07-22T07:30:42.137042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = 'uspto-dataset-debug2'\noutput_dir = f'/kaggle/working/{title}/'\nos.makedirs(output_dir, exist_ok=True)\n\npath_w = output_dir + 'test_w.txt'\n\ns = 'New file'\n\nwith open(path_w, mode='w') as f:\n    f.write(s)\n\nmake_kaggle_dataset(title, output_dir)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:30:42.139666Z","iopub.execute_input":"2024-07-22T07:30:42.140024Z","iopub.status.idle":"2024-07-22T07:30:45.196013Z","shell.execute_reply.started":"2024-07-22T07:30:42.139995Z","shell.execute_reply":"2024-07-22T07:30:45.194694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\n\nimport whoosh_utils","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実験内容  \ntitle cpc abst claim  \nhttps://www.kaggle.com/code/tanakar/precompute-cpc-title-abst-claim-max-inf\n\ndescription  \n/kaggle/input/merge-description-chunk-title-all","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 重複削除","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os\n\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\n# base_path = '/kaggle/input/precompute-claim-set-max-10000-add-claims-lowmem/'\n# base_path = '/kaggle/input/precompute-claim-set-max-10000-add-claims-allyear/'\n# base_path = '/kaggle/input/precompute-claim-set-max-10000-add-claims-fix/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-100000-allyear-claim/'\nbase_path = '/kaggle/input/precompute-cpc-title-abst-claim-max-inf/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n\n\n# 重複削除\nseen = set()\nremove_words = []\n# for word, number in tqdm(list(word_to_number.items())[::-1]): # cpc以外を優先したいので、逆順\nfor word in tqdm(list(word_to_pub_set.keys())[::-1]): # cpc以外を優先したいので、逆順\n    key = tuple(sorted(word_to_pub_set[word]))\n    \n    if key in seen:\n        remove_words.append(word)\n    else:\n        seen.add(key)\n\nprint('len(remove_words)', len(remove_words))\nfor word in remove_words:\n    word_to_pub_set.pop(word)\n    word_to_pubcount.pop(word)\n    \nprint(len(word_to_pub_set))\n\nremove_words = set(remove_words)\nfor pub in publication_to_word.keys():\n    publication_to_word[pub] = [word for word in publication_to_word[pub] if word not in remove_words]\n\noutput_dir = 'base/'\nos.makedirs(output_dir, exist_ok=True)\n\npickle.dump(word_to_number, open(output_dir+f'word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(output_dir+f'number_to_word.pkl', 'wb'))\npickle.dump(word_to_pub_set, open(output_dir+f'word_to_pub_set.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(output_dir+f'word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(output_dir+f'publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:29:02.589474Z","iopub.execute_input":"2024-07-09T06:29:02.591072Z","iopub.status.idle":"2024-07-09T06:38:36.000405Z","shell.execute_reply.started":"2024-07-09T06:29:02.591024Z","shell.execute_reply":"2024-07-09T06:38:35.999202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\n# base_path = '/kaggle/input/precompute-claim-set-max-10000-add-claims-lowmem/'\n\nbase_path = './base/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:40:46.067093Z","iopub.execute_input":"2024-07-09T06:40:46.067674Z","iopub.status.idle":"2024-07-09T06:45:08.316676Z","shell.execute_reply.started":"2024-07-09T06:40:46.067616Z","shell.execute_reply":"2024-07-09T06:45:08.315169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\nimport pickle\n# base_path = '/kaggle/input/upsto-cpc-baseline-all-feature-low-memory/'\n# base_path = '/kaggle/input/precompute-cpc-title-desc-max-10000/'\n# base_path = './desc/'\nbase_path = '/kaggle/input/merge-description-chunk-title-all/desc/'\n\n# wordを数値に変換 (省メモリ化のため)\nword_to_number_claim_only = pickle.load(open(base_path + 'word_to_number.pkl', 'rb'))\n\n# 数値をwordに変換 例: number_to_word[ti:device] → 10\nnumber_to_word_claim_only = pickle.load(open(base_path + 'number_to_word.pkl', 'rb'))\n\n\"\"\"\n特定のwordを持つpublication_numberの集合 word(cpc,title,abstract)\nrecallなどを高速に計算するために、集合で保持\n例: word_to_pub_set[cpc1] → set([pub1, pub2, pub3])\n\"\"\"\nword_to_pub_set_claim_only = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))\n\n# 特定のwordを持つpublication_numberの個数 個数は1975年以上のpublicationで事前計算\nword_to_pubcount_claim_only = pickle.load(open(base_path + 'word_to_pubcount.pkl', 'rb'))\n\n# 特定のpublication_numberが持つword\npublication_to_word_claim_only = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))\n\n# pub_to_number = pickle.load(open(base_path + 'pub_to_number.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:45:08.31929Z","iopub.execute_input":"2024-07-09T06:45:08.319799Z","iopub.status.idle":"2024-07-09T06:47:15.653477Z","shell.execute_reply.started":"2024-07-09T06:45:08.319758Z","shell.execute_reply":"2024-07-09T06:47:15.652303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(publication_to_word), len(publication_to_word_claim_only)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:47:15.655129Z","iopub.execute_input":"2024-07-09T06:47:15.655808Z","iopub.status.idle":"2024-07-09T06:47:15.662716Z","shell.execute_reply.started":"2024-07-09T06:47:15.655777Z","shell.execute_reply":"2024-07-09T06:47:15.661512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(number_to_word_claim_only.keys())","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:47:15.664816Z","iopub.execute_input":"2024-07-09T06:47:15.665133Z","iopub.status.idle":"2024-07-09T06:47:15.884522Z","shell.execute_reply.started":"2024-07-09T06:47:15.665103Z","shell.execute_reply":"2024-07-09T06:47:15.88332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# いらないデータを削除","metadata":{}},{"cell_type":"code","source":"import shutil\nshutil.rmtree('./base/')\n# shutil.rmtree('./desc/')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# claim (今回はdescription)を追加","metadata":{}},{"cell_type":"code","source":"for k, v in word_to_number_claim_only.items():\n    word_to_number[k] = v\nfor k, v in number_to_word_claim_only.items():\n    number_to_word[k] = v\nfor k, v in word_to_pub_set_claim_only.items():\n    word_to_pub_set[k] = v\nfor k, v in word_to_pubcount_claim_only.items():\n    word_to_pubcount[k] = v","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:47:15.885777Z","iopub.execute_input":"2024-07-09T06:47:15.886189Z","iopub.status.idle":"2024-07-09T06:47:34.19458Z","shell.execute_reply.started":"2024-07-09T06:47:15.886155Z","shell.execute_reply":"2024-07-09T06:47:34.193483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, v in tqdm(publication_to_word_claim_only.items()):\n    publication_to_word[k] += publication_to_word_claim_only[k]","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:47:34.195871Z","iopub.execute_input":"2024-07-09T06:47:34.19618Z","iopub.status.idle":"2024-07-09T06:47:48.886233Z","shell.execute_reply.started":"2024-07-09T06:47:34.196154Z","shell.execute_reply":"2024-07-09T06:47:48.884965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 重複削除","metadata":{}},{"cell_type":"code","source":"\"\"\"\n# 重複削除\nseen = set()\nremove_words = []\nfor word, number in tqdm(list(word_to_number.items())[::-1]):\n    key = tuple(sorted(word_to_pub_set[word]))\n    \n    if key in seen:\n        remove_words.append(word)\n    else:\n        seen.add(key)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-09T05:58:00.912176Z","iopub.execute_input":"2024-07-09T05:58:00.913181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# 重複削除\nseen = set()\nremove_words = []\n# for word, number in tqdm(list(word_to_number.items())[::-1]): # cpc以外を優先したいので、逆順\nfor word in tqdm(list(word_to_pub_set.keys())[::-1]): # cpc以外を優先したいので、逆順\n    key = tuple(sorted(word_to_pub_set[word]))\n    \n    if key in seen:\n        remove_words.append(word)\n    else:\n        seen.add(key)\n\nprint('len(remove_words)', len(remove_words))\nfor word in remove_words:\n    word_to_pub_set.pop(word)\n    word_to_pubcount.pop(word)\n    \nprint(len(word_to_pub_set))\n\nremove_words = set(remove_words)\nfor pub in publication_to_word.keys():\n    publication_to_word[pub] = [word for word in publication_to_word[pub] if word not in remove_words]\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = 'cpc-title-abst-claim-inf'\noutput_dir = f'/tmp/{title}/'\nos.makedirs(output_dir, exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(word_to_number, open(output_dir + 'word_to_number.pkl', 'wb'))\npickle.dump(number_to_word, open(output_dir + 'number_to_word.pkl', 'wb'))\npickle.dump(word_to_pub_set, open(output_dir + 'word_to_pub_set.pkl', 'wb'))\npickle.dump(word_to_pubcount, open(output_dir + 'word_to_pubcount.pkl', 'wb'))\npickle.dump(publication_to_word, open(output_dir + 'publication_to_word.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-09T06:48:10.596006Z","iopub.execute_input":"2024-07-09T06:48:10.59649Z","iopub.status.idle":"2024-07-09T06:50:44.498339Z","shell.execute_reply.started":"2024-07-09T06:48:10.596456Z","shell.execute_reply":"2024-07-09T06:50:44.496941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_kaggle_dataset(title, output_dir)","metadata":{},"execution_count":null,"outputs":[]}]}