{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"},{"sourceId":9008587,"sourceType":"datasetVersion","datasetId":5427363}],"dockerImageVersionId":30748,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install kaggle","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nimport os\n\n\"\"\"\nhttps://www.kaggle.com/code/tanakar/calc-sub-time\n\"\"\"\n\nKAGGLE_USERNAME = 'tanakar' # Change to your username\nuser_secrets = UserSecretsClient()\nos.environ[\"KAGGLE_USERNAME\"] = KAGGLE_USERNAME\nos.environ[\"KAGGLE_KEY\"] = user_secrets.get_secret(\"kaggle_api\") # Change to the key listed in kaggle.json","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle.api.kaggle_api_extended import KaggleApi\nimport json\n\ndef make_kaggle_dataset(title, output_dir):\n    # baseline_name = 'feedback-3-starter'\n\n\n    ID = 'tanakar' # '<自分のID>'\n    DATASET_DIR = title # '<datasetのpath>'\n    DATASET_TITLE  = title # '<datasetのタイトル>'\n    FOLDER_DIR = output_dir # '<データセットにアップロードするフォルダ>'\n\n    # shutil.copy(f'{exp_name}/myutils/config.py', FOLDER_DIR)\n    \n    dataset_metadata = {}\n    dataset_metadata['id'] = f'{ID}/{DATASET_DIR}'\n    dataset_metadata['licenses'] = [{'name': 'CC0-1.0'}]\n    dataset_metadata['title'] = DATASET_TITLE\n\n    with open(FOLDER_DIR + '/dataset-metadata.json', 'w') as f:\n        json.dump(dataset_metadata, f, indent=4)\n\n    api = KaggleApi()\n    api.authenticate()\n    api.dataset_create_new(folder=FOLDER_DIR, convert_to_csv=False, dir_mode='zip')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = 'uspto-dataset-debug3'\noutput_dir = f'/kaggle/working/{title}/'\nos.makedirs(output_dir, exist_ok=True)\n\npath_w = output_dir + 'test_w.txt'\n\ns = 'New file'\n\nwith open(path_w, mode='w') as f:\n    f.write(s)\n\nmake_kaggle_dataset(title, output_dir)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport glob\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport json\nimport itertools\nimport matplotlib.pyplot as plt\nimport copy\nimport math\nfrom collections import Counter\nimport numpy as np\nimport gc\nimport time\nimport random\nimport pickle\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"これを一度に読み込むとOOMするので、chunkに  \nhttps://www.kaggle.com/code/tanakar/cpc-title-abst-claim-desc-100000-v2/notebook?scriptVersionId=189039731","metadata":{}},{"cell_type":"code","source":"# base_path = '/kaggle/input/cpc-title-abst-claim-desc-100000-v2/'\nbase_path = '/kaggle/input/cpc-title-abst-claim-inf/'","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:14:14.857821Z","iopub.execute_input":"2024-07-14T06:14:14.858845Z","iopub.status.idle":"2024-07-14T06:14:14.862859Z","shell.execute_reply.started":"2024-07-14T06:14:14.858807Z","shell.execute_reply":"2024-07-14T06:14:14.861921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"publication_to_word = pickle.load(open(base_path + 'publication_to_word.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:14:16.372043Z","iopub.execute_input":"2024-07-14T06:14:16.372428Z","iopub.status.idle":"2024-07-14T06:16:06.129297Z","shell.execute_reply.started":"2024-07-14T06:14:16.372395Z","shell.execute_reply":"2024-07-14T06:16:06.128525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_to_pub_set = pickle.load(open(base_path + 'word_to_pub_set.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:16:06.130542Z","iopub.execute_input":"2024-07-14T06:16:06.13079Z","iopub.status.idle":"2024-07-14T06:20:46.459099Z","shell.execute_reply.started":"2024-07-14T06:16:06.130766Z","shell.execute_reply":"2024-07-14T06:20:46.458241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CHUNK_NUM = 20 # 10","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_size = len(publication_to_word) // CHUNK_NUM\nchunk_dict = [defaultdict(list) for i in range(CHUNK_NUM+1)]\n\nfor i, key in tqdm(enumerate(publication_to_word.keys())):\n    chunk_idx = i // chunk_size\n    value = publication_to_word[key]\n    chunk_dict[chunk_idx][key] = value","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:22:29.318046Z","iopub.execute_input":"2024-07-14T06:22:29.318459Z","iopub.status.idle":"2024-07-14T06:22:42.606779Z","shell.execute_reply.started":"2024-07-14T06:22:29.318426Z","shell.execute_reply":"2024-07-14T06:22:42.606101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = 'cpc-title-abst-claim-inf-chunk'\noutput_dir = f'/tmp/{title}/'\nos.makedirs(output_dir, exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(CHUNK_NUM+1):\n    path = output_dir + f'publication_to_word_{i}.pkl'\n    pickle.dump(chunk_dict[i], open(path, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:26:14.794591Z","iopub.execute_input":"2024-07-14T06:26:14.795432Z","iopub.status.idle":"2024-07-14T06:26:43.032827Z","shell.execute_reply.started":"2024-07-14T06:26:14.795395Z","shell.execute_reply":"2024-07-14T06:26:43.031993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_size = len(word_to_pub_set) // CHUNK_NUM\nchunk_dict = [defaultdict(list) for i in range(CHUNK_NUM+1)]\n\nfor i, key in tqdm(enumerate(word_to_pub_set.keys())):\n    chunk_idx = i // chunk_size\n    value = word_to_pub_set[key]\n    chunk_dict[chunk_idx][key] = value","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:27:07.323857Z","iopub.execute_input":"2024-07-14T06:27:07.324255Z","iopub.status.idle":"2024-07-14T06:27:27.532302Z","shell.execute_reply.started":"2024-07-14T06:27:07.32422Z","shell.execute_reply":"2024-07-14T06:27:27.53164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(CHUNK_NUM+1):\n    path = output_dir + f'word_to_pub_set_{i}.pkl'\n    pickle.dump(chunk_dict[i], open(path, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2024-07-14T06:27:27.533507Z","iopub.execute_input":"2024-07-14T06:27:27.533754Z","iopub.status.idle":"2024-07-14T06:30:07.232159Z","shell.execute_reply.started":"2024-07-14T06:27:27.533729Z","shell.execute_reply":"2024-07-14T06:30:07.231219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_kaggle_dataset(title, output_dir)","metadata":{},"execution_count":null,"outputs":[]}]}