{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from google.cloud import storage\nfrom pathlib import Path\nfrom multiprocessing import Pool\nfrom functools import partial\nfrom tqdm import tqdm\n\ndef download(client, bucket, prefix, relative):\n    for blob in client.list_blobs(bucket, prefix=prefix):\n        name = Path(blob.name).relative_to(relative)\n        name.parent.mkdir(parents=True, exist_ok=True)\n        blob.download_to_filename(name)\n        print(f\"downloaded {name}\")\n\ndef _check_blob(blob, bucket_obj):\n    if Path(blob.name).suffix != \".ckpt\":\n        return False\n    success = Path(blob.name).parent.parent / \"_SUCCESS\"\n    return bucket_obj.blob(success.as_posix()).exists()\n        \ndef _download_blob(blob, relative):\n    name = Path(blob.name).relative_to(relative)\n    name.parent.mkdir(parents=True, exist_ok=True)\n    blob.download_to_filename(name)\n    print(f\"downloaded {name}\")\n        \ndef download_success(client, bucket, prefix, relative):\n    # stupid hack\n    bucket_obj = client.get_bucket(bucket)\n    to_download = []\n    \n    blobs = list(tqdm(client.list_blobs(bucket, prefix=prefix)))\n    for blob, is_success in tqdm(zip(blobs, map(partial(_check_blob, bucket_obj=bucket_obj), blobs))):\n        if not is_success:\n            continue\n        to_download.append(blob)\n    for _ in map(partial(_download_blob, relative=relative), tqdm(to_download)):\n        pass\n    \nclient = storage.Client(project=\"dsgt-clef-2024\")\ndownload_success(client, \"dsgt-clef-birdclef-2024\", \"models\", \"models\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-21T17:05:27.007385Z","iopub.execute_input":"2024-06-21T17:05:27.007781Z","iopub.status.idle":"2024-06-21T17:09:06.371481Z","shell.execute_reply.started":"2024-06-21T17:05:27.007750Z","shell.execute_reply":"2024-06-21T17:09:06.370335Z"},"trusted":true},"execution_count":null,"outputs":[]}]}