{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from boto3.session import Session\nimport boto3\nimport os\nfrom tqdm import tqdm\nimport glob\n\n# s3-subfolder-user\n\nBUCKET_NAME = 'kaggle-birdclef24'\nACCESS_KEY = 'AKIAZHGZ27KU3E3LD4T4'\nSECRET_KEY = '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'\n\nsession = Session(aws_access_key_id=ACCESS_KEY,\n              aws_secret_access_key=SECRET_KEY)\ns3 = session.resource('s3')\nyour_bucket = s3.Bucket(BUCKET_NAME)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T15:55:09.718897Z","iopub.execute_input":"2024-06-04T15:55:09.719284Z","iopub.status.idle":"2024-06-04T15:55:10.782096Z","shell.execute_reply.started":"2024-06-04T15:55:09.719251Z","shell.execute_reply":"2024-06-04T15:55:10.781091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns = []\nfor s3_file in your_bucket.objects.all():\n    fns += [s3_file.key]","metadata":{"execution":{"iopub.status.busy":"2024-06-04T15:55:39.895514Z","iopub.execute_input":"2024-06-04T15:55:39.896166Z","iopub.status.idle":"2024-06-04T15:55:41.149684Z","shell.execute_reply.started":"2024-06-04T15:55:39.896133Z","shell.execute_reply":"2024-06-04T15:55:41.148337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = 'CH_pl_13/checkpoints/bird_003/exp_0'","metadata":{"execution":{"iopub.status.busy":"2024-06-04T15:55:41.151572Z","iopub.execute_input":"2024-06-04T15:55:41.151918Z","iopub.status.idle":"2024-06-04T15:55:41.156847Z","shell.execute_reply.started":"2024-06-04T15:55:41.151883Z","shell.execute_reply":"2024-06-04T15:55:41.155611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns  = [fn for fn in fns if f'{CFG}/' in fn ]\nfns","metadata":{"execution":{"iopub.status.busy":"2024-06-04T15:55:41.158267Z","iopub.execute_input":"2024-06-04T15:55:41.158866Z","iopub.status.idle":"2024-06-04T15:55:41.171407Z","shell.execute_reply.started":"2024-06-04T15:55:41.158833Z","shell.execute_reply":"2024-06-04T15:55:41.170108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fn in tqdm(fns):\n    target_folder = '/'.join(fn.split('/')[:-1]).replace(f'{CFG}/','')\n    target_file = fn.split('/')[-1]\n    os.makedirs(target_folder, exist_ok=True)\n    print(fn, ' -> ', target_folder + '/' + target_file)\n    s3.Bucket(BUCKET_NAME).download_file(fn, target_folder + '/' + target_file)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T15:55:50.977071Z","iopub.execute_input":"2024-06-04T15:55:50.977487Z","iopub.status.idle":"2024-06-04T15:56:00.000119Z","shell.execute_reply.started":"2024-06-04T15:55:50.977456Z","shell.execute_reply":"2024-06-04T15:55:59.999097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}