{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from boto3.session import Session\nimport boto3\nimport os\nfrom tqdm import tqdm\nimport glob\nfrom kaggle_secrets import UserSecretsClient\n\n\n# s3-subfolder-user\nuser_secrets = UserSecretsClient()\nBUCKET_NAME = 'darraghdog2'\nACCESS_KEY = user_secrets.get_secret(\"aws_access_key_id\")\nSECRET_KEY = user_secrets.get_secret(\"aws_secret_access_key\")\n\nsession = Session(aws_access_key_id=ACCESS_KEY,\n              aws_secret_access_key=SECRET_KEY)\ns3 = session.resource('s3')\nyour_bucket = s3.Bucket(BUCKET_NAME)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T22:09:46.347626Z","iopub.execute_input":"2025-01-04T22:09:46.348025Z","iopub.status.idle":"2025-01-04T22:09:47.501122Z","shell.execute_reply.started":"2025-01-04T22:09:46.34797Z","shell.execute_reply":"2025-01-04T22:09:47.500066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fns = []\nfor s3_file in your_bucket.objects.all():\n    fns += [s3_file.key]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T22:09:47.503226Z","iopub.execute_input":"2025-01-04T22:09:47.503606Z","iopub.status.idle":"2025-01-04T22:09:54.226294Z","shell.execute_reply.started":"2025-01-04T22:09:47.503579Z","shell.execute_reply":"2025-01-04T22:09:54.22514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fns  = [fn for fn in fns if 'Qwen2.5-14B-Instruct-MIX48__rd190325_trtllm' in fn ]\n# fns  = [fn for fn in fns if any(i in fn for i in 'rank0 rank1'.split()) ]\nfns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T22:09:54.227599Z","iopub.execute_input":"2025-01-04T22:09:54.227946Z","iopub.status.idle":"2025-01-04T22:09:54.241398Z","shell.execute_reply.started":"2025-01-04T22:09:54.227919Z","shell.execute_reply":"2025-01-04T22:09:54.240319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fn in tqdm(fns):\n    target_folder = '/'.join(fn.split('/')[:-1]).replace('kaggle-aimo2/kaggle/isorokin/','').strip('/')\n    target_file = fn.split('/')[-1]\n    os.makedirs(target_folder, exist_ok=True)\n    print(fn, ' -> ', target_folder + '/' + target_file)\n    s3.Bucket(BUCKET_NAME).download_file(fn, target_folder + '/' + target_file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T22:11:29.090583Z","iopub.execute_input":"2025-01-04T22:11:29.092133Z","iopub.status.idle":"2025-01-04T22:12:54.206082Z","shell.execute_reply.started":"2025-01-04T22:11:29.092081Z","shell.execute_reply":"2025-01-04T22:12:54.204863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!du -sh ./*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T22:12:54.208229Z","iopub.execute_input":"2025-01-04T22:12:54.208669Z","iopub.status.idle":"2025-01-04T22:12:55.289948Z","shell.execute_reply.started":"2025-01-04T22:12:54.208621Z","shell.execute_reply":"2025-01-04T22:12:55.288847Z"}},"outputs":[],"execution_count":null}]}