{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39763,"databundleVersionId":11756775,"sourceType":"competition"},{"sourceId":12311405,"sourceType":"datasetVersion","datasetId":7759986},{"sourceId":12313052,"sourceType":"datasetVersion","datasetId":7761113},{"sourceId":12313184,"sourceType":"datasetVersion","datasetId":7761204},{"sourceId":12383096,"sourceType":"datasetVersion","datasetId":7808293}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pickle\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:18.654535Z","iopub.execute_input":"2025-06-28T16:46:18.654875Z","iopub.status.idle":"2025-06-28T16:46:18.659602Z","shell.execute_reply.started":"2025-06-28T16:46:18.654850Z","shell.execute_reply":"2025-06-28T16:46:18.658605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --upgrade google-auth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:18.660853Z","iopub.execute_input":"2025-06-28T16:46:18.661112Z","iopub.status.idle":"2025-06-28T16:46:22.446800Z","shell.execute_reply.started":"2025-06-28T16:46:18.661092Z","shell.execute_reply":"2025-06-28T16:46:22.445940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:22.447896Z","iopub.execute_input":"2025-06-28T16:46:22.448225Z","iopub.status.idle":"2025-06-28T16:46:22.452398Z","shell.execute_reply.started":"2025-06-28T16:46:22.448196Z","shell.execute_reply":"2025-06-28T16:46:22.451700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"datasets_val = ['gwi-tfrecs-v-f32']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:22.454646Z","iopub.execute_input":"2025-06-28T16:46:22.454951Z","iopub.status.idle":"2025-06-28T16:46:22.476595Z","shell.execute_reply.started":"2025-06-28T16:46:22.454927Z","shell.execute_reply":"2025-06-28T16:46:22.475753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_path_val = [(KaggleDatasets().get_gcs_path(dataset)) \n                 for dataset in datasets_val]\npickle.dump(combined_path_val,open('combined_path_val.p', 'bw'))\ncombined_path_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:22.477680Z","iopub.execute_input":"2025-06-28T16:46:22.478418Z","iopub.status.idle":"2025-06-28T16:46:22.854318Z","shell.execute_reply.started":"2025-06-28T16:46:22.478388Z","shell.execute_reply":"2025-06-28T16:46:22.853259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\ntffiles_val = []\nfor idx, curr_tfrecords_path in enumerate(combined_path_val):\n    print(idx)\n    folders = !gsutil ls $curr_tfrecords_path\n    for folder in folders:\n        curr_tffiles = !gsutil ls $folder\n        tffiles_indices = [int(x.split('/')[-1][:-9]) for x in curr_tffiles]\n        curr_tffiles = [x for _, x in sorted(zip(tffiles_indices, curr_tffiles))]\n        tffiles_val.append(curr_tffiles)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:22.855253Z","iopub.execute_input":"2025-06-28T16:46:22.855582Z","iopub.status.idle":"2025-06-28T16:46:29.174953Z","shell.execute_reply.started":"2025-06-28T16:46:22.855529Z","shell.execute_reply":"2025-06-28T16:46:29.174071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pickle.dump(tffiles_val, open('tffiles_val.p', 'bw'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:29.175744Z","iopub.execute_input":"2025-06-28T16:46:29.175980Z","iopub.status.idle":"2025-06-28T16:46:29.180710Z","shell.execute_reply.started":"2025-06-28T16:46:29.175960Z","shell.execute_reply":"2025-06-28T16:46:29.179953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(tffiles_val[0])*50","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:46:29.181454Z","iopub.execute_input":"2025-06-28T16:46:29.181782Z","iopub.status.idle":"2025-06-28T16:46:29.198676Z","shell.execute_reply.started":"2025-06-28T16:46:29.181763Z","shell.execute_reply":"2025-06-28T16:46:29.197845Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"datasets = [f'gwi-tfrecs-test-ds-{x}' for x in range(1,3)]\ncombined_path = [(KaggleDatasets().get_gcs_path(dataset)) \n                 for dataset in datasets]\npickle.dump(combined_path,open('combined_path.p', 'bw'))\ncombined_path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:49:35.567585Z","iopub.execute_input":"2025-06-28T16:49:35.567883Z","iopub.status.idle":"2025-06-28T16:50:06.204074Z","shell.execute_reply.started":"2025-06-28T16:49:35.567861Z","shell.execute_reply":"2025-06-28T16:50:06.203357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\ntffiles = []\nfor idx, curr_tfrecords_path in enumerate(combined_path):\n    print(idx)\n    folders = !gsutil ls $curr_tfrecords_path\n    for folder in folders:\n        folder = folder\n        curr_tffiles = !gsutil ls $folder\n        tffiles_indices = [int(x.split('/')[-1][:-9]) for x in curr_tffiles]\n        curr_tffiles = [x for _, x in sorted(zip(tffiles_indices, curr_tffiles))]\n        tffiles.append(curr_tffiles)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:50:35.042439Z","iopub.execute_input":"2025-06-28T16:50:35.043275Z","iopub.status.idle":"2025-06-28T16:50:47.657327Z","shell.execute_reply.started":"2025-06-28T16:50:35.043249Z","shell.execute_reply":"2025-06-28T16:50:47.656375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(np.sum([len(x) for x in tffiles]))\nprint(50*np.sum([len(x) for x in tffiles]))\npickle.dump(tffiles, open('tffiles_test.p', 'bw'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T16:50:47.658829Z","iopub.execute_input":"2025-06-28T16:50:47.659143Z","iopub.status.idle":"2025-06-28T16:50:47.665086Z","shell.execute_reply.started":"2025-06-28T16:50:47.659117Z","shell.execute_reply":"2025-06-28T16:50:47.664231Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Generated data example","metadata":{}},{"cell_type":"code","source":"datasets = ['gwi-tfrecs-example']\ncombined_path = [(KaggleDatasets().get_gcs_path(dataset)) \n                 for dataset in datasets]\npickle.dump(combined_path,open('combined_path.p', 'bw'))\ncombined_path","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\ntffiles = []\nfor idx, curr_tfrecords_path in enumerate(combined_path):\n    print(idx)\n    folders = !gsutil ls $curr_tfrecords_path\n    for folder in folders:\n        folder = folder\n        curr_tffiles = !gsutil ls $folder\n        tffiles_indices = [int(x.split('/')[-1][:-9]) for x in curr_tffiles]\n        curr_tffiles = [x for _, x in sorted(zip(tffiles_indices, curr_tffiles))]\n        tffiles.append(curr_tffiles)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(np.sum([len(x) for x in tffiles]))\nprint(50*np.sum([len(x) for x in tffiles]))\npickle.dump(tffiles, open('tffiles_example.p', 'bw'))","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}