{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Install Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom typing import Optional, Tuple\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\nfrom scipy.signal import get_window","metadata":{"execution":{"iopub.status.busy":"2021-10-02T06:03:25.682468Z","iopub.execute_input":"2021-10-02T06:03:25.683152Z","iopub.status.idle":"2021-10-02T06:03:32.061779Z","shell.execute_reply.started":"2021-10-02T06:03:25.683052Z","shell.execute_reply":"2021-10-02T06:03:32.060777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For Private DataSet","metadata":{}},{"cell_type":"code","source":"# # Step 1: Get the credential from the Cloud SDK\n# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# user_credential = user_secrets.get_gcloud_credential()\n\n# # Step 2: Set the credentials\n# user_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2021-10-02T06:03:32.063227Z","iopub.execute_input":"2021-10-02T06:03:32.063515Z","iopub.status.idle":"2021-10-02T06:03:32.067878Z","shell.execute_reply.started":"2021-10-02T06:03:32.063488Z","shell.execute_reply":"2021-10-02T06:03:32.066597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"gcs_paths = []\n\nGCS_path = KaggleDatasets().get_gcs_path(f\"g2net-public-s-01\")\ngcs_paths.append(GCS_path)\n\n\nGCS_path = KaggleDatasets().get_gcs_path(f\"g2net-public-s-02\")\ngcs_paths.append(GCS_path)\n\nprint(gcs_paths)","metadata":{"execution":{"iopub.status.busy":"2021-10-02T06:03:32.070093Z","iopub.execute_input":"2021-10-02T06:03:32.070592Z","iopub.status.idle":"2021-10-02T06:05:36.055352Z","shell.execute_reply.started":"2021-10-02T06:03:32.070548Z","shell.execute_reply":"2021-10-02T06:05:36.054358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_files = []\nfor path in gcs_paths:\n    all_files.extend(np.sort(np.array(tf.io.gfile.glob(path + \"/test*.tfrecords\"))))\n\nprint(\"test_files: \", len(all_files))","metadata":{"execution":{"iopub.status.busy":"2021-10-02T06:05:36.057066Z","iopub.execute_input":"2021-10-02T06:05:36.057499Z","iopub.status.idle":"2021-10-02T06:05:36.623407Z","shell.execute_reply.started":"2021-10-02T06:05:36.057454Z","shell.execute_reply":"2021-10-02T06:05:36.622359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}