{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nfrom tqdm.auto import tqdm\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:20:53.171730Z","iopub.execute_input":"2023-06-19T15:20:53.172172Z","iopub.status.idle":"2023-06-19T15:20:53.178390Z","shell.execute_reply.started":"2023-06-19T15:20:53.172135Z","shell.execute_reply":"2023-06-19T15:20:53.177148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction'\nOUTPUT_DIR = './'","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:18:43.162044Z","iopub.execute_input":"2023-06-19T15:18:43.162702Z","iopub.status.idle":"2023-06-19T15:18:43.178266Z","shell.execute_reply.started":"2023-06-19T15:18:43.162667Z","shell.execute_reply":"2023-06-19T15:18:43.177223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(f'{OUTPUT_DIR}/unlabeled_np')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:18:43.179693Z","iopub.execute_input":"2023-06-19T15:18:43.180734Z","iopub.status.idle":"2023-06-19T15:18:43.191543Z","shell.execute_reply.started":"2023-06-19T15:18:43.180692Z","shell.execute_reply":"2023-06-19T15:18:43.190436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pq_files = glob.glob(f'{INPUT_DIR}/unlabeled/*.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:18:43.193337Z","iopub.execute_input":"2023-06-19T15:18:43.194328Z","iopub.status.idle":"2023-06-19T15:18:43.228355Z","shell.execute_reply.started":"2023-06-19T15:18:43.194285Z","shell.execute_reply":"2023-06-19T15:18:43.227557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = []","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:18:43.231728Z","iopub.execute_input":"2023-06-19T15:18:43.232471Z","iopub.status.idle":"2023-06-19T15:18:43.237037Z","shell.execute_reply.started":"2023-06-19T15:18:43.232425Z","shell.execute_reply":"2023-06-19T15:18:43.235940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CHUNK_SIZE = 100000","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:18:43.238717Z","iopub.execute_input":"2023-06-19T15:18:43.239436Z","iopub.status.idle":"2023-06-19T15:18:43.251267Z","shell.execute_reply.started":"2023-06-19T15:18:43.239382Z","shell.execute_reply":"2023-06-19T15:18:43.250461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fname in tqdm(pq_files):\n    data = pd.read_parquet(fname)\n    base_name = os.path.basename(fname).replace('.parquet','')\n    for ch in range(len(data)//CHUNK_SIZE):\n        arr = data.loc[ch*CHUNK_SIZE:(ch+1)*CHUNK_SIZE]\n        file_name = f'{OUTPUT_DIR}/unlabeled_np/{base_name}_chunk_{ch}.npz'\n        np.savez_compressed(file_name,arr=arr)\n        \n        metadata.append({'base_name':base_name,'file_name':file_name})\n    #break","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:21:19.026936Z","iopub.execute_input":"2023-06-19T15:21:19.027348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pd.DataFrame(metadata)\nmetadata.to_csv('{OUTPUT_DIR}/unlabeled_metadata.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:19:05.450117Z","iopub.status.idle":"2023-06-19T15:19:05.450641Z","shell.execute_reply.started":"2023-06-19T15:19:05.450322Z","shell.execute_reply":"2023-06-19T15:19:05.450341Z"},"trusted":true},"execution_count":null,"outputs":[]}]}