{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Util functions for parallel data loading\n#### from 76 to 21 seconds for loading whole training dataset","metadata":{}},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom typing import List, Iterable\nimport os, time\nfrom joblib import Parallel, delayed, parallel_backend","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:56:53.589385Z","iopub.execute_input":"2023-04-18T09:56:53.590541Z","iopub.status.idle":"2023-04-18T09:56:53.596772Z","shell.execute_reply.started":"2023-04-18T09:56:53.590483Z","shell.execute_reply":"2023-04-18T09:56:53.595080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_data = '/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction'\nG_MS2 = 9.8066","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:56:53.599877Z","iopub.execute_input":"2023-04-18T09:56:53.601038Z","iopub.status.idle":"2023-04-18T09:56:53.610059Z","shell.execute_reply.started":"2023-04-18T09:56:53.600986Z","shell.execute_reply":"2023-04-18T09:56:53.608997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_single_timeseries(ts_id:str) -> pd.DataFrame:\n    assert '.csv' not in ts_id\n    train_or_test, de_or_tdcs = find(ts_id)\n    path_to_timeseries = f\"{path_to_data}/{train_or_test}/{de_or_tdcs}fog/{ts_id}.csv\"\n    df = pd.read_csv(path_to_timeseries)\n    df['Id'] = ts_id\n    df = df.set_index(['Id', 'Time'])\n    if train_or_test == 'train' and de_or_tdcs == 'de':\n        df = df[(df['Valid']) & df['Task']]\n        df = df.drop(columns=['Valid', 'Task'])\n    if de_or_tdcs == 'tdcs':\n        df.loc[:, ['AccV', 'AccML', 'AccAP']] = df[['AccV', 'AccML', 'AccAP']] / G_MS2\n    return df\n\n\ndef load_timeseries(ts_ids:Iterable[str], n_jobs:int) -> pd.DataFrame:\n    with parallel_backend('loky'):\n        ts = pd.concat(\n            Parallel(n_jobs=n_jobs)(delayed(load_single_timeseries)(ts_id) for ts_id in ts_ids)\n        )\n    return ts\n\n\ndef get_files_in_folder(folder:str) -> List[str]:\n    return list(os.walk(folder))[0][2]\n\n\ndef get_ids(train_or_test:str, de_or_tdcs:str) -> List[str]:\n    this_folder = f\"{path_to_data}/{train_or_test}/{de_or_tdcs}fog\"\n    files = get_files_in_folder(this_folder)\n    ts_ids = [f.replace('.csv', '') for f in files]\n    return ts_ids\n\n\ndef find(ts_id: str) -> (str, str):\n    if ts_id in get_ids(train_or_test='train', de_or_tdcs='de'):\n        train_or_test, de_or_tdcs = 'train', 'de'\n    elif ts_id in get_ids(train_or_test='train', de_or_tdcs='tdcs'):\n        train_or_test, de_or_tdcs = 'train', 'tdcs'\n    elif ts_id in get_ids(train_or_test='test', de_or_tdcs='de'):\n        train_or_test, de_or_tdcs = 'test', 'de'\n    elif ts_id in get_ids(train_or_test='test', de_or_tdcs='tdcs'):\n        train_or_test, de_or_tdcs = 'test', 'tdcs'\n    else:\n        raise Exception(f'{ts_id} not found!')\n    return train_or_test, de_or_tdcs","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-18T09:56:53.611273Z","iopub.execute_input":"2023-04-18T09:56:53.612236Z","iopub.status.idle":"2023-04-18T09:56:53.642903Z","shell.execute_reply.started":"2023-04-18T09:56:53.612181Z","shell.execute_reply":"2023-04-18T09:56:53.641449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_ids = get_ids('train', 'de')\ntdcs_ids = get_ids('train', 'tdcs')","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:57:43.488472Z","iopub.execute_input":"2023-04-18T09:57:43.489262Z","iopub.status.idle":"2023-04-18T09:57:43.559505Z","shell.execute_reply.started":"2023-04-18T09:57:43.489213Z","shell.execute_reply":"2023-04-18T09:57:43.558564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_init = time.time()\nts = load_timeseries(de_ids + tdcs_ids, n_jobs=1)\nprint(f\"{time.time() - t_init:.1f}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:57:59.144281Z","iopub.execute_input":"2023-04-18T09:57:59.145090Z","iopub.status.idle":"2023-04-18T09:59:15.536543Z","shell.execute_reply.started":"2023-04-18T09:57:59.145042Z","shell.execute_reply":"2023-04-18T09:59:15.535191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_init = time.time()\nts = load_timeseries(de_ids + tdcs_ids, n_jobs=-1)\nprint(f\"{time.time() - t_init:.1f}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:59:15.538855Z","iopub.execute_input":"2023-04-18T09:59:15.539259Z","iopub.status.idle":"2023-04-18T09:59:36.763590Z","shell.execute_reply.started":"2023-04-18T09:59:15.539221Z","shell.execute_reply":"2023-04-18T09:59:36.762237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts","metadata":{"execution":{"iopub.status.busy":"2023-04-18T09:57:17.048486Z","iopub.execute_input":"2023-04-18T09:57:17.049163Z","iopub.status.idle":"2023-04-18T09:57:17.072100Z","shell.execute_reply.started":"2023-04-18T09:57:17.049115Z","shell.execute_reply":"2023-04-18T09:57:17.070756Z"},"trusted":true},"execution_count":null,"outputs":[]}]}