{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Reading time-series data often takes a lot of time, and a method to accelerate reading has been proven to be effective through practice","metadata":{"execution":{"iopub.status.busy":"2024-10-14T08:58:07.516773Z","iopub.execute_input":"2024-10-14T08:58:07.518375Z","iopub.status.idle":"2024-10-14T08:58:07.535155Z","shell.execute_reply.started":"2024-10-14T08:58:07.518288Z","shell.execute_reply":"2024-10-14T08:58:07.532447Z"}}},{"cell_type":"markdown","source":"## Read time-series data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom tqdm import tqdm\n\ndef extract_features_from_file(filepath):\n    data = pd.read_parquet(filepath)\n    \n    # Extract some basic statistical features\n    features = {\n        # examples\n        'X_mean': data['X'].mean(),\n        'X_max': data['X'].max(),\n        'X_min': data['X'].min(),\n        'X_std': data['X'].std(),\n        # ......\n    }\n    return features\n\ndef process_file(directory, filename):\n    # Complete file path\n    filepath = os.path.join(directory, filename, 'part-0.parquet')\n    features = extract_features_from_file(filepath)\n    features['id'] = filename.split('=')[1]\n    return features\n\ndef process_directory(directory, max_workers = 4):\n    all_features = []\n    ids = os.listdir(directory)\n    \n    # Use ThreadPoolExecutor for multithreading with the specified number of threads\n    with ThreadPoolExecutor(max_workers=max_workers) as executor:\n        # Submit all tasks, passing the directory\n        futures = {executor.submit(process_file, directory, idname): idname for idname in ids}\n        \n        # Use tqdm to track progress\n        for future in tqdm(as_completed(futures), total=len(futures)):\n            idname = futures[future]\n            try:\n                features = future.result()\n                all_features.append(features)\n            except Exception as e:\n                print(f\"Error processing file {idname}: {e}\")\n    \n    features_df = pd.DataFrame(all_features)\n    return features_df\n\ntrain_ts = process_directory(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\", max_workers=10)\ntest_ts = process_directory(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\", max_workers=10)\ntrain_ts","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-14T09:01:21.641986Z","iopub.execute_input":"2024-10-14T09:01:21.642537Z","iopub.status.idle":"2024-10-14T09:01:49.616865Z","shell.execute_reply.started":"2024-10-14T09:01:21.642487Z","shell.execute_reply":"2024-10-14T09:01:49.615422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read other structured data","metadata":{"execution":{"iopub.status.busy":"2024-10-14T09:00:54.915401Z","iopub.execute_input":"2024-10-14T09:00:54.916049Z","iopub.status.idle":"2024-10-14T09:00:54.922960Z","shell.execute_reply.started":"2024-10-14T09:00:54.915998Z","shell.execute_reply":"2024-10-14T09:00:54.921301Z"}}},{"cell_type":"code","source":"%%time\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-10-14T09:01:59.399626Z","iopub.execute_input":"2024-10-14T09:01:59.400138Z","iopub.status.idle":"2024-10-14T09:01:59.510879Z","shell.execute_reply.started":"2024-10-14T09:01:59.400090Z","shell.execute_reply":"2024-10-14T09:01:59.509203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge","metadata":{}},{"cell_type":"code","source":"time_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id',axis=1)\ntest = test.drop('id',axis=1)\ntrain ","metadata":{"execution":{"iopub.status.busy":"2024-10-14T09:02:01.639714Z","iopub.execute_input":"2024-10-14T09:02:01.640318Z","iopub.status.idle":"2024-10-14T09:02:01.710103Z","shell.execute_reply.started":"2024-10-14T09:02:01.640248Z","shell.execute_reply":"2024-10-14T09:02:01.708616Z"},"trusted":true},"execution_count":null,"outputs":[]}]}