{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport shutil\n\nfrom dask.distributed import Client\nimport dask.dataframe as dd\nimport dask","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-21T22:53:49.664696Z","iopub.execute_input":"2024-01-21T22:53:49.665196Z","iopub.status.idle":"2024-01-21T22:53:50.016940Z","shell.execute_reply.started":"2024-01-21T22:53:49.665144Z","shell.execute_reply":"2024-01-21T22:53:50.015940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Dataset","metadata":{}},{"cell_type":"code","source":"client = Client(n_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-01-21T22:53:52.689795Z","iopub.execute_input":"2024-01-21T22:53:52.691600Z","iopub.status.idle":"2024-01-21T22:53:55.979854Z","shell.execute_reply.started":"2024-01-21T22:53:52.691536Z","shell.execute_reply":"2024-01-21T22:53:55.978894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dask.delayed\ndef process_group(name, g):   \n    eeg_id, spctrogram_id = name\n\n    eeg = pd.read_parquet(f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\")\n    spectrogram = pd.read_parquet(f\"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/{spctrogram_id}.parquet\")\n\n    for _, row in g.iterrows():\n        eeg_offset = int( row['eeg_label_offset_seconds'] )\n        eeg_sample = eeg.iloc[eeg_offset*200:(eeg_offset+50)*200]\n        eeg_sample.insert(0, 'eeg_id', row['eeg_id'])\n        eeg_sample.insert(1, 'eeg_sub_id', row['eeg_sub_id'])\n\n        spec_offset = int( row['spectrogram_label_offset_seconds'] )\n        spectrogram_sample = spectrogram.loc[(spectrogram['time']>=spec_offset)&(spectrogram['time']<spec_offset+600)]\n        spectrogram_sample.insert(0, 'spectrogram_id', row['spectrogram_id'])\n        spectrogram_sample.insert(1, 'spectrogram_sub_id', row['spectrogram_sub_id'])\n        \n        eeg_sample.to_parquet(f\"eeg/\", partition_cols=['eeg_id','eeg_sub_id'], compression='gzip')\n        spectrogram_sample.to_parquet(f\"spectrogram/\", partition_cols=['spectrogram_id','spectrogram_sub_id'], compression='gzip')\n        \n        return name\n    ","metadata":{"execution":{"iopub.status.busy":"2024-01-21T23:02:30.083593Z","iopub.execute_input":"2024-01-21T23:02:30.084030Z","iopub.status.idle":"2024-01-21T23:02:30.094345Z","shell.execute_reply.started":"2024-01-21T23:02:30.083995Z","shell.execute_reply":"2024-01-21T23:02:30.093204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ngroups = df.groupby(['eeg_id','spectrogram_id'])\n\nif os.path.exists(\"/kaggle/working/eeg\"): \n    shutil.rmtree(\"/kaggle/working/eeg\")\n    os.makedirs(\"/kaggle/working/eeg\") \nelse:\n    os.makedirs(\"/kaggle/working/eeg\") \n    \nif os.path.exists(\"/kaggle/working/spectrogram\"): \n    shutil.rmtree(\"/kaggle/working/spectrogram\")\n    os.makedirs(\"/kaggle/working/spectrogram\") \nelse:\n    os.makedirs(\"/kaggle/working/spectrogram\") \n\nresult = []\nfor name, g in groups:\n    result.append(process_group(name, g))\n\nresult = dask.compute(result,)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-21T23:03:13.027516Z","iopub.execute_input":"2024-01-21T23:03:13.028978Z","iopub.status.idle":"2024-01-21T23:31:51.395856Z","shell.execute_reply.started":"2024-01-21T23:03:13.028910Z","shell.execute_reply":"2024-01-21T23:31:51.375321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How to read","metadata":{}},{"cell_type":"code","source":"filter = [\n    [(\"eeg_id\", \"=\", '2578018731')]\n]\ntest = pd.read_parquet('/kaggle/working/eeg/', filters=filter)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-21T23:36:28.988858Z","iopub.execute_input":"2024-01-21T23:36:28.989252Z","iopub.status.idle":"2024-01-21T23:36:42.287951Z","shell.execute_reply.started":"2024-01-21T23:36:28.989224Z","shell.execute_reply":"2024-01-21T23:36:42.286417Z"},"trusted":true},"execution_count":null,"outputs":[]}]}