{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7413074,"sourceType":"datasetVersion","datasetId":4312082}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nfrom scipy import signal\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset , DataLoader \nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import StepLR\n# from ncps.wirings import AutoNCP\n# from ncps.torch import LTC,CfC\nimport time,gc,warnings ,torch,os,pathlib\nimport joblib as jb\nfrom sklearn.model_selection import train_test_split\nwarnings.simplefilter(action='ignore', category=Warning)\nimport pywt\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n\ndset_path = pathlib.PurePath(\"/kaggle/input/hms-harmful-brain-activity-classification/\")\n\neeg_path = dset_path/\"train_eegs\"\nspec_path = dset_path/\"train_spectrograms\"\nna_list = jb.load('/kaggle/input/na-list/na_list.joblib')\ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n\n# removing nan spectrogram values\ndf = df[~df['eeg_id'].isin(na_list)].reset_index(drop=True)\n\nimport multiprocessing\n\ndef read_parquet(eeg_id, eeg_path):\n    return eeg_id, pd.read_parquet(eeg_path / f'{eeg_id}.parquet')\n\ndef load_data(df, eeg_path):\n    pool = multiprocessing.Pool()\n    results = []\n\n    for eeg_id in tqdm(df['eeg_id'].unique()):\n        results.append(pool.apply_async(read_parquet, args=(eeg_id, eeg_path)))\n\n    pool.close()\n    pool.join()\n\n    dic = {result.get()[0]: result.get()[1] for result in results}\n    return dic\n\ndic = load_data(df, eeg_path)\nimport joblib as jb\njb.dump(dic,'dic.joblib')\ndel dic\ngc.collect()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]}]}