{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.model_selection import train_test_split\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.models import resnet50\nfrom tqdm.notebook import tqdm\n\nrandom.seed(42)\nnp.random.seed(42)\ntorch.manual_seed(42)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-22T06:56:40.420366Z","iopub.execute_input":"2024-02-22T06:56:40.420831Z","iopub.status.idle":"2024-02-22T06:56:49.874937Z","shell.execute_reply.started":"2024-02-22T06:56:40.420790Z","shell.execute_reply":"2024-02-22T06:56:49.873629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_egg_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\"\ntest_egg_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs\"\ntrain_spe_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms\"\ntest_spe_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms\"\nmeta_trainval = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\nmeta_test = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-22T06:56:49.877952Z","iopub.execute_input":"2024-02-22T06:56:49.879146Z","iopub.status.idle":"2024-02-22T06:56:50.201050Z","shell.execute_reply.started":"2024-02-22T06:56:49.879110Z","shell.execute_reply":"2024-02-22T06:56:50.199742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_trainval.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T06:57:25.429756Z","iopub.execute_input":"2024-02-22T06:57:25.430294Z","iopub.status.idle":"2024-02-22T06:57:25.462059Z","shell.execute_reply.started":"2024-02-22T06:57:25.430250Z","shell.execute_reply":"2024-02-22T06:57:25.460882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid_trainval = meta_trainval.patient_id.unique().tolist()\npid_train, pid_val = train_test_split(pid_trainval, test_size=0.15, random_state=42)\nmeta_val = meta_trainval.loc[meta_trainval.patient_id.isin(pid_val)].reset_index(drop=True)\nmeta_train = meta_trainval.loc[meta_trainval.patient_id.isin(pid_train)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-22T06:58:21.663611Z","iopub.execute_input":"2024-02-22T06:58:21.664426Z","iopub.status.idle":"2024-02-22T06:58:21.705397Z","shell.execute_reply.started":"2024-02-22T06:58:21.664387Z","shell.execute_reply":"2024-02-22T06:58:21.704308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T06:58:24.241471Z","iopub.execute_input":"2024-02-22T06:58:24.241916Z","iopub.status.idle":"2024-02-22T06:58:24.260321Z","shell.execute_reply.started":"2024-02-22T06:58:24.241880Z","shell.execute_reply":"2024-02-22T06:58:24.258738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_path = \"/kaggle/working/spec_train.npy\"\nout_dict = {}\nfor i, row in tqdm(meta_train.iterrows(), total=len(meta_train)):\n    spe_path = os.path.join(train_spe_dir, f\"{row.spectrogram_id}.parquet\")\n    spec = pd.read_parquet(spe_path)\n    out_dict[row.spectrogram_id] = spec\nnp.save(out_path, out_dict, allow_pickle=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_path = \"/kaggle/working/spec_val.npy\"\nout_dict = {}\nfor i, row in tqdm(meta_val.iterrows(), total=len(meta_val)):\n    spe_path = os.path.join(train_spe_dir, f\"{row.spectrogram_id}.parquet\")\n    spec = pd.read_parquet(spe_path)\n\n    out_dict[row.spectrogram_id] = spec\nnp.save(out_path, out_dict, allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T06:53:06.798850Z","iopub.execute_input":"2024-02-21T06:53:06.799251Z","iopub.status.idle":"2024-02-21T07:05:14.831166Z","shell.execute_reply.started":"2024-02-21T06:53:06.799222Z","shell.execute_reply":"2024-02-21T07:05:14.829816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# running_sum = 0\n# running_std = 0\n# cnt = 0\n# for eeg_id in tqdm(eeg_id_train):\n#     data = pd.read_parquet(os.path.join(train_egg_dir, f\"{eeg_id}.parquet\")).values\n#     running_sum += np.nanmean(data, axis=0)\n#     running_std += np.nanstd(data, axis=0)\n#     cnt += 1\n# means = running_sum / cnt\n# stds = running_std / cnt\nmeans = np.array([107.819214,  58.57809 ,  60.128506,  62.030807,  65.56657 ,\n        61.49913 ,  67.997925,  70.11514 ,  44.7709  ,  45.620415,\n        47.321655,  70.38301 ,  61.395145,  61.97019 ,  63.248966,\n        58.07724 ,  63.16125 ,  62.047237,  63.705734, 422.67264 ],\n      dtype=np.float32)\n\nstds = np.array([380.71298, 345.01163, 326.38092, 344.96094, 333.71848, 323.2996 ,\n       342.7745 , 356.3885 , 272.11044, 275.52643, 285.21518, 355.22522,\n       327.9994 , 322.98547, 325.5279 , 321.2669 , 335.1306 , 332.12457,\n       363.74713, 829.44324], dtype=np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T13:35:54.295521Z","iopub.execute_input":"2024-02-20T13:35:54.297881Z","iopub.status.idle":"2024-02-20T13:35:54.308589Z","shell.execute_reply.started":"2024-02-20T13:35:54.297636Z","shell.execute_reply":"2024-02-20T13:35:54.307031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_means = 0\nspec_stds = 0\nspec_train = out_dict\nfor k in tqdm(spec_train.keys(), total=len(spec_train)):\n    spec_means += np.log(1 + spec_train[k]).mean(axis=0)\n    spec_stds += np.log(1 + spec_train[k]).std(axis=0)\nspec_means = spec_means.values[1:] / len(spec_train)\nspec_stds = spec_stds.values[1:] / len(spec_train)\nnp.save(\"/kaggle/working/spec_stats.npy\", {\"means\": spec_means, \"stds\": spec_stds}, allow_pickle=True)","metadata":{},"execution_count":null,"outputs":[]}]}