{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport multiprocessing as mp\nfrom joblib import Parallel, delayed","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\nSPEC_PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nEEG_PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i_nan = 0\ni_std = 0\ndef find_nan(row):\n    global i_nan\n    eeg_id = row[\"eeg_id\"]\n    eeg_label_offset_seconds = row[\"eeg_label_offset_seconds\"]\n    parquet_path = f'{EEG_PATH}{eeg_id}.parquet'\n    raw_eeg = pd.read_parquet(parquet_path)\n    time_temp = int(eeg_label_offset_seconds)\n    time_start =  round(time_temp + (50 - 50) / 2 * 200) \n    time_stop =  round(time_temp + (50 + 50) / 2 * 200)\n\n    raw_eeg = raw_eeg.loc[time_start: (time_stop - 1), :].reset_index(drop=True)\n    \n    is_nan = raw_eeg.isnull().sum().sum()\n    if is_nan/(10000*20) > 0.1:\n        print(i_nan, \". NaN: \", eeg_id, eeg_label_offset_seconds, is_nan/(10000*20), raw_eeg.shape, is_nan)\n        i_nan += 1\n    else:\n        raw_eeg = raw_eeg.values\n        std = np.std(raw_eeg)\n        if std == 0:\n            print(i_std, \". STD = 0: \", eeg_id)\n            i_std += 1\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in test.iterrows():\n    find_nan(row)","metadata":{},"execution_count":null,"outputs":[]}]}