{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\nmetadata = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nspectro_path = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms'\n\nspids = np.unique(metadata['spectrogram_id'])\n\nsp_summary = pd.DataFrame()\nfor spid in spids:\n    record = {}\n    sp_data = pd.read_parquet(os.path.join(spectro_path, str(spid) + '.parquet'))\n    record['spectrogram_id'] = spid\n    record['min_val'] = np.min(sp_data.iloc[:, 1:].values)\n    record['max_val'] = np.max(sp_data.iloc[:, 1:].values)\n    record['nbr_rows'] = sp_data.shape[0]\n    record['nbr_nan_rows'] = len(np.unique(np.where(sp_data.isna())[0]))\n    print(record)\n    sp_summary = pd.concat([sp_summary, pd.DataFrame([record])], axis=0, ignore_index=True)\n\nsp_summary.to_csv('sp_summary.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-24T02:24:19.650778Z","iopub.execute_input":"2024-01-24T02:24:19.651193Z","iopub.status.idle":"2024-01-24T02:32:45.975625Z","shell.execute_reply.started":"2024-01-24T02:24:19.651158Z","shell.execute_reply":"2024-01-24T02:32:45.974295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sp_summary.loc[sp_summary['min_val'].isna() & sp_summary['max_val'].isna(), 'spectrogram_id'].values.flatten().tolist())","metadata":{"execution":{"iopub.status.busy":"2024-01-24T02:32:45.977656Z","iopub.execute_input":"2024-01-24T02:32:45.978038Z","iopub.status.idle":"2024-01-24T02:32:45.986922Z","shell.execute_reply.started":"2024-01-24T02:32:45.977986Z","shell.execute_reply":"2024-01-24T02:32:45.985104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(sp_summary.loc[sp_summary['min_val'].isna() & sp_summary['max_val'].isna(), 'spectrogram_id'].values.flatten().tolist()))","metadata":{"execution":{"iopub.status.busy":"2024-01-24T02:32:45.990225Z","iopub.execute_input":"2024-01-24T02:32:45.990679Z","iopub.status.idle":"2024-01-24T02:32:46.003944Z","shell.execute_reply.started":"2024-01-24T02:32:45.990642Z","shell.execute_reply":"2024-01-24T02:32:46.002824Z"},"trusted":true},"execution_count":null,"outputs":[]}]}