{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-26T17:00:21.820605Z","iopub.execute_input":"2024-01-26T17:00:21.821603Z","iopub.status.idle":"2024-01-26T17:00:23.770793Z","shell.execute_reply.started":"2024-01-26T17:00:21.821510Z","shell.execute_reply":"2024-01-26T17:00:23.769496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls ../input/hms-harmful-brain-activity-classification\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-26T17:00:23.773459Z","iopub.execute_input":"2024-01-26T17:00:23.774245Z","iopub.status.idle":"2024-01-26T17:00:24.931866Z","shell.execute_reply.started":"2024-01-26T17:00:23.774194Z","shell.execute_reply":"2024-01-26T17:00:24.930695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = \"../input/hms-harmful-brain-activity-classification\"\ntrain = pd.read_csv(f\"{root}/train.csv\")\ntrain\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-26T17:00:24.933325Z","iopub.execute_input":"2024-01-26T17:00:24.933719Z","iopub.status.idle":"2024-01-26T17:00:25.495730Z","shell.execute_reply.started":"2024-01-26T17:00:24.933685Z","shell.execute_reply":"2024-01-26T17:00:25.494776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_eeg_ids = list(set(train['eeg_id']))\nlen(all_eeg_ids)\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-26T17:00:25.498631Z","iopub.execute_input":"2024-01-26T17:00:25.499056Z","iopub.status.idle":"2024-01-26T17:00:25.529290Z","shell.execute_reply.started":"2024-01-26T17:00:25.499019Z","shell.execute_reply":"2024-01-26T17:00:25.527978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quality = [-1]*len(train)\ntotal_nans = 0\ntotal = 0\nfor eeg_id in tqdm(all_eeg_ids, total=len(all_eeg_ids)):\n    all_egs = train[train['eeg_id'] == eeg_id]\n    data = np.array(pd.read_parquet(f\"{root}/train_eegs/{eeg_id}.parquet\"))\n    for i, row in all_egs.iterrows():\n        start_offset =int(row['eeg_label_offset_seconds'] * 200)\n        selected_egg = data[start_offset : start_offset + 50* 200, :].reshape(-1)\n        quality[i]  = np.count_nonzero(~np.isnan(selected_egg))/ len(selected_egg)\n        total_nans += np.count_nonzero(np.isnan(selected_egg))\n        total += len(selected_egg)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:00:25.531097Z","iopub.execute_input":"2024-01-26T17:00:25.531570Z","iopub.status.idle":"2024-01-26T17:09:58.246614Z","shell.execute_reply.started":"2024-01-26T17:00:25.531532Z","shell.execute_reply":"2024-01-26T17:09:58.243783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"EEG's total Nan% for 50secs\", total_nans/total*100","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:10:20.099070Z","iopub.execute_input":"2024-01-26T17:10:20.100097Z","iopub.status.idle":"2024-01-26T17:10:20.107948Z","shell.execute_reply.started":"2024-01-26T17:10:20.100058Z","shell.execute_reply":"2024-01-26T17:10:20.106664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quality = [-1]*len(train)\ntotal_nans = 0\ntotal = 0\nfor eeg_id in tqdm(all_eeg_ids, total=len(all_eeg_ids)):\n    all_egs = train[train['eeg_id'] == eeg_id]\n    data = np.array(pd.read_parquet(f\"{root}/train_eegs/{eeg_id}.parquet\"))\n    for i, row in all_egs.iterrows():\n        start_offset =int(row['eeg_label_offset_seconds'] * 200) + 4000\n        selected_egg = data[start_offset : start_offset + 2000, :].reshape(-1)\n        quality[i]  = np.count_nonzero(~np.isnan(selected_egg))/ len(selected_egg)\n        total_nans += np.count_nonzero(np.isnan(selected_egg))\n        total += len(selected_egg)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-26T17:10:27.565913Z","iopub.execute_input":"2024-01-26T17:10:27.566438Z","iopub.status.idle":"2024-01-26T17:14:59.190836Z","shell.execute_reply.started":"2024-01-26T17:10:27.566398Z","shell.execute_reply":"2024-01-26T17:14:59.189019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"EEG's total Nan% for 10secs\", total_nans/total*100","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.193854Z","iopub.execute_input":"2024-01-26T17:14:59.194560Z","iopub.status.idle":"2024-01-26T17:14:59.203999Z","shell.execute_reply.started":"2024-01-26T17:14:59.194514Z","shell.execute_reply":"2024-01-26T17:14:59.202947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['egg_quality'] = quality","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.205498Z","iopub.execute_input":"2024-01-26T17:14:59.206526Z","iopub.status.idle":"2024-01-26T17:14:59.251205Z","shell.execute_reply.started":"2024-01-26T17:14:59.206485Z","shell.execute_reply":"2024-01-26T17:14:59.249820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.hist(train['egg_quality'], bins=20, color='skyblue')\nplt.xlabel('Egg Quality')\nplt.ylabel('Frequency')\nplt.title('Distribution of Egg Quality')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.254763Z","iopub.execute_input":"2024-01-26T17:14:59.255252Z","iopub.status.idle":"2024-01-26T17:14:59.837498Z","shell.execute_reply.started":"2024-01-26T17:14:59.255208Z","shell.execute_reply":"2024-01-26T17:14:59.836300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['egg_quality'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.840453Z","iopub.execute_input":"2024-01-26T17:14:59.841097Z","iopub.status.idle":"2024-01-26T17:14:59.862723Z","shell.execute_reply.started":"2024-01-26T17:14:59.841044Z","shell.execute_reply":"2024-01-26T17:14:59.861815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_spectrogram_ids = list(set(train['spectrogram_id']))\nlen(all_spectrogram_ids)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.864129Z","iopub.execute_input":"2024-01-26T17:14:59.864962Z","iopub.status.idle":"2024-01-26T17:14:59.896063Z","shell.execute_reply.started":"2024-01-26T17:14:59.864922Z","shell.execute_reply":"2024-01-26T17:14:59.894549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sepc_quality = [-1]*len(train)\ntotal_nans = 0\ntotal = 0\nfor spectrogram_id in tqdm(all_spectrogram_ids, total=len(all_spectrogram_ids)):\n    all_specs = train[train['spectrogram_id'] == spectrogram_id]\n    data = np.array(pd.read_parquet(f\"{root}/train_spectrograms/{spectrogram_id}.parquet\"))\n    for i, row in all_specs.iterrows():\n        start_offset =int(row['spectrogram_label_offset_seconds']/2)\n        selected_spec = data[start_offset : start_offset + 300, :].reshape(-1)\n        sepc_quality[i]  = np.count_nonzero(~np.isnan(selected_spec))/ len(selected_spec)\n        total_nans += np.count_nonzero(np.isnan(selected_spec))\n        total += len(selected_spec)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:14:59.897736Z","iopub.execute_input":"2024-01-26T17:14:59.898138Z","iopub.status.idle":"2024-01-26T17:27:42.930381Z","shell.execute_reply.started":"2024-01-26T17:14:59.898097Z","shell.execute_reply":"2024-01-26T17:27:42.928674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"Spectrogram's total Nan% for 600secs\", total_nans/total*100\n","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:27:42.932651Z","iopub.execute_input":"2024-01-26T17:27:42.933069Z","iopub.status.idle":"2024-01-26T17:27:42.944550Z","shell.execute_reply.started":"2024-01-26T17:27:42.933034Z","shell.execute_reply":"2024-01-26T17:27:42.942693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sepc_quality = [-1]*len(train)\ntotal_nans = 0\ntotal = 0\nfor spectrogram_id in tqdm(all_spectrogram_ids, total=len(all_spectrogram_ids)):\n    all_specs = train[train['spectrogram_id'] == spectrogram_id]\n    data = np.array(pd.read_parquet(f\"{root}/train_spectrograms/{spectrogram_id}.parquet\"))\n    for i, row in all_specs.iterrows():\n        start_offset =int(row['spectrogram_label_offset_seconds']/2) + 147\n        selected_spec = data[start_offset : start_offset + 6, :].reshape(-1)\n        sepc_quality[i]  = np.count_nonzero(~np.isnan(selected_spec))/ len(selected_spec)\n        total_nans += np.count_nonzero(np.isnan(selected_spec))\n        total += len(selected_spec)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:27:42.946392Z","iopub.execute_input":"2024-01-26T17:27:42.948117Z","iopub.status.idle":"2024-01-26T17:36:35.482967Z","shell.execute_reply.started":"2024-01-26T17:27:42.948053Z","shell.execute_reply":"2024-01-26T17:36:35.481656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"Spectrogram's total Nan% for 12secs\", total_nans/total*100","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:35.489174Z","iopub.execute_input":"2024-01-26T17:36:35.489694Z","iopub.status.idle":"2024-01-26T17:36:35.500551Z","shell.execute_reply.started":"2024-01-26T17:36:35.489650Z","shell.execute_reply":"2024-01-26T17:36:35.498755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['spec_quality'] = sepc_quality\ntrain['spec_quality'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:35.502072Z","iopub.execute_input":"2024-01-26T17:36:35.502500Z","iopub.status.idle":"2024-01-26T17:36:35.561031Z","shell.execute_reply.started":"2024-01-26T17:36:35.502465Z","shell.execute_reply":"2024-01-26T17:36:35.559698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.hist(train['spec_quality'], bins=20, color='skyblue')\nplt.xlabel('Spec Quality')\nplt.ylabel('Frequency')\nplt.title('Distribution of Spec Quality')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:35.562640Z","iopub.execute_input":"2024-01-26T17:36:35.563046Z","iopub.status.idle":"2024-01-26T17:36:35.944389Z","shell.execute_reply.started":"2024-01-26T17:36:35.563013Z","shell.execute_reply":"2024-01-26T17:36:35.942838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[ train['spec_quality'] != 1]","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:35.947054Z","iopub.execute_input":"2024-01-26T17:36:35.947685Z","iopub.status.idle":"2024-01-26T17:36:35.991718Z","shell.execute_reply.started":"2024-01-26T17:36:35.947632Z","shell.execute_reply":"2024-01-26T17:36:35.990120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[ train['egg_quality'] != 1]","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:35.993462Z","iopub.execute_input":"2024-01-26T17:36:35.994674Z","iopub.status.idle":"2024-01-26T17:36:36.030968Z","shell.execute_reply.started":"2024-01-26T17:36:35.994577Z","shell.execute_reply":"2024-01-26T17:36:36.029421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['max_quality'] = train.apply(lambda x: max(x['egg_quality'], x['spec_quality']), axis=1)\ntrain['min_quality'] = train.apply(lambda x: min(x['egg_quality'], x['spec_quality']), axis=1)\ntrain['avg_quality'] = train.apply(lambda x: (x['egg_quality'] + x['spec_quality'])/2, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:36.033835Z","iopub.execute_input":"2024-01-26T17:36:36.034324Z","iopub.status.idle":"2024-01-26T17:36:41.599257Z","shell.execute_reply.started":"2024-01-26T17:36:36.034280Z","shell.execute_reply":"2024-01-26T17:36:41.597814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['max_quality'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:41.601197Z","iopub.execute_input":"2024-01-26T17:36:41.601697Z","iopub.status.idle":"2024-01-26T17:36:41.619474Z","shell.execute_reply.started":"2024-01-26T17:36:41.601655Z","shell.execute_reply":"2024-01-26T17:36:41.618111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n\naxes[0].boxplot(train['max_quality'])\naxes[0].set_xlabel(f'Max Quality')\naxes[0].set_ylabel('Value')\naxes[0].set_title(f'Max Quality - {len(train[ train[\"max_quality\"] != 1])} - {len(train[ train[\"max_quality\"] != 1])/len(train)*100:.2f}%')\n\naxes[1].boxplot(train['min_quality'])\naxes[1].set_xlabel(f'Min Quality')\naxes[1].set_ylabel('Value')\naxes[1].set_title(f'Min Quality - {len(train[ train[\"min_quality\"] != 1])} - {len(train[ train[\"min_quality\"] != 1])/len(train)*100:.2f}%')\n\naxes[2].boxplot(train['avg_quality'])\naxes[2].set_xlabel(f'Avg Quality')\naxes[2].set_ylabel('Value')\naxes[2].set_title(f'Avg Quality - {len(train[ train[\"avg_quality\"] != 1])} - {len(train[ train[\"avg_quality\"] != 1])/len(train)*100:.2f}%')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:41.620719Z","iopub.execute_input":"2024-01-26T17:36:41.621162Z","iopub.status.idle":"2024-01-26T17:36:42.353381Z","shell.execute_reply.started":"2024-01-26T17:36:41.621120Z","shell.execute_reply":"2024-01-26T17:36:42.352011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(train['egg_quality'], train['spec_quality'])\nplt.xlabel('EEG Quality')\nplt.ylabel('Spectrogram Quality')\nplt.title('EEG Quality vs Spectrogram Quality')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:42.354945Z","iopub.execute_input":"2024-01-26T17:36:42.355319Z","iopub.status.idle":"2024-01-26T17:36:42.901332Z","shell.execute_reply.started":"2024-01-26T17:36:42.355286Z","shell.execute_reply":"2024-01-26T17:36:42.900036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['min_quality'] != 1]","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:42.903439Z","iopub.execute_input":"2024-01-26T17:36:42.903990Z","iopub.status.idle":"2024-01-26T17:36:42.947281Z","shell.execute_reply.started":"2024-01-26T17:36:42.903945Z","shell.execute_reply":"2024-01-26T17:36:42.945629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train = train[train['min_quality'] == 1].reset_index(drop=True)\nfinal_train.shape\n","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:42.949458Z","iopub.execute_input":"2024-01-26T17:36:42.950082Z","iopub.status.idle":"2024-01-26T17:36:42.980549Z","shell.execute_reply.started":"2024-01-26T17:36:42.950027Z","shell.execute_reply":"2024-01-26T17:36:42.979095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train.to_csv(\"/kaggle/working/train_clean.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T17:36:44.374082Z","iopub.execute_input":"2024-01-26T17:36:44.374972Z","iopub.status.idle":"2024-01-26T17:36:45.908306Z","shell.execute_reply.started":"2024-01-26T17:36:44.374927Z","shell.execute_reply":"2024-01-26T17:36:45.906577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}