\n In this cell, we configure important settings, including random seed and batch size, and prepare the dataset for model training. We load the dataset, select one sample per EEG ID, and ensure the data is ready for further processing. ππ\n
","metadata":{}},{"cell_type":"code","source":"class CFG:\n verbose = 1 # Verbosity\n pandas_warning = None #Pandas Chained Assignment Warning Mode, default = 'warn'\n seed = 42 # Random seed\n batch_size = 32\n epochs = 20\n\npd.options.mode.chained_assignment = CFG.pandas_warning\nkeras.utils.set_random_seed(CFG.seed)\n\nBASE_PATH = \"/kaggle/input/hms-harmful-brain-activity-classification\"\nEEG_DIR = \"/tmp/dataset/hms-hbac/numpy_eegs\"\nos.makedirs(EEG_DIR, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:34.570913Z","iopub.execute_input":"2025-01-31T12:48:34.571255Z","iopub.status.idle":"2025-01-31T12:48:34.576027Z","shell.execute_reply.started":"2025-01-31T12:48:34.571223Z","shell.execute_reply":"2025-01-31T12:48:34.575275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(f'{BASE_PATH}/train.csv')\ndisplay(df.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:35.613341Z","iopub.execute_input":"2025-01-31T12:48:35.613699Z","iopub.status.idle":"2025-01-31T12:48:35.767089Z","shell.execute_reply.started":"2025-01-31T12:48:35.613669Z","shell.execute_reply":"2025-01-31T12:48:35.766423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# As each file points to only one EEG ID, we will select only 1 sample from metadata.\ndf = df.groupby(\"eeg_id\").head(1).reset_index(drop=True)\ndf.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:36.722705Z","iopub.execute_input":"2025-01-31T12:48:36.723010Z","iopub.status.idle":"2025-01-31T12:48:36.748762Z","shell.execute_reply.started":"2025-01-31T12:48:36.722985Z","shell.execute_reply":"2025-01-31T12:48:36.748052Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n This cell focuses on assigning binary labels for seizure detection, splitting the dataset into seizure and non-seizure classes, and balancing the data through undersampling. We ensure a 1:1 class distribution for training the model. ππ€\n
","metadata":{}},{"cell_type":"code","source":"# As we are working on only seizure / non-seizure classification, we will not use default class labels of the dataset.\ndf['is_seizure'] = df['expert_consensus'] == 'Seizure'\n\n# Train + Valid + Test (We will use train_test_split to split the data)\ndf['eeg_path'] = f'{BASE_PATH}/train_eegs/'+df['eeg_id'].astype(str)+'.parquet'\n# df['eeg2_path'] = f'{EEG_DIR}/'+df['eeg_id'].astype(str)+'.npy'\ndf['spec_path'] = f'{BASE_PATH}/train_spectrograms/'+df['spectrogram_id'].astype(str)+'.parquet'\n\ndisplay(df.head(2))\nprint(f\"Total Data: {len(df)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:41.328759Z","iopub.execute_input":"2025-01-31T12:48:41.329056Z","iopub.status.idle":"2025-01-31T12:48:41.362535Z","shell.execute_reply.started":"2025-01-31T12:48:41.329030Z","shell.execute_reply":"2025-01-31T12:48:41.361715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter for 'Seizure' class and a subset of 'non-Seizure' classes\nseizure_df = df[df['expert_consensus'] == 'Seizure']\nnon_seizure_df = df[df['expert_consensus'] != 'Seizure']\n\n# Undersample non-Seizure data (e.g., 1:1 ratio)\nnon_seizure_sampled = non_seizure_df.sample(n=len(seizure_df), random_state=CFG.seed)\n\n# Combine and shuffle the new dataset\ndf = pd.concat([seizure_df, non_seizure_sampled]).sample(frac=1, random_state=CFG.seed)\ndf.sort_values(by=[\"eeg_id\", \"eeg_label_offset_seconds\"], ignore_index=True, inplace=True)\n\nclass_counts = df['expert_consensus'].value_counts()\ndisplay(class_counts)\nclass_counts.plot(kind='bar', title='Class Distribution')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:41.542842Z","iopub.execute_input":"2025-01-31T12:48:41.543148Z","iopub.status.idle":"2025-01-31T12:48:41.739778Z","shell.execute_reply.started":"2025-01-31T12:48:41.543122Z","shell.execute_reply":"2025-01-31T12:48:41.738853Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this cell, we randomly sample EEG and spectrogram files from the dataset to inspect their shapes, feature ranges, and overall characteristics. This helps us understand the structure and distribution of the data before model training. ππ‘\n
","metadata":{}},{"cell_type":"code","source":"random_samples = np.random.randint(0, len(df) - 1, size=(10))\nfor i in random_samples:\n reading_spec = pd.read_parquet(f\"{BASE_PATH}/train_spectrograms/{df['spectrogram_id'].iloc[i]}.parquet\")\n reading_eeg = pd.read_parquet(f\"{BASE_PATH}/train_eegs/{df['eeg_id'].iloc[i]}.parquet\")\n print(f\"Rows in EEG File: {reading_eeg.shape[0]}\\t Rows in Spectrogram File: {reading_spec.shape[0]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:48:52.311367Z","iopub.execute_input":"2025-01-31T12:48:52.311707Z","iopub.status.idle":"2025-01-31T12:48:52.629781Z","shell.execute_reply.started":"2025-01-31T12:48:52.311681Z","shell.execute_reply":"2025-01-31T12:48:52.628999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Information regarding a single spectrogram\ntrain_df_samples = len(df) # There are total 106800 training samples\nprint(f\"Total Samples: {train_df_samples}\")\n\nrandom_index = np.random.randint(0, train_df_samples - 1)\n\nrandom_spec_id = df['spectrogram_id'].iloc[random_index]\nreading_spec = pd.read_parquet(f\"{BASE_PATH}/train_spectrograms/{random_spec_id}.parquet\")\nrandom_eeg_id = df['eeg_id'].iloc[random_index]\nreading_eeg = pd.read_parquet(f\"{BASE_PATH}/train_eegs/{random_eeg_id}.parquet\")\n\nprint(\"\\nSPECTROGRAM INFORMATION:\")\nprint(f\"Shape of spectrogram: {reading_spec.shape}\")\nprint(f\"Spectrogram data min: {reading_spec.iloc[:, 1:].values.min()}\")\nprint(f\"Spectrogram data max: {reading_spec.iloc[:, 1:].values.max()}\")\nprint(f\"Spectrogram data mean: {reading_spec.iloc[:, 1:].values.mean()}\")\nprint(\"Features of Spectrogram:\\n\")\ndisplay(reading_spec.columns)\n\nprint(\"\\nEEG INFORMATION:\")\nprint(f\"Shape of EEG: {reading_eeg.shape}\")\nprint(f\"EEG data min: {reading_eeg.iloc[:, :-1].values.min()}\")\nprint(f\"EEG data max: {reading_eeg.iloc[:, :-1].values.max()}\")\nprint(f\"EEG data mean: {reading_eeg.iloc[:, :-1].values.mean()}\")\nprint(\"Features of Spectrogram:\\n\")\ndisplay(reading_eeg.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:51:59.180961Z","iopub.execute_input":"2025-01-31T12:51:59.181434Z","iopub.status.idle":"2025-01-31T12:51:59.233571Z","shell.execute_reply.started":"2025-01-31T12:51:59.181393Z","shell.execute_reply":"2025-01-31T12:51:59.232750Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this cell, we are showcasing example figures that represent how the dataset for our Seizure Detection Model might look. Before diving into the actual dataset, we're giving you a preview of the types of figures and data visualizations you'll encounter in the real dataset. π§ π‘\n
","metadata":{}},{"cell_type":"code","source":"import fitz # PyMuPDF\nfrom PIL import Image\nfrom io import BytesIO\n\n# Folder containing the PDF files\nfolder_path = f\"{BASE_PATH}/example_figures\"\n\n# List all files in the folder\npdf_files = [f for f in os.listdir(folder_path) if f.endswith('.pdf')]\n\n# Loop through all PDF files and display them\nfor pdf_file in pdf_files:\n # Construct the full path of the PDF file\n pdf_path = os.path.join(folder_path, pdf_file)\n \n # Open the PDF file\n doc = fitz.open(pdf_path)\n \n # Loop through the pages and display each one\n for page_num in range(doc.page_count):\n page = doc.load_page(page_num) # Load the page\n pix = page.get_pixmap() # Convert page to a pixmap (image)\n\n # Convert to a PIL Image\n img = Image.open(BytesIO(pix.tobytes(\"png\")))\n\n # Display the image in the notebook\n display(img)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T17:13:21.261102Z","iopub.execute_input":"2025-01-31T17:13:21.261484Z","iopub.status.idle":"2025-01-31T17:13:25.194712Z","shell.execute_reply.started":"2025-01-31T17:13:21.261455Z","shell.execute_reply":"2025-01-31T17:13:25.193783Z"},"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this cell, we verify the consistency of spectrogram time data, check the quality of signals over time, and visualize the spectrogram to ensure the data is clean and ready for model training. ππΆ\n
","metadata":{}},{"cell_type":"code","source":"reading_spec = pd.read_parquet(f\"{BASE_PATH}/train_spectrograms/{df['spectrogram_id'].iloc[np.random.randint(0, train_df_samples - 1)]}.parquet\")\n\n# Step 1: Verify time consistency\ndef check_time_consistency(spectrogram_data, metadata, spectrogram_id):\n \"\"\"\n Verify if the 'time' column in the spectrogram data matches the metadata offsets.\n \"\"\"\n sub_metadata = metadata[metadata[\"spectrogram_id\"] == spectrogram_id]\n metadata_times = sub_metadata[\"spectrogram_label_offset_seconds\"].values\n data_times = spectrogram_data[\"time\"].values\n\n # Check for missing or mismatched times\n missing_times = set(metadata_times) - set(data_times)\n extra_times = set(data_times) - set(metadata_times)\n\n print(\"Time Consistency Check:\")\n if missing_times:\n print(f\" Missing time points in spectrogram data: {missing_times}\")\n if extra_times:\n print(f\" Extra time points in spectrogram data: {extra_times}\")\n if not missing_times and not extra_times:\n print(\" All time points match!\")\n return missing_times, extra_times\n\n# missing_times, extra_times = check_time_consistency(reading_spec, train_df, random_id)\n\n# Step 2: Check for signal quality across time\ndef check_signal_quality(df, min_variance=1e-5):\n \"\"\"\n Analyze the signal quality over time for anomalies or low variance.\n \"\"\"\n print(\"\\nSignal Quality Check:\")\n time_series = df[\"time\"].diff().dropna()\n\n # Check for irregular time gaps\n if time_series.std() > 0.1: # Adjust threshold as needed\n print(f\" Irregular time gaps detected! Std Dev: {time_series.std()}\")\n else:\n print(\" Time gaps are consistent.\")\n\n # Check variance in frequency bands over time\n variances = df.iloc[:, 1:].var(axis=0) # Exclude 'time' column\n low_variance_cols = variances[variances < min_variance]\n if not low_variance_cols.empty:\n print(f\" Low variance detected in columns: {low_variance_cols.index.tolist()}\")\n else:\n print(\" All frequency bands have sufficient variance.\")\n\ncheck_signal_quality(reading_spec)\n \nimport matplotlib.colors as colors\n\n# Step 3: Visualize time-based trends\ndef plot_spectrogram_over_time(df):\n \"\"\"\n Plot the spectrogram over time to visually inspect for noise or artifacts.\n \"\"\"\n\n # Extract time and spectrogram data\n time = df[\"time\"].values\n spectrogram_data = df.iloc[:, 1:].values # Excluding the time column\n\n # Check the min, max, and mean values to understand the scale\n spec_min = spectrogram_data.min()\n spec_max = spectrogram_data.max()\n print(f\"Spectrogram data min: {spec_min}\")\n print(f\"Spectrogram data max: {spec_max}\")\n print(f\"Spectrogram data mean: {spectrogram_data.mean()}\")\n\n colormaps = ['hsv', \"viridis\", 'gist_ncar', 'gist_ncar_r', 'gist_rainbow', 'gist_rainbow_r']\n plt.figure(figsize=(12, len(colormaps) * 6))\n for i, cm in enumerate(colormaps):\n plt.subplot(len(colormaps), 1, i + 1)\n plt.imshow(spectrogram_data.T, aspect=\"auto\", cmap=cm,\n norm=colors.LogNorm(),\n extent=[time.min(), time.max(), 0, spectrogram_data.shape[0] - 1])\n plt.colorbar(label=\"Amplitude\")\n plt.xlabel(\"Time (seconds)\")\n plt.ylabel(\"Frequency Bands\")\n plt.title(f\"Spectrogram Over Time (Color Map: {cm})\")\n plt.show()\n\nplot_spectrogram_over_time(reading_spec)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:10:08.582694Z","iopub.execute_input":"2025-01-31T12:10:08.582993Z","iopub.status.idle":"2025-01-31T12:10:11.616367Z","shell.execute_reply.started":"2025-01-31T12:10:08.582971Z","shell.execute_reply":"2025-01-31T12:10:11.615239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n This cell performs a detailed analysis of EEG signal quality, including variance checks, frequency band power calculation, correlation between electrodes, and filtering of noisy signals. We aim to enhance the quality of the data for effective seizure detection. π§ β‘\n
","metadata":{}},{"cell_type":"code","source":"# Bandpass filter function (to remove noise outside 0.5-50 Hz range)\ndef bandpass_filter(data, lowcut=0.5, highcut=50, fs=200, order=5):\n nyquist = 0.5 * fs\n low = lowcut / nyquist\n high = highcut / nyquist\n b, a = butter(order, [low, high], btype=\"band\")\n return filtfilt(b, a, data, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:50:11.856785Z","iopub.execute_input":"2025-01-31T12:50:11.857103Z","iopub.status.idle":"2025-01-31T12:50:11.861648Z","shell.execute_reply.started":"2025-01-31T12:50:11.857079Z","shell.execute_reply":"2025-01-31T12:50:11.860722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load a sample EEG file (update with actual file paths)\nreading_eeg = pd.read_parquet(f\"{BASE_PATH}/train_eegs/{df['eeg_id'].iloc[np.random.randint(0, train_df_samples - 1)]}.parquet\")\neeg_data = reading_eeg.drop(columns=[\"EKG\"])\n\n# Sampling frequency (Hz)\nFS = 200\n\n# Analysis Results Dictionary\nresults = {}\n\n### 1. Check Signal Variance\nvariances = eeg_data.var()\nmean_variance = variances.mean()\nstd_variance = variances.std()\nhigh_variance_threshold = mean_variance + 2 * std_variance\nlow_variance_threshold = mean_variance - 2 * std_variance\n\n# Identify electrodes with high/low variance\nhigh_variance_electrodes = variances[variances > high_variance_threshold]\nlow_variance_electrodes = variances[variances < low_variance_threshold]\n\nresults[\"high_variance\"] = high_variance_electrodes\nresults[\"low_variance\"] = low_variance_electrodes\n\n### 2. Frequency Analysis (FFT)\nfor column in eeg_data.columns:\n signal = eeg_data[column]\n freq = np.fft.rfftfreq(len(signal), d=1 / FS) # Frequency axis\n fft_magnitude = np.abs(np.fft.rfft(signal)) # Magnitudes\n\n # Calculate power in specific frequency bands\n power = { \n \"delta\": np.sum(fft_magnitude[(freq >= 0.5) & (freq < 4)]),\n \"theta\": np.sum(fft_magnitude[(freq >= 4) & (freq < 8)]),\n \"alpha\": np.sum(fft_magnitude[(freq >= 8) & (freq < 13)]),\n \"beta\": np.sum(fft_magnitude[(freq >= 13) & (freq < 30)]),\n \"gamma\": np.sum(fft_magnitude[(freq >= 30) & (freq < 50)]),\n \"high_freq_noise\": np.sum(fft_magnitude[freq >= 50]),\n }\n\n # Print power in each band for the first few channels\n if column in eeg_data.columns[:3]:\n print(f\"Power for {column}: {power}\")\n \n # Record high-frequency noise if it dominates\n if power[\"high_freq_noise\"] > 0.2 * sum(power.values()):\n results[f\"{column}_high_freq_noise\"] = True\n else:\n results[f\"{column}_high_freq_noise\"] = False\n\n### 3. Signal Consistency Check\n# Compute correlation between electrodes\ncorr_matrix = eeg_data.corr()\ncorrelated_channels = (corr_matrix > 0.9).sum(axis=1) - 1 # Exclude self-correlation\n\n# Mark channels with high correlation to others (possible redundancy)\nresults[\"highly_correlated_electrodes\"] = correlated_channels[correlated_channels > 5]\n\n# Visualize correlation matrix\nplt.figure(figsize=(30, 30))\nsns.set(font_scale=1.5)\nsns.heatmap(corr_matrix, annot=True, cmap=\"coolwarm\")\nplt.title(\"Correlation Matrix of EEG Signals\")\nplt.show()\n\n### 4. Filtered Signal Inspection\nfor column in eeg_data.columns[:3]: # Check a few electrodes\n filtered_signal = bandpass_filter(eeg_data[column], 0.5, 50, FS)\n plt.figure(figsize=(10, 3))\n plt.plot(filtered_signal, label=f\"Filtered {column}\")\n plt.title(f\"Filtered EEG Signal: {column}\")\n plt.xlabel(\"Time (samples)\")\n plt.ylabel(\"Amplitude\")\n plt.legend()\n plt.show()\n\n### Summary Report\nprint(\"\\n=== EEG Signal Noise Analysis Summary ===\")\nprint(f\"Mean Variance: {mean_variance}\")\nprint(f\"High Variance Threshold: {high_variance_threshold}\")\nprint(f\"Low Variance Threshold: {low_variance_threshold}\")\nprint(f\"Electrodes with High Variance:\\n{high_variance_electrodes}\")\nprint(f\"Electrodes with Low Variance:\\n{low_variance_electrodes}\")\nprint(f\"Electrodes with High Variance:\\n{results['high_variance']}\")\nprint(f\"Electrodes with Low Variance:\\n{results['low_variance']}\")\nprint(f\"Electrodes with High-Frequency Noise:\")\nfor key, val in results.items():\n if \"high_freq_noise\" in key and val:\n print(f\" {key.replace('_high_freq_noise', '')}\")\n\nprint(\"\\nHighly Correlated Electrodes (likely redundant):\")\nprint(results[\"highly_correlated_electrodes\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:52:17.001479Z","iopub.execute_input":"2025-01-31T12:52:17.001852Z","iopub.status.idle":"2025-01-31T12:52:19.113392Z","shell.execute_reply.started":"2025-01-31T12:52:17.001821Z","shell.execute_reply":"2025-01-31T12:52:19.112685Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n This cell visualizes the temporal and frequency domain properties of EEG signals, comparing seizure and non-seizure samples. The signals are preprocessed by clipping and filling NaN values to improve signal quality before plotting. β‘π§ \n
","metadata":{}},{"cell_type":"code","source":"def plot_temporal_signals(eeg_df, num_samples=3, electrodes=[\"Fp1\", \"F3\", \"C3\"]):\n \"\"\"\n Plots temporal EEG signals for a random selection of seizure and non-seizure samples.\n \n Parameters:\n eeg_df (DataFrame): Dataframe containing EEG data and metadata.\n num_samples (int): Number of samples to visualize for seizure and non-seizure each.\n electrodes (list): List of electrode names to plot.\n \"\"\"\n seizure_samples = eeg_df[eeg_df['is_seizure']].sample(n=num_samples, random_state=42)\n non_seizure_samples = eeg_df[~eeg_df['is_seizure']].sample(n=num_samples, random_state=42)\n\n # Total number of samples to plot\n total_samples = len(seizure_samples) + len(non_seizure_samples)\n \n # Adjust the number of rows in the subplot grid\n fig, axes = plt.subplots(total_samples, len(electrodes), figsize=(15, 5 * total_samples))\n\n for idx, sample in enumerate([*seizure_samples.iterrows(), *non_seizure_samples.iterrows()]):\n sample_data = sample[1]\n eeg_path = sample_data['eeg_path']\n eeg_data = pd.read_parquet(eeg_path)\n\n for col_idx, electrode in enumerate(electrodes):\n signal = eeg_data[electrode]\n\n # Handle single subplot case when there is only 1 row\n ax = axes[idx, col_idx] if total_samples > 1 else axes[col_idx]\n ax.plot(signal)\n ax.set_title(f\"{'Seizure' if sample_data['is_seizure'] else 'Non-Seizure'} - {electrode}\")\n ax.set_xlabel(\"Time (ms)\")\n ax.set_ylabel(\"Amplitude\")\n\n plt.tight_layout()\n plt.savefig(\"/kaggle/working/Temporal Consistency Analysis.png\")\n plt.show()\n\n# Execute the plotting\nplot_temporal_signals(df, num_samples=3, electrodes=[\"Fp1\", \"F3\", \"C3\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:11:14.731749Z","iopub.execute_input":"2025-01-31T12:11:14.732046Z","iopub.status.idle":"2025-01-31T12:11:21.781463Z","shell.execute_reply.started":"2025-01-31T12:11:14.732023Z","shell.execute_reply":"2025-01-31T12:11:21.780512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_signal(signal, clip_threshold=1e6):\n \"\"\"\n Preprocess the EEG signal by clipping and replacing NaN values.\n\n Parameters:\n signal (array-like): The raw EEG signal.\n clip_threshold (float): The threshold to clip spikes in the signal.\n\n Returns:\n array-like: The preprocessed EEG signal.\n \"\"\"\n # Replace NaN values with the mean of the signal\n signal = np.nan_to_num(signal, nan=np.nanmean(signal))\n\n # Clip large spikes in the signal\n signal = np.clip(signal, -clip_threshold, clip_threshold)\n\n return signal\n\n# Update the frequency analysis function\ndef plot_frequency_analysis_with_preprocessing(eeg_df, num_samples=3, electrodes=[\"Fp1\", \"F3\", \"C3\"], fs=200):\n seizure_samples = eeg_df[eeg_df['is_seizure']].sample(n=num_samples, random_state=42)\n non_seizure_samples = eeg_df[~eeg_df['is_seizure']].sample(n=num_samples, random_state=42)\n\n total_samples = len(seizure_samples) + len(non_seizure_samples)\n fig, axes = plt.subplots(total_samples, len(electrodes), figsize=(15, 5 * total_samples))\n\n for idx, sample in enumerate([*seizure_samples.iterrows(), *non_seizure_samples.iterrows()]):\n sample_data = sample[1]\n eeg_path = sample_data['eeg_path']\n eeg_data = pd.read_parquet(eeg_path)\n\n for col_idx, electrode in enumerate(electrodes):\n signal = preprocess_signal(eeg_data[electrode])\n\n freqs, psd = welch(signal, fs=fs, nperseg=fs * 2)\n\n ax = axes[idx, col_idx] if total_samples > 1 else axes[col_idx]\n ax.plot(freqs, 10 * np.log10(psd + 1e-12)) # Add small constant to avoid log10(0)\n ax.set_title(f\"{'Seizure' if sample_data['is_seizure'] else 'Non-Seizure'} - {electrode}\")\n ax.set_xlabel(\"Frequency (Hz)\")\n ax.set_ylabel(\"Power (dB/Hz)\")\n\n plt.tight_layout()\n plt.savefig(\"/kaggle/working/Frequency Analysis.png\")\n plt.show()\n\n# Run the updated analysis\nplot_frequency_analysis_with_preprocessing(df, num_samples=3, electrodes=[\"Fp1\", \"F3\", \"C3\"], fs=200)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:11:31.295406Z","iopub.execute_input":"2025-01-31T12:11:31.295721Z","iopub.status.idle":"2025-01-31T12:11:37.588333Z","shell.execute_reply.started":"2025-01-31T12:11:31.295696Z","shell.execute_reply":"2025-01-31T12:11:37.587272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this section, we perform a comprehensive analysis of EEG data to generate insightful visualizations, statistical summaries, and a well-structured report π. This includes:\n
\n\n\n Let's dive in and analyze the EEG data! π¬\n
","metadata":{}},{"cell_type":"code","source":"# Create a folder for saving output if it doesn't exist\noutput_dir = '/kaggle/working/EEG Analysis'\nif not os.path.exists(output_dir):\n os.makedirs(output_dir)\n\n# Create a Word document\ndoc = Document()\ndoc.add_heading('EEG Data Analysis', 0)\n\n# Function to add figures to the Word document\ndef add_figure_to_doc(fig, caption):\n img_stream = BytesIO()\n fig.savefig(img_stream, format='png')\n img_stream.seek(0) # Rewind to the beginning of the image\n doc.add_picture(img_stream, width=Inches(5)) # Add image to the Word doc\n doc.add_paragraph(caption)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:13:55.903116Z","iopub.execute_input":"2025-01-31T12:13:55.903452Z","iopub.status.idle":"2025-01-31T12:13:55.929613Z","shell.execute_reply.started":"2025-01-31T12:13:55.903427Z","shell.execute_reply":"2025-01-31T12:13:55.928645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sampling_rate = 200 # Replace with your actual EEG sampling rate\n\n# Channels\nchannels = eeg_data.columns\n\n# 1. Visual Inspection: Plot raw EEG signals\ndef plot_raw_signals(eeg_data, channels, duration=5, fs=200):\n time = np.arange(eeg_data.shape[0]) / fs\n fig, axes = plt.subplots(5, 1, figsize=(15, 10))\n for i, channel in enumerate(channels[:5]): # Plot first 5 channels\n axes[i].plot(time[:fs * duration], eeg_data[channel].iloc[:fs * duration])\n axes[i].set_title(f\"Raw EEG Signal - {channel}\")\n axes[i].set_xlabel(\"Time (s)\")\n axes[i].set_ylabel(\"Amplitude\")\n plt.tight_layout()\n add_figure_to_doc(fig, 'Raw EEG Signal for First 5 Channels')\n plt.show()\n\nplot_raw_signals(eeg_data, channels)\n\n# 2. Power Spectral Density (PSD) Analysis\ndef plot_psd(eeg_data, channels, fs=200):\n fig, axes = plt.subplots(5, 1, figsize=(15, 10))\n for i, channel in enumerate(channels[:5]): # Plot first 5 channels\n freqs, psd = welch(eeg_data[channel], fs=fs, nperseg=fs * 2)\n axes[i].semilogy(freqs, psd)\n axes[i].set_title(f\"Power Spectral Density - {channel}\")\n axes[i].set_xlabel(\"Frequency (Hz)\")\n axes[i].set_ylabel(\"Power (dB/Hz)\")\n plt.tight_layout()\n add_figure_to_doc(fig, 'Power Spectral Density for First 5 Channels')\n plt.show()\n\nplot_psd(eeg_data, channels)\n\n# 3. Check for Missing Data or Flat Channels\ndef check_missing_flat_channels(eeg_data):\n channel_stats = eeg_data.describe().T\n flat_channels = channel_stats.loc[channel_stats['std'] == 0].index.tolist()\n doc.add_paragraph(f\"Flat Channels: {flat_channels}\")\n print(f\"Flat Channels: {flat_channels}\")\n return flat_channels\n\nflat_channels = check_missing_flat_channels(eeg_data)\n\n# 4. Band-Specific Analysis\ndef compute_band_power(eeg_data, fs=200):\n bands = {\"Delta\": (0.5, 4), \"Theta\": (4, 8), \"Alpha\": (8, 13), \"Beta\": (13, 30), \"Gamma\": (30, 40)}\n band_power = {}\n for band, (low, high) in bands.items():\n band_power[band] = eeg_data.apply(lambda x: welch(x, fs=fs, nperseg=fs * 2)[1][\n (low <= welch(x, fs=fs, nperseg=fs * 2)[0]) & \n (welch(x, fs=fs, nperseg=fs * 2)[0] <= high)].sum(), axis=0)\n band_power = pd.DataFrame(band_power)\n band_power.plot(kind='bar', figsize=(12, 6), title=\"Relative Band Power\")\n plt.ylabel(\"Power\")\n plt.xlabel(\"Channels\")\n plt.tight_layout()\n add_figure_to_doc(plt, 'Band-Specific Power Analysis')\n plt.show()\n\ncompute_band_power(eeg_data)\n\n# 5. Independent Component Analysis (ICA)\ndef run_ica(eeg_data, sampling_rate=200, highpass_freq=1.0):\n n_channels = len(channels)\n info = create_info(ch_names=list(channels), sfreq=sampling_rate, ch_types=[\"eeg\"] * n_channels)\n \n # Create random 3D coordinates for each channel\n # This is just for visualization purposes, ideally you'd use a proper montage\n info.set_montage('standard_1020') # Standard system of EEG recordings\n \n # Create a RawArray from the EEG data\n raw_array = RawArray(eeg_data.values.T, info)\n \n # Apply high-pass filter to the data\n raw_array.filter(l_freq=highpass_freq, h_freq=None) # High-pass filter with 1.0 Hz lower bound\n \n ica = ICA(n_components=n_channels, random_state=42)\n ica.fit(raw_array) # Now fitting to RawArray\n \n # Plot ICA components and save the figure manually\n ica_fig = ica.plot_components(picks=None, show=False) # This returns a figure\n ica_fig.savefig(os.path.join(output_dir, 'ica_components.png')) # Save ICA components plot to file\n \n # Add the ICA components plot to the document\n add_figure_to_doc(ica_fig, 'Independent Component Analysis (ICA) Components')\n return ica\n\nica = run_ica(eeg_data)\n\n# 6. Correlation Between Channels\ndef plot_channel_correlation(eeg_data):\n corr_matrix = eeg_data.corr()\n fig, ax = plt.subplots(figsize=(10, 8))\n cax = ax.imshow(corr_matrix, cmap='viridis', interpolation='none')\n fig.colorbar(cax, ax=ax, label='Correlation')\n ax.set_title(\"Channel Correlation Matrix\")\n ax.set_xlabel(\"Channels\")\n ax.set_ylabel(\"Channels\")\n plt.tight_layout()\n add_figure_to_doc(fig, 'Channel Correlation Matrix')\n plt.show()\n\nplot_channel_correlation(eeg_data)\n\n# 7. Signal-to-Noise Ratio (SNR)\ndef compute_snr(eeg_data):\n signal_power = (eeg_data ** 2).mean(axis=0)\n noise_power = eeg_data.diff().dropna() ** 2\n noise_power = noise_power.mean(axis=0)\n snr = 10 * np.log10(signal_power / noise_power)\n doc.add_paragraph(\"Signal-to-Noise Ratio (SNR) for each channel:\")\n doc.add_paragraph(snr.to_string())\n print(\"SNR for each channel:\")\n print(snr)\n return snr\n\nsnr = compute_snr(eeg_data)\n\n# 8. Epoch-Level Noise Analysis\ndef check_epoch_noise(eeg_data, epoch_duration=2, fs=200):\n epoch_samples = fs * epoch_duration\n num_epochs = eeg_data.shape[0] // epoch_samples\n variances = []\n for i in range(num_epochs):\n epoch = eeg_data.iloc[i * epoch_samples:(i + 1) * epoch_samples]\n variances.append(epoch.var(axis=0))\n variances = pd.DataFrame(variances)\n fig, ax = plt.subplots(figsize=(10, 6))\n ax.boxplot(variances.values)\n ax.set_title(\"Epoch-Level Variance Across Channels\")\n ax.set_xlabel(\"Channels\")\n ax.set_ylabel(\"Variance\")\n plt.tight_layout()\n add_figure_to_doc(fig, 'Epoch-Level Variance Across Channels')\n plt.show()\n\ncheck_epoch_noise(eeg_data)\n\n# 9. Statistical Properties\ndef plot_statistical_properties(eeg_data):\n stats = {\n \"Mean\": eeg_data.mean(),\n \"Std\": eeg_data.std(),\n \"Skewness\": eeg_data.apply(skew),\n \"Kurtosis\": eeg_data.apply(kurtosis),\n }\n stats_df = pd.DataFrame(stats)\n stats_df.plot(kind='bar', figsize=(15, 6), subplots=True, layout=(2, 2), title=\"Statistical Properties\")\n plt.tight_layout()\n add_figure_to_doc(plt, 'Statistical Properties')\n plt.show()\n\nplot_statistical_properties(eeg_data)\n\n# Save the Word document\ndoc.save(os.path.join(output_dir, 'EEG_Analysis_Report.docx'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:14:08.368263Z","iopub.execute_input":"2025-01-31T12:14:08.368636Z","iopub.status.idle":"2025-01-31T12:14:27.769685Z","shell.execute_reply.started":"2025-01-31T12:14:08.368604Z","shell.execute_reply":"2025-01-31T12:14:27.768968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this section, we compute the Signal-to-Noise Ratio (SNR) for each EEG file, filter out channels with low SNR, and use a classification model to identify most important channels in seizure detection. The steps include:\n
\n\n\n Let's extract meaningful insights from EEG data and important channels for seizure detection! β‘\n
","metadata":{}},{"cell_type":"code","source":"# Function to compute SNR for one file\ndef compute_snr_for_file(eeg_file, clip_threshold=1e6):\n eeg_data = pd.read_parquet(eeg_file)\n eeg_data.drop(columns=[\"EKG\"], inplace=True)\n \n # Handle NaNs or Infs\n if eeg_data.isnull().values.any() or np.isinf(eeg_data.values).any():\n eeg_data = eeg_data.fillna(0)\n eeg_data = eeg_data.replace([np.inf, -np.inf], 0)\n \n eeg_data = eeg_data.clip(-clip_threshold, clip_threshold) # Clip extreme values\n\n # Compute signal and noise power\n signal_power = (eeg_data ** 2).mean(axis=0)\n if np.isnan(signal_power).any():\n print(f\"Warning: NaN values detected in signal power for {eeg_file}.\")\n \n noise_power = eeg_data.diff().dropna() ** 2\n noise_power = noise_power.mean(axis=0)\n if np.isnan(noise_power).any():\n print(f\"Warning: NaN values detected in noise power for {eeg_file}.\")\n \n # Add small epsilon to avoid log(0)\n epsilon = 1e-10\n snr = 10 * np.log10((signal_power + epsilon) / (noise_power + epsilon))\n \n return snr\n\n# Randomly sample EEG files for analysis\nsample_files = df.sample(100, random_state=CFG.seed)['eeg_path']\n\n# Compute SNR for each file\nsnr_results = []\nfor file in sample_files:\n snr = compute_snr_for_file(file)\n snr_results.append(snr)\n\n# Aggregate SNR results across files\nsnr_df = pd.DataFrame(snr_results).mean(axis=0)\nprint(\"Average SNR for Channels:\")\nprint(snr_df)\n\n# Filter channels with high SNR\nsnr_threshold = 0 # Adjust based on results\nhigh_snr_channels = snr_df[snr_df > snr_threshold].index.tolist()\nprint(\"High-SNR Channels:\", high_snr_channels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:15:36.808905Z","iopub.execute_input":"2025-01-31T12:15:36.809212Z","iopub.status.idle":"2025-01-31T12:15:41.022237Z","shell.execute_reply.started":"2025-01-31T12:15:36.809189Z","shell.execute_reply":"2025-01-31T12:15:41.021038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select a balanced subset of seizure and non-seizure data\nseizure_files = df[df['is_seizure']]['eeg_path'].sample(50, random_state=CFG.seed)\nnon_seizure_files = df[~df['is_seizure']]['eeg_path'].sample(50, random_state=CFG.seed)\nfiles_to_process = pd.concat([seizure_files, non_seizure_files])\n\n# Prepare data for classification\nX, y = [], []\nfor file in files_to_process:\n eeg_data = pd.read_parquet(file)[high_snr_channels]\n X.append(eeg_data.mean(axis=0)) # Aggregate features (e.g., mean)\n y.append(df.loc[df['eeg_path'] == file, 'is_seizure'].values[0])\n\n# Convert to DataFrame\nX = pd.DataFrame(X, columns=high_snr_channels)\ny = np.array(y)\n\n# Train-test split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=CFG.seed)\n\n# Train a Random Forest Classifier\nrf_model = RandomForestClassifier(random_state=CFG.seed)\nrf_model.fit(X_train, y_train)\n\n# Feature importance\nimportances = pd.Series(rf_model.feature_importances_, index=high_snr_channels)\nimportances.sort_values(ascending=False).plot(kind='bar', figsize=(10, 6), title=\"Channel Importance\")\nplt.savefig(\"Feature Importance.png\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:15:53.722356Z","iopub.execute_input":"2025-01-31T12:15:53.722799Z","iopub.status.idle":"2025-01-31T12:15:57.425803Z","shell.execute_reply.started":"2025-01-31T12:15:53.722759Z","shell.execute_reply":"2025-01-31T12:15:57.425070Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this section, we perform detailed preprocessing of EEG data to ensure high-quality features for further analysis:\n
\n\n\n After processing, we check the distribution of expert consensus and patient data. π\n
","metadata":{}},{"cell_type":"code","source":"# Selected top channels based on analysis\nSELECTED_CHANNELS = ['Fp1', 'O2', 'T6', 'Fz', 'F4', 'T3', 'Cz', 'T5', 'C4', 'P3']\n\n# Define thresholds\nLOW_VARIANCE_THRESHOLD = 0.001\nHIGH_VARIANCE_THRESHOLD = 10\n\ndef wavelet_denoise(data, wavelet=\"db4\", level=2):\n coeffs = pywt.wavedec(data, wavelet, axis=0)\n threshold = np.median(np.abs(coeffs[-1])) / 0.6745 # Universal threshold\n coeffs = [pywt.threshold(c, threshold, mode=\"soft\") for c in coeffs]\n return pywt.waverec(coeffs, wavelet, axis=0)\n\ndef calculate_snr(signal):\n mean_signal = np.mean(signal)\n noise = signal - mean_signal\n snr = 10 * np.log10(np.var(signal) / np.var(noise))\n return snr\n\n# Define a function to process a single eeg_id\ndef evaluate_eeg(eeg_id):\n eeg_path = f\"{BASE_PATH}/train_eegs/{eeg_id}.parquet\"\n try:\n # Load EEG data\n eeg = pd.read_parquet(eeg_path)\n \n # Select relevant channels\n chan_eeg = eeg[SELECTED_CHANNELS]\n \n # Replace NaN and infinite values\n chan_eeg = chan_eeg.replace([np.inf, -np.inf], 0).fillna(0)\n\n # Apply bandpass filtering\n chan_eeg = bandpass_filter(chan_eeg)\n\n # Wavelet denoising\n chan_eeg = wavelet_denoise(chan_eeg)\n\n # Normalize data with safeguard for zero standard deviation\n stds = chan_eeg.std(axis=0)\n stds[stds == 0] = 1e-8 # Replace zero stds with a small constant\n normalized_eeg = (chan_eeg - chan_eeg.mean(axis=0)) / stds\n normalized_eeg = np.nan_to_num(normalized_eeg) # Ensure no NaN or inf values\n\n # Variance checks\n variances = normalized_eeg.var(axis=0)\n if (variances < LOW_VARIANCE_THRESHOLD).any() or (variances > HIGH_VARIANCE_THRESHOLD).any():\n # print(f\"Discarded {eeg_id} due to variance thresholds after preprocessing.\")\n return None\n\n # SNR checks\n snr_values = np.apply_along_axis(calculate_snr, axis=0, arr=normalized_eeg)\n if (snr_values < 0).any():\n # print(f\"Discarded {eeg_id} due to negative SNR after preprocessing.\")\n return None\n \n # Undersample to 6,000 rows\n # The EEGs were recored at 200 samples / second\n # 200 * 30 = 6000 Samples\n # Therefore we will pick first 6000 samples which will consolidate 30 seconds of recording\n sub_eeg = normalized_eeg[:6000, :]\n \n # Convert to NumPy and save\n # eeg_np = sub_eeg.to_numpy(dtype='float', na_value=0)\n return sub_eeg # Return eeg_id if successfully processed\n \n except Exception as e:\n # print(f\"Error processing {eeg_id}: {e}\")\n return None\n\ndef process_and_save_eeg(eeg_id):\n processed = evaluate_eeg(eeg_id)\n if processed is not None:\n np.save(f\"{EEG_DIR}/{eeg_id}.npy\", processed)\n return eeg_id # Return ID if successfully processed\n return None\n\nif __name__ == \"__main__\":\n # Collect all processed EEG IDs\n results = Parallel(n_jobs=-1, backend=\"loky\")(\n delayed(process_and_save_eeg)(eeg_id)\n for eeg_id in tqdm(df[\"eeg_id\"], desc=\"Processing EEG files\")\n )\n\n # Filter out None values and save successfully processed IDs\n accepted_eegs = [eeg_id for eeg_id in results if eeg_id is not None]\n print(f\"Successfully processed {len(accepted_eegs)} EEG files.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:53:23.701462Z","iopub.execute_input":"2025-01-31T12:53:23.701832Z","iopub.status.idle":"2025-01-31T12:55:00.126687Z","shell.execute_reply.started":"2025-01-31T12:53:23.701798Z","shell.execute_reply":"2025-01-31T12:55:00.125760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accepted_df = df[df[\"eeg_id\"].isin(accepted_eegs)]\nclass_counts = accepted_df['expert_consensus'].value_counts()\npatients = accepted_df[\"patient_id\"].value_counts()\ndisplay(class_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T15:57:56.442364Z","iopub.execute_input":"2025-01-31T15:57:56.442716Z","iopub.status.idle":"2025-01-31T15:57:56.453593Z","shell.execute_reply.started":"2025-01-31T15:57:56.442684Z","shell.execute_reply":"2025-01-31T15:57:56.452866Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this section, we perform the following key steps for model training:\n
\n\n\n These prepared datasets are ready for training deep learning models on EEG data. π§ π‘\n
","metadata":{}},{"cell_type":"code","source":"# Encode the \"is_seizure\" label (True -> 1, False -> 0)\naccepted_df[\"is_seizure\"] = accepted_df[\"is_seizure\"].astype(int)\n\n# Convert to NumPy arrays\neeg_ids = accepted_df[\"eeg_id\"].to_numpy()\nlabels = accepted_df[\"is_seizure\"].to_numpy()\n\n# Number of folds\nn_splits = 5\n\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=CFG.seed)\nsplits = list(skf.split(eeg_ids, labels))\n\n# Placeholder to store datasets for each fold\nfold_datasets = []\n\n# Function to load EEG data from .npy files\ndef load_eeg(eeg_id, label):\n eeg_id_str = tf.strings.as_string(eeg_id)\n eeg_path = tf.strings.join([EEG_DIR, tf.strings.join([eeg_id_str, \".npy\"], separator=\"\")], separator=\"/\")\n eeg_data = tf.numpy_function(np.load, [eeg_path], tf.float64)\n eeg_data = tf.cast(eeg_data, tf.float32)\n eeg_data = tf.ensure_shape(eeg_data, [6000, 10])\n return eeg_data, label\n\n# Function to apply data augmentation\ndef augment_data(eeg_data, label):\n # Add Gaussian noise\n noise = tf.random.normal(tf.shape(eeg_data), mean=0.0, stddev=0.02)\n eeg_data = eeg_data + noise\n\n # Random time shifting\n time_shift = tf.random.uniform([], minval=-50, maxval=50, dtype=tf.int32)\n eeg_data = tf.roll(eeg_data, shift=time_shift, axis=0)\n\n return eeg_data, label\n\n# Function to preprocess the data\ndef preprocess_data(eeg_data, label):\n eeg_data = tf.expand_dims(eeg_data, axis=-1) # Add a channel dimension: (6000, 10) -> (6000, 10, 1)\n return eeg_data, label\n\n# Function to create TensorFlow datasets for a fold\ndef create_fold_dataset(train_idx, test_idx):\n # Get train and test EEG IDs and labels\n train_ids, train_labels = eeg_ids[train_idx], labels[train_idx]\n test_ids, test_labels = eeg_ids[test_idx], labels[test_idx]\n\n # Create TensorFlow datasets\n train_dataset = tf.data.Dataset.from_tensor_slices((train_ids, train_labels))\n test_dataset = tf.data.Dataset.from_tensor_slices((test_ids, test_labels))\n\n # Map the loading and preprocessing functions\n train_dataset = train_dataset.map(\n lambda eeg_id, label: load_eeg(eeg_id, label), num_parallel_calls=tf.data.AUTOTUNE\n )\n train_dataset = train_dataset.map(preprocess_data, num_parallel_calls=tf.data.AUTOTUNE)\n test_dataset = test_dataset.map(\n lambda eeg_id, label: load_eeg(eeg_id, label), num_parallel_calls=tf.data.AUTOTUNE\n )\n test_dataset = test_dataset.map(preprocess_data, num_parallel_calls=tf.data.AUTOTUNE)\n\n # Shuffle, batch, and prefetch\n batch_size = CFG.batch_size\n train_dataset = train_dataset.shuffle(buffer_size=1000).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n test_dataset = test_dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n\n return train_dataset, test_dataset\n\n# Loop through each fold and create datasets\nfor i, (train_idx, test_idx) in enumerate(splits):\n print(f\"Creating datasets for Fold {i + 1}\")\n train_dataset, test_dataset = create_fold_dataset(train_idx, test_idx)\n fold_datasets.append((train_dataset, test_dataset))\n\nprint(f\"Successfully created datasets for {n_splits} folds.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T12:55:12.074987Z","iopub.execute_input":"2025-01-31T12:55:12.075373Z","iopub.status.idle":"2025-01-31T12:55:13.859536Z","shell.execute_reply.started":"2025-01-31T12:55:12.075337Z","shell.execute_reply":"2025-01-31T12:55:13.858550Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n This model is designed for EEG data classification, specifically to distinguish between seizure and non-seizure events, using a combination of Convolutional Neural Networks (CNNs) and Long Short-Term Memory (LSTM) networks. Below is a detailed breakdown of each layer and its purpose.\n
\n\n\n The CNN-LSTM architecture efficiently combines both spatial feature extraction (through CNN layers) and temporal dependency capture (through LSTM layers). This hybrid approach enables the model to effectively handle the spatiotemporal nature of EEG data. The model works as follows:\n
\n\n\n This layer receives the EEG data in the shape of (6000, 10, 1). The first dimension represents 6000 time steps, the second represents 10 EEG channels, and the third represents a single feature (since the data is 1D in this case).\n
\n\n\n This block includes two convolutional layers followed by batch normalization and max-pooling layers:\n
\n The output from the CNN block is reshaped from (6000, 2, 64) to (6000, 128) in preparation for the LSTM layers. This transformation ensures that the data is compatible for sequential learning by the LSTM, which is vital for capturing temporal dependencies.\n
\n\n\n The LSTM layers are designed to capture long-range temporal dependencies in the EEG signals:\n
\n The fully connected layers are used for classification:\n
\n Functionality: The Adam optimizer is an adaptive learning rate optimization algorithm that combines the advantages of both RMSprop and momentum. It computes adaptive learning rates for each parameter from estimates of first and second moments of the gradients.\n
Reason for Usage: Adam is chosen for its efficiency, scalability, and ability to work well with sparse gradients, which is particularly useful in EEG classification where the data can be noisy and sparse.\n
\n Functionality: This loss function is used for multi-class classification problems where labels are provided as integers. It computes the cross-entropy loss between the true labels and the predicted probabilities.\n
Reason for Usage: Sparse categorical crossentropy is used since the target labels (seizure vs. non-seizure) are integers, making it suitable for this classification task.\n
\n Accuracy is a standard metric for classification tasks. It calculates the proportion of correct predictions over the total number of predictions.\n
","metadata":{}},{"cell_type":"code","source":"def create_cnn_lstm(input_shape=(6000, 10, 1), num_classes=2):\n model = models.Sequential(name=\"CNN-BiLSTM-Model\")\n\n # Input Layer\n model.add(layers.Input(shape=input_shape, name=\"Input-Layer\"))\n \n # 1. CNN Block (Spatial Feature Extraction)\n model.add(layers.Conv2D(name=\"Convolution-1\", filters=32, kernel_size=(1, 5), activation='relu', padding='same')) # Added padding\n model.add(layers.BatchNormalization(name=\"Normalization-1\"))\n model.add(layers.MaxPooling2D(name=\"Pooling-1\", pool_size=(1, 2))) # Reduces the width by a factor of 2\n\n model.add(layers.Conv2D(name=\"Convolution-2\", filters=64, kernel_size=(1, 3), activation='relu', padding='same')) # Added padding\n model.add(layers.BatchNormalization(name=\"Normalization-2\"))\n model.add(layers.MaxPooling2D(name=\"Pooling-2\", pool_size=(1, 2))) # Reduces the width by a factor of 2\n \n # After Conv2D and MaxPooling layers, the shape is reduced to:\n # Height = 6000 (unchanged), Width = 10 -> (10 / 2 / 2) = 2, Depth = 64\n # So, the shape after this block is (6000, 2, 64)\n \n # 2. Temporal Downsampling (Reshape for LSTM)\n # The output shape after CNN and Pooling is (6000, 2, 64), so we reshape it to (6000, 128)\n model.add(layers.Reshape(name=\"Reshape-Conv-Output\", target_shape=(6000, 128))) # 6000 time steps, 128 features\n \n # 3. LSTM Block (Temporal Dependency Capture)\n model.add(layers.Bidirectional(layers.LSTM(128, return_sequences=True), name=\"Bi-LSTM-1\"))\n model.add(layers.Dropout(0.2, name=\"Dropout-1\"))\n model.add(layers.Bidirectional(layers.LSTM(64), name=\"Bi-LSTM-2\"))\n\n # 4. Fully Connected Layers (Classification)\n model.add(layers.Dense(64, name=\"Dense-1\", activation='relu'))\n model.add(layers.Dropout(0.2, name=\"Dropout-2\"))\n model.add(layers.Dense(32, name=\"Dense-2\", activation='relu'))\n model.add(layers.Dense(num_classes, name=\"Output-Layer\", activation='softmax'))\n\n # Compile the model\n model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n return model\n\n# Instantiate the model\nmodel = create_cnn_lstm()\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T13:21:50.722936Z","iopub.execute_input":"2025-01-31T13:21:50.723330Z","iopub.status.idle":"2025-01-31T13:21:50.934856Z","shell.execute_reply.started":"2025-01-31T13:21:50.723296Z","shell.execute_reply":"2025-01-31T13:21:50.934147Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this cell, we implement the following key callbacks for optimizing model training:\n
\n\n\n These callbacks will help in optimizing the training process and achieving better model performance. π»β¨\n
\n","metadata":{}},{"cell_type":"code","source":"ckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n 'cnn_lstm.keras', # File path to save the model\n monitor=\"val_loss\",\n save_best_only=True, # Save regardless of validation loss\n save_weights_only=False, # Save the full model (not just weights)\n save_freq='epoch', # Save at the end of each epoch (which is the default)\n verbose=0 # Display a message when saving\n)\n\n# Early stopping callback\nerst_cb = EarlyStopping(\n monitor=\"val_loss\", # Monitor validation loss\n patience=3, # Number of epochs to wait before stopping\n restore_best_weights=True, # Restore the model with the best weights\n verbose=CFG.verbose\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T13:19:34.493027Z","iopub.execute_input":"2025-01-31T13:19:34.493369Z","iopub.status.idle":"2025-01-31T13:19:34.497749Z","shell.execute_reply.started":"2025-01-31T13:19:34.493327Z","shell.execute_reply":"2025-01-31T13:19:34.496928Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n In this section, we perform the following key steps to train and evaluate the model across multiple folds:\n
\n\n\n This process helps in assessing the model's generalization ability and robustness over multiple data splits. π±π‘\n
","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nepochs = 10\n\n# Loop through each fold\nfor fold_idx, (train_dataset, test_dataset) in enumerate(fold_datasets):\n print(f\"\\nTraining Fold {fold_idx + 1}/{len(fold_datasets)}\")\n \n # Split train_dataset into train and validation datasets\n validation_split = 0.2 # 20% for validation\n total_train_size = len(train_dataset)\n val_size = int(total_train_size * validation_split)\n \n train_dataset = train_dataset.skip(val_size)\n val_dataset = train_dataset.take(val_size)\n \n # Train the model\n training = model.fit(\n train_dataset,\n validation_data=val_dataset,\n epochs=epochs,\n callbacks=[ckpt_cb, erst_cb],\n verbose=CFG.verbose,\n )\n\n # Extract loss and accuracy values from the history object\n train_loss = training.history['loss']\n val_loss = training.history['val_loss']\n train_acc = training.history['accuracy']\n val_acc = training.history['val_accuracy']\n \n # Plotting the training and validation loss\n plt.figure(figsize=(12, 6))\n plt.subplot(1, 2, 1)\n plt.plot(range(1, len(train_loss) + 1), train_loss, label='Training Loss')\n plt.plot(range(1, len(val_loss) + 1), val_loss, label='Validation Loss')\n plt.title('Loss over Epochs')\n plt.xlabel('Epochs')\n plt.ylabel('Loss')\n plt.legend()\n \n # Plotting the training and validation accuracy\n plt.subplot(1, 2, 2)\n plt.plot(range(1, len(train_acc) + 1), train_acc, label='Training Accuracy')\n plt.plot(range(1, len(val_acc) + 1), val_acc, label='Validation Accuracy')\n plt.title('Accuracy over Epochs')\n plt.xlabel('Epochs')\n plt.ylabel('Accuracy')\n plt.legend()\n \n # Display the plots\n plt.tight_layout()\n plt.show()\n \n # Evaluate on the testing dataset\n test_loss, test_accuracy = model.evaluate(test_dataset, verbose=0)\n print(f\"Fold {fold_idx + 1} - Test Accuracy: {test_accuracy:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T13:22:03.970927Z","iopub.execute_input":"2025-01-31T13:22:03.971243Z","iopub.status.idle":"2025-01-31T14:39:33.507845Z","shell.execute_reply.started":"2025-01-31T13:22:03.971220Z","shell.execute_reply":"2025-01-31T14:39:33.507013Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n This section is dedicated to evaluating the model's performance on the test set across all folds. We compute important classification metrics and visualize the results with a confusion matrix.\n
\n\n\n These evaluations and metrics provide an overview of the model's ability to classify seizure events accurately. The model is then saved for further deployment and analysis. π§ πΎ\n
","metadata":{}},{"cell_type":"code","source":"# Collect true and predicted labels for the test set across all folds\ny_true_all = []\ny_pred_all = []\n\nprint(\"Evaluation Running...\")\n\n# Loop through each fold's test dataset\nfor fold_idx, (train_dataset, test_dataset) in enumerate(fold_datasets):\n print(f\"\\nEvaluating Fold {fold_idx + 1}/{len(fold_datasets)}\")\n \n y_true = []\n y_pred = []\n \n for eeg_data, labels in test_dataset:\n predictions = model.predict(eeg_data, verbose=0)\n y_true.extend(labels.numpy())\n y_pred.extend(tf.argmax(predictions, axis=1).numpy())\n \n # Append results for all folds\n y_true_all.extend(y_true)\n y_pred_all.extend(y_pred)\n\nprint(\"Evaluation Ends!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T14:40:40.188179Z","iopub.execute_input":"2025-01-31T14:40:40.188549Z","iopub.status.idle":"2025-01-31T14:42:23.023256Z","shell.execute_reply.started":"2025-01-31T14:40:40.188519Z","shell.execute_reply":"2025-01-31T14:42:23.022451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cm = confusion_matrix(y_true_all, y_pred_all)\n# Plot confusion matrix heatmap\nplt.figure(figsize=(8, 5))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.show()\n\n# Print the average accuracy & specificity model acheived\nTN, FP, FN, TP = cm.ravel()\nprecision = TP / (TP + FP)\nsensitivity = TP / (TP + FN)\nspecificity = TN / (TN + FP)\nacc = accuracy_score(y_true_all, y_pred_all)\nprint(f\"\\033[1mModel Precision: {round(precision, 2) * 100}%\\033[0m\")\nprint(f\"\\033[1mModel Sensitivity: {round(sensitivity, 2) * 100}%\\033[0m\")\nprint(f\"\\033[1mModel Specificity: {round(specificity, 2) * 100}%\\033[0m\")\nprint(f\"\\033[1mModel Mean Accuracy: {round(acc, 2) * 100}%\\n\\033[0m\")\n\n# Generate a classification report\nprint(\"\\t\\t\\tCLASSIFICATION REPORT:\")\nprint(classification_report(y_true_all, y_pred_all, target_names=[\"Non-Seizure\", \"Seizure\"]))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-31T15:32:56.920088Z","iopub.execute_input":"2025-01-31T15:32:56.920424Z","iopub.status.idle":"2025-01-31T15:32:57.132269Z","shell.execute_reply.started":"2025-01-31T15:32:56.920396Z","shell.execute_reply":"2025-01-31T15:32:57.131360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save(\"cnn_lstm.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-26T10:32:23.078563Z","iopub.execute_input":"2025-01-26T10:32:23.078910Z","iopub.status.idle":"2025-01-26T10:32:23.180595Z","shell.execute_reply.started":"2025-01-26T10:32:23.078884Z","shell.execute_reply":"2025-01-26T10:32:23.179808Z"}},"outputs":[],"execution_count":null}]}