{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Importing relevant libraries\nimport pandas as pd\nimport numpy as np\nfrom scipy import signal\nfrom scipy.signal import butter, filtfilt, iirnotch\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom scipy import stats\nimport sklearn\nimport os\nimport mne\nfrom scipy.fft import fft, ifft","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-23T17:13:00.786171Z","iopub.execute_input":"2024-03-23T17:13:00.786995Z","iopub.status.idle":"2024-03-23T17:13:00.793709Z","shell.execute_reply.started":"2024-03-23T17:13:00.786953Z","shell.execute_reply":"2024-03-23T17:13:00.792648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Read .csv file📁","metadata":{}},{"cell_type":"code","source":"csv = '/kaggle/input/hms-harmful-brain-activity-classification/train.csv'\ndf = pd.read_csv(csv)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T17:13:00.808022Z","iopub.execute_input":"2024-03-23T17:13:00.808730Z","iopub.status.idle":"2024-03-23T17:13:01.011865Z","shell.execute_reply.started":"2024-03-23T17:13:00.808694Z","shell.execute_reply":"2024-03-23T17:13:01.010616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Signal Processing 📶","metadata":{}},{"cell_type":"code","source":"def butterfilt(data, lowcut, highcut, fs, order=4, padlen = None):\n    nyquist = 0.5 * fs\n    low = lowcut / nyquist\n    high = highcut / nyquist\n    b, a = butter(order, [low, high], btype='band')\n    bandpass_filtered_data = filtfilt(b, a, data, axis=0)\n    return bandpass_filtered_data\n\ndef notchfilt(data, fs, notch_freq=50, Q=10):\n    b, a = iirnotch(notch_freq, Q, fs=fs)\n    notched_data = filtfilt(b, a, data, axis=0)\n    return notched_data","metadata":{"execution":{"iopub.status.busy":"2024-03-23T17:13:01.014061Z","iopub.execute_input":"2024-03-23T17:13:01.014391Z","iopub.status.idle":"2024-03-23T17:13:01.024161Z","shell.execute_reply.started":"2024-03-23T17:13:01.014363Z","shell.execute_reply":"2024-03-23T17:13:01.022888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_filters(signal_data):\n    # Apply notch filter\n    notched_data = notchfilt(signal_data, fs=200)\n    # Apply bandpass filter\n    filtered_data = butterfilt(notched_data, lowcut=0.1, highcut=50, fs=200, padlen=None)\n    return filtered_data","metadata":{"execution":{"iopub.status.busy":"2024-03-23T17:13:01.025714Z","iopub.execute_input":"2024-03-23T17:13:01.026139Z","iopub.status.idle":"2024-03-23T17:13:01.036253Z","shell.execute_reply.started":"2024-03-23T17:13:01.026101Z","shell.execute_reply":"2024-03-23T17:13:01.035135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Feature Extraction ⚗️\nStatistics used as a proxy for the features of the signal:\n1. Mean: Central tendency of the signal, this may be higher in seizures and GPD due to higher overall activity, whereas thet might be slightly lower in GRDA and LRDA due to the presence of slow delta activiy. Might also reflect asymmetry in LPD and LRDA.\n2. Standard deviation: Variability of the signal around the mean: this may be higher in seizures to due drastic changes in amplitude, as well as in LRDA, LPD and GPD.\n3. Peak-to-peak amplitude: difference between the highest and lowest points in the data. Likely to be higher in seizures due to more drastic changes in amplitude.\n4. Variation: Measure of spread, higher in seizures due to more drastic and frequent changes, as well as in GPD, LRDA and LPD.\n5. The minimum and maximum values of the signal: assessment of amplitude.\n6. Square root of the signal\n7. Skew: Asymmetry of the distribution, positive in seizures due to asymmetry compared to baseline activity, but also seen in LRDA and LPD as these are lateralised.\n8. Kurtosis: Peakedness/flatness of the signal, may be higher in seizure due to more peaks.","metadata":{}},{"cell_type":"code","source":"def extract_features(signal_data):\n    features = []\n    \n    # Define feature extraction functions\n    def mean(x):\n        return np.mean(x, axis=0)\n    \n    def std(x):\n        return np.std(x, axis=0)\n    \n    def ptp(x):\n        return np.ptp(x, axis=0)\n    \n    def var(x):\n        return np.var(x, axis=0)\n    \n    def minim(x):\n        return np.min(x, axis=0)\n    \n    def maxim(x):\n        return np.max(x, axis=0)\n    \n    def sqrt(x):\n        return np.sqrt(np.mean(x ** 2, axis=0))\n    \n    def abs_diff_sigma(x):\n        return np.sum(np.abs(np.diff(x, axis=0)), axis=0)\n    \n    def skew(x):\n        return stats.skew(x, axis=0)\n    \n    def kurtosis(x):\n        return stats.kurtosis(x, axis=0)\n    \n    # Extract features from signal data\n    signal_features = [\n        mean(signal_data),\n        std(signal_data),\n        ptp(signal_data),\n        var(signal_data),\n        minim(signal_data),\n        maxim(signal_data),\n        sqrt(signal_data),\n        abs_diff_sigma(signal_data),\n        skew(signal_data),\n        kurtosis(signal_data)\n    ]\n    \n    return signal_features","metadata":{"execution":{"iopub.status.busy":"2024-03-23T17:13:01.039178Z","iopub.execute_input":"2024-03-23T17:13:01.039470Z","iopub.status.idle":"2024-03-23T17:13:01.052652Z","shell.execute_reply.started":"2024-03-23T17:13:01.039445Z","shell.execute_reply":"2024-03-23T17:13:01.051463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Preparing Data and Labels 👩‍🍳","metadata":{}},{"cell_type":"code","source":"# Step 4: Prepare Data and Labels\nfilename_to_label = dict(zip(df['eeg_id'], df['expert_consensus']))\n\nparquet_folder = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs'\nparquet_files = os.listdir(parquet_folder)\n\nfeatures = []\nlabels = []\n\ntotal_files = len(parquet_files)\nprocessed_files = 0\n\nfor _, row in df.iterrows():\n    parquet_file = row['eeg_id']\n    # Read Parquet data\n    parquet_data = pd.read_parquet(os.path.join(parquet_folder, f\"{parquet_file}.parquet\"))\n    \n    # Convert offset seconds to integer\n    offset_seconds = int(row['eeg_label_offset_seconds'])\n    signal_data = parquet_data[offset_seconds:offset_seconds+50]  # Use the integer value for slicing\n    filtered_signal_data = apply_filters(signal_data)\n    signal_features = extract_features(signal_data)\n    features.append(signal_features)\n    labels.append(filename_to_label[parquet_file])\n    \n    processed_files += 1\n    #print(f\"Processed {parquet_file} | Progress: {processed_files}/{total_files} ({processed_files/total_files*100:.2f}%)\")\n\nX = np.array(features)\ny = np.array(labels)\n\nprint('Processing Complete')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Train Classifier 📈","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nX_train = X_train.reshape(len(X_train), -1)\ny_train = y_train.reshape(len(y_train), -1)\nX_test = X_test.reshape(len(X_test), -1)\ny_test = y_test.reshape(len(y_test), -1)\n\nrf_classifier = sklearn.ensemble.HistGradientBoostingClassifier(max_iter=100, random_state=42)\nrf_classifier.fit(X_train, y_train)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-23T18:11:13.532235Z","iopub.execute_input":"2024-03-23T18:11:13.532612Z","iopub.status.idle":"2024-03-23T18:11:48.170246Z","shell.execute_reply.started":"2024-03-23T18:11:13.532579Z","shell.execute_reply":"2024-03-23T18:11:48.169435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Model Accuracy (Validation)\naccuracy = rf_classifier.score(X_test, y_test)\nprint(\"Accuracy:\", accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T18:11:48.173582Z","iopub.execute_input":"2024-03-23T18:11:48.173923Z","iopub.status.idle":"2024-03-23T18:11:48.852952Z","shell.execute_reply.started":"2024-03-23T18:11:48.173898Z","shell.execute_reply":"2024-03-23T18:11:48.852199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Submission  📋","metadata":{}},{"cell_type":"code","source":"parquet_folder_test = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs'\nparquet_files_test = os.listdir(parquet_folder_test)\n\n\ntest_features = []\ntest_csv = '/kaggle/input/hms-harmful-brain-activity-classification/test.csv'\ndf_test = pd.read_csv(test_csv)\ndf_test.head()\n\nfor _, row in df_test.iterrows():\n    parquet_file_test = row['eeg_id']\n    # Read Parquet data\n    parquet_data_test = pd.read_parquet(os.path.join(parquet_folder_test, f\"{parquet_file_test}.parquet\"))\n    filtered_signal_test = apply_filters(parquet_data_test)\n    signal_features_test = extract_features(filtered_signal_test)\n    test_features.append(signal_features_test)\n\nX_submission = np.array(test_features)\nX_submission = X_submission.reshape(len(X_submission), -1)\n\nprint('Processing Complete')\n\n\n# Step 2: Perform prediction\nprediction_probabilities = rf_classifier.predict_proba(X_submission)[0]  # Predict probabilities\n\n# Step 3: Initialize a DataFrame to store predictions\npredictions_df = pd.DataFrame(columns=['eeg_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'])\n\n# Step 4: Create a row for the new image prediction\nrow = pd.DataFrame({\n    'eeg_id': ['3911565283'],  \n    'seizure_vote': [prediction_probabilities[0]],\n    'lpd_vote': [prediction_probabilities[1]],\n    'gpd_vote': [prediction_probabilities[2]],\n    'lrda_vote': [prediction_probabilities[3]],\n    'grda_vote': [prediction_probabilities[4]],\n    'other_vote': [prediction_probabilities[5]]\n})\n\n# Step 5: Concatenate the new row with the predictions DataFrame\npredictions_df = pd.concat([predictions_df, row], ignore_index=True)\n\n# Step 6: Save predictions to a CSV file\npredictions_df.to_csv('submission.csv', index=False)\n!head submission.csv\n","metadata":{"execution":{"iopub.status.busy":"2024-03-23T18:11:48.854730Z","iopub.execute_input":"2024-03-23T18:11:48.855098Z","iopub.status.idle":"2024-03-23T18:11:49.083457Z","shell.execute_reply.started":"2024-03-23T18:11:48.855063Z","shell.execute_reply":"2024-03-23T18:11:49.082785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from IPython.display import FileLink\n\n# Create a download link for the file\n#FileLink('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-23T18:11:49.086494Z","iopub.execute_input":"2024-03-23T18:11:49.088512Z","iopub.status.idle":"2024-03-23T18:11:49.095530Z","shell.execute_reply.started":"2024-03-23T18:11:49.088475Z","shell.execute_reply":"2024-03-23T18:11:49.094776Z"},"trusted":true},"execution_count":null,"outputs":[]}]}