{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"colab":{"provenance":[]},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"3_jzt7rCE5PA","cell_type":"markdown","source":"# HMS - Harmful Brain Activity Classification","metadata":{"id":"3_jzt7rCE5PA"}},{"id":"ykx6tlP7E5PE","cell_type":"markdown","source":"### IMPORTING THE NECESSARY LIBRARIES","metadata":{"id":"ykx6tlP7E5PE"}},{"id":"a6684e18","cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:48.354806Z","iopub.status.busy":"2024-07-03T04:19:48.35446Z","iopub.status.idle":"2024-07-03T04:19:50.094163Z","shell.execute_reply":"2024-07-03T04:19:50.093118Z","shell.execute_reply.started":"2024-07-03T04:19:48.354778Z"},"id":"a6684e18","trusted":true},"outputs":[],"execution_count":null},{"id":"bd783d7e","cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.096803Z","iopub.status.busy":"2024-07-03T04:19:50.096309Z","iopub.status.idle":"2024-07-03T04:19:50.101355Z","shell.execute_reply":"2024-07-03T04:19:50.100381Z","shell.execute_reply.started":"2024-07-03T04:19:50.096767Z"},"id":"bd783d7e","trusted":true},"outputs":[],"execution_count":null},{"id":"Jeaws__qE5PH","cell_type":"markdown","source":"## LOADING THE DATASET","metadata":{"id":"Jeaws__qE5PH"}},{"id":"g9Oqhiz_E5PI","cell_type":"code","source":"BASE_DIR = \"/kaggle/input/hms-harmful-brain-activity-classification/\"","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.102976Z","iopub.status.busy":"2024-07-03T04:19:50.102634Z","iopub.status.idle":"2024-07-03T04:19:50.109867Z","shell.execute_reply":"2024-07-03T04:19:50.108902Z","shell.execute_reply.started":"2024-07-03T04:19:50.102942Z"},"trusted":true,"id":"g9Oqhiz_E5PI"},"outputs":[],"execution_count":null},{"id":"ab2bfbe1","cell_type":"code","source":"df = pd.read_csv(f\"{BASE_DIR}train.csv\")\ndf.head()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.111566Z","iopub.status.busy":"2024-07-03T04:19:50.111216Z","iopub.status.idle":"2024-07-03T04:19:50.3997Z","shell.execute_reply":"2024-07-03T04:19:50.398946Z","shell.execute_reply.started":"2024-07-03T04:19:50.111541Z"},"id":"ab2bfbe1","trusted":true,"outputId":"ec0175fe-7edd-4eb9-e651-ca98cef259eb"},"outputs":[],"execution_count":null},{"id":"yF_8pj33E5PJ","cell_type":"markdown","source":"### BASIC CHECKS","metadata":{"id":"yF_8pj33E5PJ"}},{"id":"c0c2ac52","cell_type":"code","source":"df.tail()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.402303Z","iopub.status.busy":"2024-07-03T04:19:50.401999Z","iopub.status.idle":"2024-07-03T04:19:50.41707Z","shell.execute_reply":"2024-07-03T04:19:50.416154Z","shell.execute_reply.started":"2024-07-03T04:19:50.402277Z"},"id":"c0c2ac52","outputId":"8d61aefa-a834-4c02-db27-cb90b28ed904","trusted":true},"outputs":[],"execution_count":null},{"id":"a5fb31fa","cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.418769Z","iopub.status.busy":"2024-07-03T04:19:50.418471Z","iopub.status.idle":"2024-07-03T04:19:50.425817Z","shell.execute_reply":"2024-07-03T04:19:50.424745Z","shell.execute_reply.started":"2024-07-03T04:19:50.418737Z"},"id":"a5fb31fa","outputId":"bcb8422f-8f44-4a4a-ac90-344f9c9793d4","trusted":true},"outputs":[],"execution_count":null},{"id":"3c203298","cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.427197Z","iopub.status.busy":"2024-07-03T04:19:50.426898Z","iopub.status.idle":"2024-07-03T04:19:50.435334Z","shell.execute_reply":"2024-07-03T04:19:50.434471Z","shell.execute_reply.started":"2024-07-03T04:19:50.427167Z"},"id":"3c203298","outputId":"87a8fd0e-f916-4f70-c72c-c8da658fccbe","trusted":true},"outputs":[],"execution_count":null},{"id":"o1tnVvgfE5PL","cell_type":"markdown","source":"## DATA ANALYSIS","metadata":{"id":"o1tnVvgfE5PL"}},{"id":"24fe4409","cell_type":"markdown","source":"1. **`eeg_id`**: Unique identifier for the entire EEG recording. It represents a specific EEG session or recording from a patient.\n\n2. **`eeg_sub_id`**: An ID for the specific 50-second long subsample to which the row's labels apply. This column helps identify a particular subset or segment within the larger EEG recording.\n\n3. **`eeg_label_offset_seconds`**: The time between the beginning of the consolidated EEG and the subsample. It indicates the time offset for the EEG subsample within the entire EEG recording.\n\n4. **`spectrogram_id`**: Unique identifier for the entire EEG recording, similar to `eeg_id`. It is related to the spectrogram data.\n\n5. **`spectrogram_sub_id`**: An ID for the specific 10-minute subsample to which the row's labels apply. This corresponds to a subset within the larger spectrogram data.\n\n6. **`spectrogram_label_offset_seconds`**: The time between the beginning of the consolidated spectrogram and the subsample. It indicates the time offset for the spectrogram subsample within the entire spectrogram recording.\n\n7. **`label_id`**: An ID for this set of labels. It helps distinguish different sets of labels within the dataset.\n\n8. **`patient_id`**: An ID for the patient who donated the data. It uniquely identifies each patient.\n\n9. **`expert_consensus`**: The consensus annotator label for convenience. This column may provide a summary or agreement among expert annotators regarding the type of brain activity in the given subsample.\n\n10. **`seizure_vote`**, **`lpd_vote`**, **`gpd_vote`**, **`lrda_vote`**, **`grda_vote`**, **`other_vote`**: These columns represent the count of annotator votes for specific brain activity classes. The classes are:\n    - `seizure_vote`: Count of votes for seizure.\n    - `lpd_vote`: Count of votes for lateralized periodic discharges.\n    - `gpd_vote`: Count of votes for generalized periodic discharges.\n    - `lrda_vote`: Count of votes for lateralized rhythmic delta activity.\n    - `grda_vote`: Count of votes for generalized rhythmic delta activity.\n    - `other_vote`: Count of votes for other types of brain activity.\n\n11. **Target Variable**: The target variable in this dataset is the actual brain activity class for each subsample. It could be any of the following:\n    - Seizure (`seizure_vote`): Represents the count of votes for seizure.\n    - Lateralized Periodic Discharges (`lpd_vote`): Represents the count of votes for lateralized periodic discharges.\n    - Generalized Periodic Discharges (`gpd_vote`): Represents the count of votes for generalized periodic discharges.\n    - Lateralized Rhythmic Delta Activity (`lrda_vote`): Represents the count of votes for lateralized rhythmic delta activity.\n    - Generalized Rhythmic Delta Activity (`grda_vote`): Represents the count of votes for generalized rhythmic delta activity.\n    - Other (`other_vote`): Represents the count of votes for other types of brain activity.\n\nThe target variable is the type of brain activity class (seizure, lpd, gpd, lrda, grda, other), and the goal of the competition or analysis is likely to predict or classify the correct brain activity class for each EEG subsample.","metadata":{"id":"24fe4409"}},{"id":"6a4d1266","cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.437089Z","iopub.status.busy":"2024-07-03T04:19:50.436786Z","iopub.status.idle":"2024-07-03T04:19:50.461365Z","shell.execute_reply":"2024-07-03T04:19:50.46033Z","shell.execute_reply.started":"2024-07-03T04:19:50.437057Z"},"id":"6a4d1266","outputId":"2897b253-e47a-4bdb-e90f-8dbeff3de890","trusted":true},"outputs":[],"execution_count":null},{"id":"16931f94","cell_type":"code","source":"df.nunique()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.462767Z","iopub.status.busy":"2024-07-03T04:19:50.462488Z","iopub.status.idle":"2024-07-03T04:19:50.497998Z","shell.execute_reply":"2024-07-03T04:19:50.497146Z","shell.execute_reply.started":"2024-07-03T04:19:50.462743Z"},"id":"16931f94","outputId":"45ab9b55-f749-422f-c6c7-dd0028bbf309","trusted":true},"outputs":[],"execution_count":null},{"id":"3dbfbdf0","cell_type":"code","source":"object_columns = df.select_dtypes(include=['object', 'bool']).columns\nprint(\"Object type columns:\")\nprint(object_columns)\n\nnumerical_columns = df.select_dtypes(include=['int64', 'float64']).columns\nprint(\"\\nNumerical type columns:\")\nprint(numerical_columns)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.499683Z","iopub.status.busy":"2024-07-03T04:19:50.499263Z","iopub.status.idle":"2024-07-03T04:19:50.50999Z","shell.execute_reply":"2024-07-03T04:19:50.509031Z","shell.execute_reply.started":"2024-07-03T04:19:50.499648Z"},"id":"3dbfbdf0","outputId":"e12bcadd-c5a8-4a1a-daf4-5c8a77c22dea","trusted":true},"outputs":[],"execution_count":null},{"id":"8ee6e8ce","cell_type":"code","source":"def classify_features(df):\n    categorical_features = []\n    non_categorical_features = []\n    discrete_features = []\n    continuous_features = []\n\n    for column in df.columns:\n        if df[column].dtype in ['object', 'bool']:\n            if df[column].nunique() < 15:\n                categorical_features.append(column)\n            else:\n                non_categorical_features.append(column)\n        elif df[column].dtype in ['int64', 'float64']:\n            if df[column].nunique() < 10:\n                discrete_features.append(column)\n            else:\n                continuous_features.append(column)\n\n    return categorical_features, non_categorical_features, discrete_features, continuous_features","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.511492Z","iopub.status.busy":"2024-07-03T04:19:50.511181Z","iopub.status.idle":"2024-07-03T04:19:50.518513Z","shell.execute_reply":"2024-07-03T04:19:50.517661Z","shell.execute_reply.started":"2024-07-03T04:19:50.511467Z"},"id":"8ee6e8ce","trusted":true},"outputs":[],"execution_count":null},{"id":"dc6734b4","cell_type":"code","source":"categorical, non_categorical, discrete, continuous = classify_features(df)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.519744Z","iopub.status.busy":"2024-07-03T04:19:50.519494Z","iopub.status.idle":"2024-07-03T04:19:50.549965Z","shell.execute_reply":"2024-07-03T04:19:50.549269Z","shell.execute_reply.started":"2024-07-03T04:19:50.519721Z"},"id":"dc6734b4","trusted":true},"outputs":[],"execution_count":null},{"id":"de33f77a","cell_type":"code","source":"print(\"Categorical Features:\", categorical)\nprint(\"Non-Categorical Features:\", non_categorical)\nprint(\"Discrete Features:\", discrete)\nprint(\"Continuous Features:\", continuous)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.551551Z","iopub.status.busy":"2024-07-03T04:19:50.551102Z","iopub.status.idle":"2024-07-03T04:19:50.557319Z","shell.execute_reply":"2024-07-03T04:19:50.556401Z","shell.execute_reply.started":"2024-07-03T04:19:50.551516Z"},"id":"de33f77a","outputId":"a29604e3-9f3a-4342-c9ed-6892e0ae0ecb","trusted":true},"outputs":[],"execution_count":null},{"id":"855580ed","cell_type":"code","source":"for i in categorical:\n    print(i, ':')\n    print(df[i].unique())\n    print()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.564323Z","iopub.status.busy":"2024-07-03T04:19:50.564041Z","iopub.status.idle":"2024-07-03T04:19:50.576116Z","shell.execute_reply":"2024-07-03T04:19:50.575187Z","shell.execute_reply.started":"2024-07-03T04:19:50.564301Z"},"id":"855580ed","outputId":"c0387edf-ad70-430e-eac3-98a6ba0ac49c","trusted":true},"outputs":[],"execution_count":null},{"id":"69615077","cell_type":"code","source":"for i in categorical:\n    print(i, ':')\n    print(df[i].value_counts())\n    print()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:50.577725Z","iopub.status.busy":"2024-07-03T04:19:50.577375Z","iopub.status.idle":"2024-07-03T04:19:50.60125Z","shell.execute_reply":"2024-07-03T04:19:50.600329Z","shell.execute_reply.started":"2024-07-03T04:19:50.577691Z"},"id":"69615077","outputId":"869ac4c1-f917-46ed-e036-74fb6c40bfd1","trusted":true},"outputs":[],"execution_count":null},{"id":"e1h0-WAwE5PR","cell_type":"markdown","source":"## EXPLORATORY DATA ANALYSIS","metadata":{"id":"e1h0-WAwE5PR"}},{"id":"1a36b1f0","cell_type":"code","source":"votes_columns = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\ndf_votes = df[votes_columns].melt(var_name='Brain Activity', value_name='Votes')","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.371837Z","iopub.status.busy":"2024-07-03T04:19:51.371486Z","iopub.status.idle":"2024-07-03T04:19:51.41009Z","shell.execute_reply":"2024-07-03T04:19:51.409322Z","shell.execute_reply.started":"2024-07-03T04:19:51.371809Z"},"id":"1a36b1f0","trusted":true},"outputs":[],"execution_count":null},{"id":"b9ac5e00","cell_type":"markdown","source":"1. **`votes_columns`**: This is a list containing the names of columns in your original DataFrame (`df`) that represent the counts of votes for different brain activity classes. Each element in the list corresponds to a specific brain activity class.\n\n2. **`df_votes`**: This line creates a new DataFrame (`df_votes`) by selecting only the columns specified in `votes_columns` from the original DataFrame (`df`). The selected columns are essentially the counts of votes for different brain activity classes.\n\n3. **`.melt(var_name='Brain Activity', value_name='Votes')`**: This part of the code transforms the DataFrame from wide format to long format using the `melt` function. It takes the selected columns (`votes_columns`) and \"melts\" or unpivots them, creating two new columns:\n   - `Brain Activity`: This column stores the variable names from the original DataFrame (`df`) that represent different brain activity classes (e.g., 'seizure_vote', 'lpd_vote', etc.).\n   - `Votes`: This column stores the corresponding values (vote counts) for each brain activity class.","metadata":{"id":"b9ac5e00"}},{"id":"02fb6640","cell_type":"code","source":"df_votes","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.411456Z","iopub.status.busy":"2024-07-03T04:19:51.411185Z","iopub.status.idle":"2024-07-03T04:19:51.424312Z","shell.execute_reply":"2024-07-03T04:19:51.42335Z","shell.execute_reply.started":"2024-07-03T04:19:51.411432Z"},"id":"02fb6640","outputId":"2d2a2ef8-bca4-4356-faac-6bc70453b862","trusted":true},"outputs":[],"execution_count":null},{"id":"f1db71b7","cell_type":"code","source":"pivot_table_continuous = pd.pivot_table(df, values=continuous, index='expert_consensus', aggfunc='mean', fill_value=0)\npivot_table_continuous","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.433767Z","iopub.status.busy":"2024-07-03T04:19:51.433416Z","iopub.status.idle":"2024-07-03T04:19:51.479693Z","shell.execute_reply":"2024-07-03T04:19:51.478703Z","shell.execute_reply.started":"2024-07-03T04:19:51.433728Z"},"id":"f1db71b7","outputId":"ff712249-e1be-4e04-94c9-4e69f291f0c4","trusted":true},"outputs":[],"execution_count":null},{"id":"63af5038","cell_type":"markdown","source":"1. **eeg_id:**\n   - GPD has an average EEG recording identifier of approximately 2.1126e+09.\n   - Other classes have varying average EEG recording identifiers.\n\n   **Interpretation:** The average EEG recording identifier may not provide direct insights into the characteristics of brain activity classes.\n\n2. **eeg_label_offset_seconds:**\n   - GPD has an average EEG label offset of approximately 271.79 seconds.\n   - Other classes have varying average EEG label offsets.\n\n   **Interpretation:** GPD tends to have EEG signals labeled around 271.79 seconds, suggesting a specific temporal pattern associated with this class.\n\n3. **spectrogram_id:**\n   - GPD has an average spectrogram identifier of approximately 1.01299e+09.\n   - Other classes have varying average spectrogram identifiers.\n\n   **Interpretation:** The average spectrogram identifier for GPD may indicate a specific pattern or set of characteristics in the spectrogram associated with this class.\n\n4. **seizure_vote:**\n   - Seizure has the highest average vote count, indicating a higher level of agreement among annotators for this class.\n   - Other classes have lower average vote counts.\n\n   **Interpretation:** Seizure tends to have a higher level of agreement among annotators, suggesting that it might be a more easily identifiable brain activity class compared to others.\n\nThese interpretations provide a high-level understanding of the average characteristics associated with different brain activity classes based on the provided features. It's important to note that further in-depth analysis and domain expertise may be required to draw more specific conclusions about the significance of these averages in the context of EEG signal processing and brain activity classification.","metadata":{"id":"63af5038"}},{"id":"0848ac82","cell_type":"code","source":"df['Brain Activity'] = df[votes_columns].idxmax(axis=1).apply(lambda x: x.replace('_vote', ''))","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.481264Z","iopub.status.busy":"2024-07-03T04:19:51.480907Z","iopub.status.idle":"2024-07-03T04:19:51.556654Z","shell.execute_reply":"2024-07-03T04:19:51.555745Z","shell.execute_reply.started":"2024-07-03T04:19:51.48122Z"},"id":"0848ac82","trusted":true},"outputs":[],"execution_count":null},{"id":"664598db","cell_type":"code","source":"grouped_continuous = df.groupby('eeg_id')[continuous].mean().reset_index(drop=True)\ngrouped_continuous","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.581489Z","iopub.status.busy":"2024-07-03T04:19:51.581144Z","iopub.status.idle":"2024-07-03T04:19:51.633965Z","shell.execute_reply":"2024-07-03T04:19:51.632961Z","shell.execute_reply.started":"2024-07-03T04:19:51.581461Z"},"id":"664598db","outputId":"c5c884f4-d62f-44df-c665-194756e8b155","trusted":true},"outputs":[],"execution_count":null},{"id":"bb435de0","cell_type":"code","source":"targets = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.635477Z","iopub.status.busy":"2024-07-03T04:19:51.635202Z","iopub.status.idle":"2024-07-03T04:19:51.642097Z","shell.execute_reply":"2024-07-03T04:19:51.641138Z","shell.execute_reply.started":"2024-07-03T04:19:51.635447Z"},"id":"bb435de0","trusted":true},"outputs":[],"execution_count":null},{"id":"b0d882f9","cell_type":"code","source":"total_votes_per_pat = df.groupby('patient_id')[targets].sum().sum(axis=1)\nnormalized_votes = df.groupby('patient_id')[targets].sum().div(total_votes_per_pat, axis=0)\nmean_vote_ratio = normalized_votes.mean()\nprint( mean_vote_ratio )","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.643514Z","iopub.status.busy":"2024-07-03T04:19:51.643117Z","iopub.status.idle":"2024-07-03T04:19:51.66472Z","shell.execute_reply":"2024-07-03T04:19:51.663873Z","shell.execute_reply.started":"2024-07-03T04:19:51.643476Z"},"id":"b0d882f9","outputId":"9a7dd695-7240-4202-a0b5-9724612ae1b0","trusted":true},"outputs":[],"execution_count":null},{"id":"60ea4718","cell_type":"code","source":"target_eeg_id = 1000913311\ndf_filtered = df[df['eeg_id'] == target_eeg_id]\ndf_filtered","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.666679Z","iopub.status.busy":"2024-07-03T04:19:51.665888Z","iopub.status.idle":"2024-07-03T04:19:51.682612Z","shell.execute_reply":"2024-07-03T04:19:51.681536Z","shell.execute_reply.started":"2024-07-03T04:19:51.666642Z"},"id":"60ea4718","outputId":"5b8b6e0a-3c15-4f41-b417-33e8b7b97653","trusted":true},"outputs":[],"execution_count":null},{"id":"b9cf22d8","cell_type":"code","source":"df_filtered = pd.read_parquet(f'{BASE_DIR}/train_eegs/1000913311.parquet')","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.684466Z","iopub.status.busy":"2024-07-03T04:19:51.684147Z","iopub.status.idle":"2024-07-03T04:19:51.850205Z","shell.execute_reply":"2024-07-03T04:19:51.849383Z","shell.execute_reply.started":"2024-07-03T04:19:51.68443Z"},"id":"b9cf22d8","scrolled":true,"trusted":true},"outputs":[],"execution_count":null},{"id":"bba88c2d","cell_type":"code","source":"df_filtered","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.851688Z","iopub.status.busy":"2024-07-03T04:19:51.851386Z","iopub.status.idle":"2024-07-03T04:19:51.88135Z","shell.execute_reply":"2024-07-03T04:19:51.880347Z","shell.execute_reply.started":"2024-07-03T04:19:51.851662Z"},"id":"bba88c2d","outputId":"321f66ac-a5f5-4b46-a328-8576ba2dd679","trusted":true},"outputs":[],"execution_count":null},{"id":"2941807e","cell_type":"markdown","source":"Data has a lot of peaks and troughs, indicating a high variation\nThe scale of most of the plots ranging from -200 to 100, excluding EKG chart\nThe line seems to fluctuate above and below a central value, which seems to be around zero value","metadata":{"id":"2941807e"}},{"id":"f3a46e3b","cell_type":"code","source":"df_filtered.shape","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.882764Z","iopub.status.busy":"2024-07-03T04:19:51.882462Z","iopub.status.idle":"2024-07-03T04:19:51.888689Z","shell.execute_reply":"2024-07-03T04:19:51.887643Z","shell.execute_reply.started":"2024-07-03T04:19:51.882737Z"},"id":"f3a46e3b","outputId":"c4c7097c-4eec-4e00-e8c7-73491fdaa36e","trusted":true},"outputs":[],"execution_count":null},{"id":"7263fcdc","cell_type":"code","source":"df_filtered.columns","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.890261Z","iopub.status.busy":"2024-07-03T04:19:51.889934Z","iopub.status.idle":"2024-07-03T04:19:51.897413Z","shell.execute_reply":"2024-07-03T04:19:51.896491Z","shell.execute_reply.started":"2024-07-03T04:19:51.890233Z"},"id":"7263fcdc","outputId":"f4a9b66d-a092-4c8a-f328-d326e0deaf52","trusted":true},"outputs":[],"execution_count":null},{"id":"7054d7b8","cell_type":"code","source":"electrodes = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2', 'EKG']","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.950155Z","iopub.status.busy":"2024-07-03T04:19:51.949476Z","iopub.status.idle":"2024-07-03T04:19:51.955016Z","shell.execute_reply":"2024-07-03T04:19:51.954138Z","shell.execute_reply.started":"2024-07-03T04:19:51.950104Z"},"id":"7054d7b8","trusted":true},"outputs":[],"execution_count":null},{"id":"408eb9f9","cell_type":"code","source":"unique_eeg_ids = df.drop_duplicates(subset='Brain Activity')['eeg_id'].tolist()\nprint(unique_eeg_ids)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.967223Z","iopub.status.busy":"2024-07-03T04:19:51.966891Z","iopub.status.idle":"2024-07-03T04:19:51.983178Z","shell.execute_reply":"2024-07-03T04:19:51.982145Z","shell.execute_reply.started":"2024-07-03T04:19:51.967194Z"},"id":"408eb9f9","outputId":"282f7989-027c-4b5e-c0ec-62d24c4c23bf","trusted":true},"outputs":[],"execution_count":null},{"id":"086559d2","cell_type":"code","source":"eeg_activity_dict = {}\n\nfor eeg_id in unique_eeg_ids:\n    subset_df = df[df['eeg_id'] == eeg_id]\n\n    unique_brain_activity = subset_df['Brain Activity'].unique()\n\n    if len(unique_brain_activity) == 1:\n        eeg_activity_dict[eeg_id] = unique_brain_activity[0]\n\nprint(eeg_activity_dict)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.984967Z","iopub.status.busy":"2024-07-03T04:19:51.984552Z","iopub.status.idle":"2024-07-03T04:19:51.997378Z","shell.execute_reply":"2024-07-03T04:19:51.996291Z","shell.execute_reply.started":"2024-07-03T04:19:51.984929Z"},"id":"086559d2","outputId":"0f3f1d28-a8f8-4875-8acd-b61fe9f0429c","trusted":true},"outputs":[],"execution_count":null},{"id":"69d43134","cell_type":"markdown","source":"1. `eeg_activity_dict = {}`: Initializes an empty dictionary to store EEG IDs and their corresponding brain activities.\n\n2. `for eeg_id in unique_eeg_ids:`: Iterates through each unique EEG ID.\n\n3. `subset_df = df[df['eeg_id'] == eeg_id]`: Creates a subset DataFrame (`subset_df`) containing only rows where the 'eeg_id' column matches the current `eeg_id`.\n\n4. `unique_brain_activity = subset_df['Brain Activity'].unique()`: Extracts unique values from the 'Brain Activity' column within the subset DataFrame.\n\n5. `if len(unique_brain_activity) == 1:`: Checks if there is only one unique brain activity in the subset.\n\n6. `eeg_activity_dict[eeg_id] = unique_brain_activity[0]`: If there is only one unique brain activity, it adds an entry to the dictionary with the EEG ID as the key and the unique brain activity as the value.\n\n7. Finally, the dictionary `eeg_activity_dict` contains mappings from unique EEG IDs to their corresponding unique brain activities.","metadata":{"id":"69d43134"}},{"id":"48aeb7d7","cell_type":"code","source":"print(\"Categorical Features:\", categorical)\nprint(\"Non-Categorical Features:\", non_categorical)\nprint(\"Discrete Features:\", discrete)\nprint(\"Continuous Features:\", continuous)","metadata":{"id":"48aeb7d7","outputId":"1fb66dd1-0dbe-4b09-e167-f2849f0249a6"},"outputs":[],"execution_count":null},{"id":"187b300e","cell_type":"markdown","source":"## FEATURE ENGINEERING","metadata":{"id":"187b300e"}},{"id":"f13563b6","cell_type":"code","source":"correlation_matrix = combined_df.corr()\n\nprint(\"Correlation Matrix:\")\ncorrelation_matrix","metadata":{"id":"f13563b6","outputId":"24be8b52-082d-4603-c7c1-d1a55b6c444d"},"outputs":[],"execution_count":null},{"id":"c076a129","cell_type":"code","source":"plt.figure(figsize=(15, 8))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=.5)\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"id":"c076a129","outputId":"1f355c51-253f-4a20-9205-73ace1d34618"},"outputs":[],"execution_count":null},{"id":"kZnexLMBE5P8","cell_type":"code","source":"EEGid_label_list = df[[\"eeg_id\", \"Brain Activity\", \"eeg_label_offset_seconds\"]].values.tolist()\nEEGid_label_list","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:51.999268Z","iopub.status.busy":"2024-07-03T04:19:51.998636Z","iopub.status.idle":"2024-07-03T04:19:52.190035Z","shell.execute_reply":"2024-07-03T04:19:52.189179Z","shell.execute_reply.started":"2024-07-03T04:19:51.999234Z"},"scrolled":true,"trusted":true,"id":"kZnexLMBE5P8","outputId":"3f1690ea-8c60-4d0f-faf3-511e19a29b07"},"outputs":[],"execution_count":null},{"id":"7_wYIR_UE5P8","cell_type":"markdown","source":"#### DEFINE X AND Y","metadata":{"id":"7_wYIR_UE5P8"}},{"id":"532afdf9","cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX = []\ny = []\nprev_eegId = \"\"\nbrain_activities = ['Seizure', 'GPD', 'LRDA', 'Other', 'GRDA', 'LPD']\nactivity_mapping = {activity.lower(): idx for idx, activity in enumerate(brain_activities)}\nfor x in EEGid_label_list[:20000]:\n    eeg_id, label, offset = x\n    if(eeg_id!=prev_eegId):\n        temp_df = pd.read_parquet(f'{BASE_DIR}train_eegs/{eeg_id}.parquet')\n#         C1 = ['Fp1', 'Fp2', 'F7', 'F8', 'T3', 'T4', 'T5', 'T6', 'Fp1', 'Fp2','F3', 'F4', 'C3', 'C4', 'P3', 'P4']\n\n        C = ['Fp1', 'T3', 'P4']\n        temp_arr = temp_df[C].to_numpy().T\n        temp_arr[np.isnan(temp_arr)] = 0\n        temp_arr = scaler.fit_transform(temp_arr)\n    start = 200*int(offset)\n    X.append(temp_arr[:, start:start+10000])\n    y.append(activity_mapping[label])\n    # print(eeg_id, activity_mapping[label], temp_arr.shape)\n    prev_eegId = eeg_id\nX = np.array(X)\ny = np.array(y)\nprint(\"X:\", X.shape)\nprint(\"y:\", y.shape)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:19:52.191412Z","iopub.status.busy":"2024-07-03T04:19:52.191114Z","iopub.status.idle":"2024-07-03T04:20:54.322759Z","shell.execute_reply":"2024-07-03T04:20:54.321737Z","shell.execute_reply.started":"2024-07-03T04:19:52.191387Z"},"id":"532afdf9","trusted":true,"outputId":"fc77a038-2434-4f09-b3d8-dc68696b3a2a"},"outputs":[],"execution_count":null},{"id":"jXkFzPaUE5P9","cell_type":"code","source":"# CNN will be used. Channel dimension is added.\nX = X[:, :, :, np.newaxis]\n\nX.shape","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:20:54.33017Z","iopub.status.busy":"2024-07-03T04:20:54.329305Z","iopub.status.idle":"2024-07-03T04:20:54.336242Z","shell.execute_reply":"2024-07-03T04:20:54.335208Z","shell.execute_reply.started":"2024-07-03T04:20:54.330108Z"},"trusted":true,"id":"jXkFzPaUE5P9","outputId":"c47e722f-c4c6-4553-8d2b-755dff9fca3c"},"outputs":[],"execution_count":null},{"id":"465754bf","cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report , confusion_matrix\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify = y, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, stratify = y_test, random_state=42)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:20:54.338034Z","iopub.status.busy":"2024-07-03T04:20:54.337666Z","iopub.status.idle":"2024-07-03T04:20:55.356712Z","shell.execute_reply":"2024-07-03T04:20:55.355596Z","shell.execute_reply.started":"2024-07-03T04:20:54.337978Z"},"id":"465754bf","trusted":true},"outputs":[],"execution_count":null},{"id":"aoiffAWxE5P-","cell_type":"code","source":"print(\"Train Set:\", X_train.shape)\nprint(\"Val Set:\", X_val.shape)\nprint(\"Test Set:\", X_test.shape)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:20:55.358458Z","iopub.status.busy":"2024-07-03T04:20:55.358051Z","iopub.status.idle":"2024-07-03T04:20:55.364396Z","shell.execute_reply":"2024-07-03T04:20:55.363298Z","shell.execute_reply.started":"2024-07-03T04:20:55.358424Z"},"trusted":true,"id":"aoiffAWxE5P-","outputId":"7ec6a047-86a5-45f9-8325-3bdeda0dd3a7"},"outputs":[],"execution_count":null},{"id":"QRpgik8IE5P-","cell_type":"markdown","source":"## MODEL CREATION","metadata":{"id":"QRpgik8IE5P-"}},{"id":"6f3e04eb","cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:20:55.366411Z","iopub.status.busy":"2024-07-03T04:20:55.366047Z","iopub.status.idle":"2024-07-03T04:21:07.570498Z","shell.execute_reply":"2024-07-03T04:21:07.569425Z","shell.execute_reply.started":"2024-07-03T04:20:55.366384Z"},"id":"6f3e04eb","trusted":true,"outputId":"1836521c-6f2d-4e59-fd69-3590d16dbd0a"},"outputs":[],"execution_count":null},{"id":"137aee49","cell_type":"code","source":"## deep learning model\n\nimport os\nsaved_model_path = '/kaggle/working/HMS_model4_v1.h5'\n\nif(os.path.isfile(saved_model_path)):\n    model = keras.models.load_model(saved_model_path)\n    print(\"Model Loaded :\", saved_model_path)\nelse:\n    model = keras.models.Sequential()\n\n    model.add(layers.Conv2D(filters=64, kernel_size=(2, 4), padding='same', activation='relu', input_shape=X_train.shape[1:]))\n    model.add(layers.Conv2D(filters=64, kernel_size=(2, 4), strides=(1, 2),padding='same', activation='relu'))\n    model.add(layers.MaxPooling2D((1, 2)))\n\n    model.add(layers.Conv2D(filters=128, kernel_size=(2, 4), padding='same', activation='relu'))\n    model.add(layers.Conv2D(filters=128, kernel_size=(2, 4), strides=(1, 2), padding='same', activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n\n    model.add(layers.Conv2D(filters=256, kernel_size=(4, 4), padding='same', activation='relu'))\n    model.add(layers.Conv2D(filters=256, kernel_size=(4, 4), strides=(1, 2), padding='same', activation='relu'))\n    model.add(layers.MaxPooling2D((1, 2)))\n\n    model.add(layers.GlobalAveragePooling2D())\n    #model.add(layers.Flatten())\n    model.add(layers.Dense(256, activation='relu'))\n    model.add(layers.Dropout(0.25))\n    model.add(layers.Dense(128, activation='relu'))\n    model.add(layers.Dense(64, activation='relu'))\n    model.add(layers.Dropout(0.25))\n    model.add(layers.Dense(6, activation='softmax'))","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:07.572617Z","iopub.status.busy":"2024-07-03T04:21:07.571866Z","iopub.status.idle":"2024-07-03T04:21:08.558391Z","shell.execute_reply":"2024-07-03T04:21:08.557545Z","shell.execute_reply.started":"2024-07-03T04:21:07.572586Z"},"id":"137aee49","trusted":true},"outputs":[],"execution_count":null},{"id":"619041d4","cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:08.559779Z","iopub.status.busy":"2024-07-03T04:21:08.559491Z","iopub.status.idle":"2024-07-03T04:21:08.59126Z","shell.execute_reply":"2024-07-03T04:21:08.590397Z","shell.execute_reply.started":"2024-07-03T04:21:08.559754Z"},"id":"619041d4","outputId":"89ce51a1-1fed-43e0-a8b0-4a4cf6d2dd65","scrolled":true,"trusted":true},"outputs":[],"execution_count":null},{"id":"P122vYlGE5P_","cell_type":"markdown","source":"### Setup for training\n### At least, **OPTIMIZER** and **LOSS** has to be set.","metadata":{"id":"P122vYlGE5P_"}},{"id":"815894c5","cell_type":"code","source":"LEARNING_RATE = 1e-5\nOPTIMIZER = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n\nmodel.compile(optimizer=OPTIMIZER, loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:08.592864Z","iopub.status.busy":"2024-07-03T04:21:08.592519Z","iopub.status.idle":"2024-07-03T04:21:08.607936Z","shell.execute_reply":"2024-07-03T04:21:08.607154Z","shell.execute_reply.started":"2024-07-03T04:21:08.592831Z"},"id":"815894c5","trusted":true},"outputs":[],"execution_count":null},{"id":"4d9f6635","cell_type":"code","source":"# callbacks\nVERBOSE=1\n#lr = ReduceLROnPlateau(monitor='val_loss', factor=0.75, patience=5, verbose=VERBOSE, min_le=1e-8)\nes = EarlyStopping(monitor='val_loss', patience=50, verbose=VERBOSE, mode='auto', restore_best_weights=True)\n\ncallbacks = [es]","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:08.609187Z","iopub.status.busy":"2024-07-03T04:21:08.608903Z","iopub.status.idle":"2024-07-03T04:21:08.614063Z","shell.execute_reply":"2024-07-03T04:21:08.613189Z","shell.execute_reply.started":"2024-07-03T04:21:08.609162Z"},"id":"4d9f6635","trusted":true},"outputs":[],"execution_count":null},{"id":"AjbA1NBME5QB","cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\ny_train_one_hot = to_categorical(y_train)\ny_val_one_hot = to_categorical(y_val)\ny_test_one_hot = to_categorical(y_test)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:08.615706Z","iopub.status.busy":"2024-07-03T04:21:08.615368Z","iopub.status.idle":"2024-07-03T04:21:08.6249Z","shell.execute_reply":"2024-07-03T04:21:08.623571Z","shell.execute_reply.started":"2024-07-03T04:21:08.615674Z"},"trusted":true,"id":"AjbA1NBME5QB"},"outputs":[],"execution_count":null},{"id":"f90fe7fb","cell_type":"code","source":"hist = model.fit(\n    x=X_train, y=y_train_one_hot,\n    validation_data=(X_val, y_val_one_hot),\n    epochs=200,\n    batch_size=64,\n    callbacks=callbacks\n)","metadata":{"execution":{"iopub.execute_input":"2024-07-03T04:21:08.6265Z","iopub.status.busy":"2024-07-03T04:21:08.626178Z","iopub.status.idle":"2024-07-03T09:16:44.030039Z","shell.execute_reply":"2024-07-03T09:16:44.029069Z","shell.execute_reply.started":"2024-07-03T04:21:08.62647Z"},"id":"f90fe7fb","trusted":true,"outputId":"37e3465f-f33f-48f1-ff8e-d50253554c74"},"outputs":[],"execution_count":null},{"id":"QJ9hBfUvE5QF","cell_type":"code","source":"model.save('/kaggle/working/HMS_model4_v2.h5')","metadata":{"execution":{"iopub.execute_input":"2024-07-03T09:16:44.037316Z","iopub.status.busy":"2024-07-03T09:16:44.037002Z","iopub.status.idle":"2024-07-03T09:16:44.130077Z","shell.execute_reply":"2024-07-03T09:16:44.129316Z","shell.execute_reply.started":"2024-07-03T09:16:44.03729Z"},"trusted":true,"id":"QJ9hBfUvE5QF"},"outputs":[],"execution_count":null},{"id":"b2c1f206","cell_type":"code","source":"plt.figure(figsize=(12, 6))\n\nplt.subplot(1, 2, 1)\nplt.plot(hist.history['accuracy'])\nplt.plot(hist.history['val_accuracy'])\nplt.title('Model accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(['Train', 'Validation'], loc='upper left')\n\nplt.subplot(1, 2, 2)\nplt.plot(hist.history['loss'])\nplt.plot(hist.history['val_loss'])\nplt.title('Model loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend(['Train', 'Validation'], loc='upper left')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-07-03T09:16:56.296002Z","iopub.status.busy":"2024-07-03T09:16:56.295352Z","iopub.status.idle":"2024-07-03T09:16:56.918171Z","shell.execute_reply":"2024-07-03T09:16:56.917266Z","shell.execute_reply.started":"2024-07-03T09:16:56.295959Z"},"id":"b2c1f206","outputId":"a27c16af-ca6c-4d13-8520-b991a754f7ce","trusted":true},"outputs":[],"execution_count":null},{"id":"f50fb5cb","cell_type":"code","source":"test_loss, test_accuracy = model.evaluate(X_test, y_test_one_hot)\nprint(\"Test Accuracy: {:.2f}%\".format(test_accuracy * 100))","metadata":{"id":"f50fb5cb","outputId":"51ea4276-3417-4492-958f-3edc34077715"},"outputs":[],"execution_count":null}]}