{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:# Group by patient_id and summarize the votes\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-22T12:58:54.193826Z","iopub.execute_input":"2024-01-22T12:58:54.194224Z","iopub.status.idle":"2024-01-22T12:58:54.688193Z","shell.execute_reply.started":"2024-01-22T12:58:54.194190Z","shell.execute_reply":"2024-01-22T12:58:54.686760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler\nimport numpy as np\nfrom scipy.io import loadmat\nimport os\nimport dask.dataframe as dd\nfrom dask.multiprocessing import get\nimport warnings\nwarnings.filterwarnings('ignore', category=Warning)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:54.690596Z","iopub.execute_input":"2024-01-22T12:58:54.691131Z","iopub.status.idle":"2024-01-22T12:58:57.638592Z","shell.execute_reply.started":"2024-01-22T12:58:54.691096Z","shell.execute_reply":"2024-01-22T12:58:57.637573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/hms-harmful-brain-activity-classification/train.csv')\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:57.640612Z","iopub.execute_input":"2024-01-22T12:58:57.641582Z","iopub.status.idle":"2024-01-22T12:58:58.066629Z","shell.execute_reply.started":"2024-01-22T12:58:57.641527Z","shell.execute_reply":"2024-01-22T12:58:58.064976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:58.070518Z","iopub.execute_input":"2024-01-22T12:58:58.071141Z","iopub.status.idle":"2024-01-22T12:58:58.094910Z","shell.execute_reply.started":"2024-01-22T12:58:58.071087Z","shell.execute_reply":"2024-01-22T12:58:58.093592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_cols = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\ntrain_df[label_cols].sum().plot(kind='bar')\nplt.title('Distribution of Annotator Votes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:58.096583Z","iopub.execute_input":"2024-01-22T12:58:58.096985Z","iopub.status.idle":"2024-01-22T12:58:58.487348Z","shell.execute_reply.started":"2024-01-22T12:58:58.096949Z","shell.execute_reply":"2024-01-22T12:58:58.485884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['expert_consensus'].value_counts().plot(kind='bar')\nplt.title('Distribution of Expert Consensus Labels')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:58.488914Z","iopub.execute_input":"2024-01-22T12:58:58.489319Z","iopub.status.idle":"2024-01-22T12:58:58.840052Z","shell.execute_reply.started":"2024-01-22T12:58:58.489284Z","shell.execute_reply":"2024-01-22T12:58:58.838621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote_columns = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n\nsns.boxplot(data=train_df[vote_columns])\nplt.title('Boxplot of Annotator Votes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:58.842678Z","iopub.execute_input":"2024-01-22T12:58:58.843578Z","iopub.status.idle":"2024-01-22T12:58:59.455666Z","shell.execute_reply.started":"2024-01-22T12:58:58.843527Z","shell.execute_reply":"2024-01-22T12:58:59.454406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pairwise comparison of votes to see if there are any patterns\nsns.pairplot(train_df[vote_columns])\nplt.title('Pairwise Distribution of Votes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:58:59.457560Z","iopub.execute_input":"2024-01-22T12:58:59.457986Z","iopub.status.idle":"2024-01-22T12:59:23.688541Z","shell.execute_reply.started":"2024-01-22T12:58:59.457948Z","shell.execute_reply":"2024-01-22T12:59:23.687096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = train_df[vote_columns].corr()\nsns.heatmap(corr, annot=True, cmap='coolwarm')\nplt.title('Correlation Matrix of Annotator Votes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:59:23.690359Z","iopub.execute_input":"2024-01-22T12:59:23.691904Z","iopub.status.idle":"2024-01-22T12:59:24.197061Z","shell.execute_reply.started":"2024-01-22T12:59:23.691840Z","shell.execute_reply":"2024-01-22T12:59:24.195876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_group = train_df.groupby('patient_id')[vote_columns].sum()\n\npatient_group.plot(kind='bar', stacked=True)\nplt.title('Sum of Votes per Patient')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:59:24.203160Z","iopub.execute_input":"2024-01-22T12:59:24.204186Z","iopub.status.idle":"2024-01-22T13:00:04.279052Z","shell.execute_reply.started":"2024-01-22T12:59:24.204149Z","shell.execute_reply":"2024-01-22T13:00:04.277297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['expert_consensus'].value_counts().plot(kind='pie', autopct='%1.1f%%')\nplt.title('Class Distribution in Expert Consensus')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:00:04.281017Z","iopub.execute_input":"2024-01-22T13:00:04.281588Z","iopub.status.idle":"2024-01-22T13:00:04.512037Z","shell.execute_reply.started":"2024-01-22T13:00:04.281533Z","shell.execute_reply":"2024-01-22T13:00:04.509991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_data = pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet')\neeg_data","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:00:04.515069Z","iopub.execute_input":"2024-01-22T13:00:04.516236Z","iopub.status.idle":"2024-01-22T13:00:04.756354Z","shell.execute_reply.started":"2024-01-22T13:00:04.516162Z","shell.execute_reply":"2024-01-22T13:00:04.755031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\neeg_data_scaled = pd.DataFrame(scaler.fit_transform(eeg_data), columns=eeg_data.columns)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:00:04.758582Z","iopub.execute_input":"2024-01-22T13:00:04.759476Z","iopub.status.idle":"2024-01-22T13:00:04.777141Z","shell.execute_reply.started":"2024-01-22T13:00:04.759430Z","shell.execute_reply":"2024-01-22T13:00:04.775509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_eeg = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2', 'EKG']\n\nplt.figure(figsize=(15, 10))\nfor i, feature in enumerate(features_eeg):\n    plt.subplot(len(features_eeg), 1, i+1)\n    plt.plot(eeg_data[feature])\n    plt.title(feature, loc='left')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:00:04.779036Z","iopub.execute_input":"2024-01-22T13:00:04.779565Z","iopub.status.idle":"2024-01-22T13:00:09.611141Z","shell.execute_reply.started":"2024-01-22T13:00:04.779516Z","shell.execute_reply":"2024-01-22T13:00:09.609479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_eeg_segment(eeg_data, offset_seconds, duration, sampling_rate):\n    start_sample = int(offset_seconds * sampling_rate)\n    end_sample = start_sample + int(duration * sampling_rate)\n    return eeg_data.iloc[start_sample:end_sample]\n\nsampling_rate = 200  \nsegment_duration = 1\n\n# Extracting EEG segments and aligning them with labels\naligned_eeg_data = []\nfor index, row in train_df.iterrows():\n    eeg_segment = extract_eeg_segment(eeg_data_scaled, row['eeg_label_offset_seconds'], segment_duration, sampling_rate)\n    aligned_eeg_data.append(eeg_segment.mean(axis=0))\n\nfeatures_df = pd.DataFrame(aligned_eeg_data)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:00:09.613431Z","iopub.execute_input":"2024-01-22T13:00:09.613893Z","iopub.status.idle":"2024-01-22T13:01:12.002511Z","shell.execute_reply.started":"2024-01-22T13:00:09.613850Z","shell.execute_reply":"2024-01-22T13:01:12.000748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.multioutput import MultiOutputClassifier\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(features_df, train_df[label_cols], test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:01:12.004508Z","iopub.execute_input":"2024-01-22T13:01:12.004942Z","iopub.status.idle":"2024-01-22T13:01:12.410706Z","shell.execute_reply.started":"2024-01-22T13:01:12.004906Z","shell.execute_reply":"2024-01-22T13:01:12.409025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer\n\nimputer = SimpleImputer(strategy='mean') \nX_train_imputed = imputer.fit_transform(X_train)\nX_val_imputed = imputer.transform(X_val)\n\nmodel = MultiOutputClassifier(LogisticRegression(max_iter=1000))\nmodel.fit(X_train_imputed, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:01:12.412412Z","iopub.execute_input":"2024-01-22T13:01:12.412821Z","iopub.status.idle":"2024-01-22T13:04:15.181666Z","shell.execute_reply.started":"2024-01-22T13:01:12.412770Z","shell.execute_reply":"2024-01-22T13:04:15.180044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict_proba(X_val_imputed)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.184742Z","iopub.execute_input":"2024-01-22T13:04:15.196466Z","iopub.status.idle":"2024-01-22T13:04:15.348172Z","shell.execute_reply.started":"2024-01-22T13:04:15.196342Z","shell.execute_reply":"2024-01-22T13:04:15.346529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import entropy\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nflat_predictions = np.hstack([pred[:, 1].reshape(-1, 1) for pred in predictions])\n\n# Convert y_val to binary format\nmlb = MultiLabelBinarizer()\nactual_binary = mlb.fit_transform(y_val.apply(lambda x: [i for i, val in enumerate(x) if val], axis=1))\n\n# Adjusted KL Divergence Calculation\ndef calculate_kl_divergence(predicted, actual):\n    kl_divergence = 0\n    for pred, act in zip(predicted.T, actual.T):  # Transpose to iterate over each label\n        # Adding a small value to predicted probabilities to avoid log(0) error\n        pred = np.clip(pred, 1e-15, 1 - 1e-15)\n        kl_divergence += entropy(act, pred)\n    return kl_divergence / actual.shape[1]\n\nkl_divergence_score = calculate_kl_divergence(flat_predictions, actual_binary)\n\nprint(f\"KL Divergence on validation set: {kl_divergence_score}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.358123Z","iopub.execute_input":"2024-01-22T13:04:15.363978Z","iopub.status.idle":"2024-01-22T13:04:15.717625Z","shell.execute_reply.started":"2024-01-22T13:04:15.363887Z","shell.execute_reply":"2024-01-22T13:04:15.716170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/hms-harmful-brain-activity-classification/test.csv')\n\nsubmission_data = []\n\nfor eeg_id in test_df['eeg_id'].unique():\n    eeg_test_data = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/{eeg_id}.parquet')\n    eeg_test_data_scaled = pd.DataFrame(scaler.transform(eeg_test_data[features_eeg]), columns=features_eeg)\n\n    test_pred = model.predict_proba(eeg_test_data_scaled)\n    test_flat_pred = np.hstack([pred[:, 1].reshape(-1, 1) for pred in test_pred])\n    \n    test_flat_pred /= np.sum(test_flat_pred, axis=1, keepdims=True)\n\n    eeg_pred_aggregated = np.mean(test_flat_pred, axis=0)\n\n    submission_data.append({'eeg_id': eeg_id, **dict(zip(label_cols, eeg_pred_aggregated))})\n\nsubmission_df = pd.DataFrame(submission_data, columns=['eeg_id'] + label_cols)\nsubmission_df.apply(lambda x: x/x.sum(), axis=1)\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.719425Z","iopub.execute_input":"2024-01-22T13:04:15.720183Z","iopub.status.idle":"2024-01-22T13:04:15.869538Z","shell.execute_reply.started":"2024-01-22T13:04:15.720137Z","shell.execute_reply":"2024-01-22T13:04:15.868081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sampling_rate = 200\n\n# aligned_eeg = []\n\n# # Loop through each row in train_df\n# for index, row in train_df.iterrows():\n#     start_sample = int(row['eeg_label_offset_seconds'] * sampling_rate)\n#     end_sample = start_sample + sampling_rate  # Adjust the window size as needed\n    \n#     # Aggregate the EEG data within this window\n#     aggregated_data = eeg_data_scaled.iloc[start_sample:end_sample].mean()\n#     aligned_eeg.append(aggregated_data)\n\n# # Convert the list to DataFrame\n# aligned_eeg_df = pd.DataFrame(aligned_eeg)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.871822Z","iopub.execute_input":"2024-01-22T13:04:15.872756Z","iopub.status.idle":"2024-01-22T13:04:15.881890Z","shell.execute_reply.started":"2024-01-22T13:04:15.872687Z","shell.execute_reply":"2024-01-22T13:04:15.879895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(aligned_eeg_df.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.883818Z","iopub.execute_input":"2024-01-22T13:04:15.884320Z","iopub.status.idle":"2024-01-22T13:04:15.895899Z","shell.execute_reply.started":"2024-01-22T13:04:15.884272Z","shell.execute_reply":"2024-01-22T13:04:15.894324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# y = train_df[label_cols]\n\n# X_train, X_test, y_train, y_test = train_test_split(aligned_eeg_df, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.898601Z","iopub.execute_input":"2024-01-22T13:04:15.900141Z","iopub.status.idle":"2024-01-22T13:04:15.910039Z","shell.execute_reply.started":"2024-01-22T13:04:15.899857Z","shell.execute_reply":"2024-01-22T13:04:15.908100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.linear_model import LogisticRegression\n# from sklearn.multiclass import OneVsRestClassifier\n\n# imputer = SimpleImputer(strategy='mean')\n# X_train_imputed = imputer.fit_transform(X_train)\n# X_test_imputed = imputer.transform(X_test)\n\n# models = {}\n# for label in label_cols:\n#     clf = LogisticRegression()\n#     clf.fit(X_train_imputed, y_train[label])\n#     models[label] = clf","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.913100Z","iopub.execute_input":"2024-01-22T13:04:15.913696Z","iopub.status.idle":"2024-01-22T13:04:15.925311Z","shell.execute_reply.started":"2024-01-22T13:04:15.913635Z","shell.execute_reply":"2024-01-22T13:04:15.923504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test = pd.read_csv('../input/hms-harmful-brain-activity-classification/test.csv')\n\n# for label in label_cols:\n#     # Predict probabilities for each label\n#     label_prob = models[label].predict_proba(df_test)[:, 1]\n    \n#     # Check if the number of predictions matches the number of eeg_id in submission\n#     if len(label_prob) == len(submission['eeg_id']):\n#         submission[label] = label_prob\n#     else:\n#         raise ValueError(\"Mismatch in the number of predictions and number of eeg_ids\")","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.927978Z","iopub.execute_input":"2024-01-22T13:04:15.928715Z","iopub.status.idle":"2024-01-22T13:04:15.943214Z","shell.execute_reply.started":"2024-01-22T13:04:15.928534Z","shell.execute_reply":"2024-01-22T13:04:15.941678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df = pd.DataFrame(prob_predictions, columns=label_cols)\n# submission_df['eeg_id'] = train_df['eeg_id']  # Add 'eeg_id' column\n\n# # Reorder columns to match the desired format\n# submission_df = submission_df[['eeg_id'] + label_cols]\n# submission_df\n# # Calculate the sum of probabilities for each row and normalize them\n# row_sums = submission_df[label_cols].sum(axis=1)\n# submission_df[label_cols] = submission_df[label_cols].div(row_sums, axis=0)\n\n# # Save the submission DataFrame to a CSV file\n# submission_df.to_csv('submission.csv', index=False)\n\n# # Display the modified submission DataFrame\n# print(submission_df.head())","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:04:15.945577Z","iopub.execute_input":"2024-01-22T13:04:15.946537Z","iopub.status.idle":"2024-01-22T13:04:15.957766Z","shell.execute_reply.started":"2024-01-22T13:04:15.946478Z","shell.execute_reply":"2024-01-22T13:04:15.956233Z"},"trusted":true},"execution_count":null,"outputs":[]}]}