{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7460364,"sourceType":"datasetVersion","datasetId":4342156},{"sourceId":7470029,"sourceType":"datasetVersion","datasetId":4348549},{"sourceId":7471171,"sourceType":"datasetVersion","datasetId":4349405},{"sourceId":7471726,"sourceType":"datasetVersion","datasetId":4349790}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import os,gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\n\n# 禁用所有警告\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:36.864738Z","iopub.execute_input":"2024-01-27T12:31:36.865176Z","iopub.status.idle":"2024-01-27T12:31:37.803924Z","shell.execute_reply.started":"2024-01-27T12:31:36.865149Z","shell.execute_reply":"2024-01-27T12:31:37.802990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data","metadata":{}},{"cell_type":"code","source":"PATH = '/kaggle/input/hms-harmful-brain-activity-classification/'\ndf = pd.read_csv(PATH + 'train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-27T12:31:37.805805Z","iopub.execute_input":"2024-01-27T12:31:37.806304Z","iopub.status.idle":"2024-01-27T12:31:38.108146Z","shell.execute_reply.started":"2024-01-27T12:31:37.806269Z","shell.execute_reply":"2024-01-27T12:31:38.107245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Non-Overlapping Eeg Id Train Data","metadata":{}},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spectrogram_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:38.109682Z","iopub.execute_input":"2024-01-27T12:31:38.110341Z","iopub.status.idle":"2024-01-27T12:31:38.203901Z","shell.execute_reply.started":"2024-01-27T12:31:38.110308Z","shell.execute_reply":"2024-01-27T12:31:38.202884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EEG_PATH = PATH + 'train_eegs/'\n\nrow = train.iloc[1]\neeg_id = row['eeg_id']\n\n# 读取 sample EEG 数据\neeg_file_path = os.path.join(EEG_PATH, f'{eeg_id}.parquet')\nsample_eeg_data = pd.read_parquet(eeg_file_path)\nsample_eeg_data = sample_eeg_data.iloc[:, :-1]\nsample_eeg_data = sample_eeg_data.iloc[:, [col for col in range(sample_eeg_data.shape[1]) if col not in [8, 9, 10]]]\n# 计算起始时间点\nstart_time_point = int((sample_eeg_data.shape[0] - 10_000) // 2)\n\n# 获取中间50秒的时间点数据\neeg_slice = sample_eeg_data.iloc[start_time_point : start_time_point + 10_000, :]\n\neeg_slice","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:38.206766Z","iopub.execute_input":"2024-01-27T12:31:38.207055Z","iopub.status.idle":"2024-01-27T12:31:38.434650Z","shell.execute_reply.started":"2024-01-27T12:31:38.207030Z","shell.execute_reply":"2024-01-27T12:31:38.433629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FLAG = False","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:38.436000Z","iopub.execute_input":"2024-01-27T12:31:38.436343Z","iopub.status.idle":"2024-01-27T12:31:38.441401Z","shell.execute_reply.started":"2024-01-27T12:31:38.436312Z","shell.execute_reply":"2024-01-27T12:31:38.440339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if FLAG:\n    fs = 200\n    EEG_PATH = PATH + 'train_eegs/'\n\n    # 初始化一个数组来保存所有处理后的 EEG 数据\n    all_eeg_data = np.zeros((len(train), 10000, 19))\n\n    # 遍历每个 eeg_id\n    for row_idx in range(len(train)):\n        row = train.iloc[row_idx]\n        eeg_id = row['eeg_id']\n\n        # 读取 EEG 数据\n        eeg_file_path = os.path.join(EEG_PATH, f'{eeg_id}.parquet')\n        eeg_data = pd.read_parquet(eeg_file_path)\n        eeg_data = eeg_data.iloc[:, :-1]\n        #eeg_data = eeg_data.iloc[:, [col for col in range(eeg_data.shape[1]) if col not in [8, 9, 10]]]\n        # 计算起始时间点\n        start_time_point = int((eeg_data.shape[0] - 10_000) // 2)\n\n        # 获取中间50秒的时间点数据\n        eeg_slice = eeg_data.iloc[start_time_point : start_time_point + 10_000, :]\n\n        # 更新 all_eeg_data\n        all_eeg_data[row_idx, :, :] = eeg_slice\n\n    # 输出数组的形状\n    print(\"Shape of all_eeg_data:\", all_eeg_data.shape)\nelse:\n    print(\"FLAG is set to False. The code below is not executed.\")","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:38.442937Z","iopub.execute_input":"2024-01-27T12:31:38.443869Z","iopub.status.idle":"2024-01-27T12:31:38.453398Z","shell.execute_reply.started":"2024-01-27T12:31:38.443837Z","shell.execute_reply":"2024-01-27T12:31:38.452594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{}},{"cell_type":"code","source":"import scipy.stats\n\ndef extract_eeg_features(eeg_data):\n    \"\"\"\n    Extract features from EEG data, including LL Spec, LP Spec, RP Spec, RL Spec\n    \n    Parameters:\n    - eeg_data: EEG data with shape (number of samples, time points, number of electrodes)\n    \n    Returns:\n    - Feature matrix with shape (number of samples, number of features)\n    \"\"\"\n    # Initialize the feature matrix\n    num_samples, num_time_points, num_electrodes = eeg_data.shape\n    num_features = 8 * num_electrodes  # Mean, standard deviation, minimum, maximum, skewness, kurtosis, energy, entropy for each electrode + 4 global features\n    features = np.zeros((num_samples, num_features))\n\n    # Extract features\n    for sample_idx in range(num_samples):\n        for electrode_idx in range(num_electrodes):\n            electrode_data = eeg_data[sample_idx, :, electrode_idx]\n            feature_idx = electrode_idx * 8\n\n            # Mean\n            features[sample_idx, feature_idx] = np.mean(electrode_data)\n            # Standard deviation\n            features[sample_idx, feature_idx + 1] = np.std(electrode_data)\n            # Minimum\n            features[sample_idx, feature_idx + 2] = np.min(electrode_data)\n            # Maximum\n            features[sample_idx, feature_idx + 3] = np.max(electrode_data)\n            # Skewness\n            features[sample_idx, feature_idx + 4] = scipy.stats.skew(electrode_data)\n            # Kurtosis\n            features[sample_idx, feature_idx + 5] = scipy.stats.kurtosis(electrode_data)\n            # Energy\n            features[sample_idx, feature_idx + 6] = np.sum(electrode_data**2) / len(electrode_data)\n            # Entropy\n            features[sample_idx, feature_idx + 7] = scipy.stats.entropy(np.abs(electrode_data))\n        \"\"\"\n        # Additional features: LL Spec, LP Spec, RP Spec, RL Spec\n        ll_spec = (np.abs(np.fft.fft(eeg_data[sample_idx, :, 0] - eeg_data[sample_idx, :, 4]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 4] - eeg_data[sample_idx, :, 5]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 5] - eeg_data[sample_idx, :, 6]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 6] - eeg_data[sample_idx, :, 7]))**2) / 4\n        features[sample_idx, -4] = np.mean(ll_spec)\n\n        lp_spec = (np.abs(np.fft.fft(eeg_data[sample_idx, :, 0] - eeg_data[sample_idx, :, 1]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 1] - eeg_data[sample_idx, :, 2]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 2] - eeg_data[sample_idx, :, 3]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 3] - eeg_data[sample_idx, :, 7]))**2) / 4\n        features[sample_idx, -3] = np.mean(lp_spec)\n\n        rp_spec = (np.abs(np.fft.fft(eeg_data[sample_idx, :, 8] - eeg_data[sample_idx, :, 9]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 9] - eeg_data[sample_idx, :, 10]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 10] - eeg_data[sample_idx, :, 11]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 11] - eeg_data[sample_idx, :, 15]))**2) / 4\n        features[sample_idx, -2] = np.mean(rp_spec)\n\n        rl_spec = (np.abs(np.fft.fft(eeg_data[sample_idx, :, 8] - eeg_data[sample_idx, :, 12]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 12] - eeg_data[sample_idx, :, 13]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 13] - eeg_data[sample_idx, :, 14]))**2 +\n                   np.abs(np.fft.fft(eeg_data[sample_idx, :, 14] - eeg_data[sample_idx, :, 15]))**2) / 4\n        features[sample_idx, -1] = np.mean(rl_spec)\n        \"\"\"\n    return features","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:38.454797Z","iopub.execute_input":"2024-01-27T12:31:38.455452Z","iopub.status.idle":"2024-01-27T12:31:39.136463Z","shell.execute_reply.started":"2024-01-27T12:31:38.455420Z","shell.execute_reply":"2024-01-27T12:31:39.135419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport scipy\n\nfrom scipy.stats import skew, kurtosis\nfrom scipy.stats.mstats import moment\nfrom sklearn.preprocessing import StandardScaler\n\nif FLAG:\n\n    # 使用函数提取特征\n    extracted_features = extract_eeg_features(all_eeg_data)\n\n    # 输出特征矩阵的形状\n    print(\"Shape of extracted features:\", extracted_features.shape)\nelse:\n    print(\"FLAG is set to False. The code below is not executed.\")","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:39.137646Z","iopub.execute_input":"2024-01-27T12:31:39.138001Z","iopub.status.idle":"2024-01-27T12:31:39.264604Z","shell.execute_reply.started":"2024-01-27T12:31:39.137966Z","shell.execute_reply":"2024-01-27T12:31:39.263675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if FLAG:\n    # 计算每列的平均值\n    mean_values = np.nanmean(extracted_features, axis=0)\n\n    # 找到 NaN 值的位置\n    nan_indices = np.isnan(extracted_features)\n\n    # 用平均值替代 NaN 值\n    extracted_features[nan_indices] = np.take(mean_values, nan_indices.nonzero()[1])\n\n    # 输出替代 NaN 后的特征形状\n    print(\"Shape of eeg_features after replacing NaN values:\", extracted_features.shape)\n    \n    # 保存提取的特征到文件\n    save_path = '/kaggle/working/extracted_eeg_features.npy'\n    np.save(save_path, extracted_features)\n    print(f\"Extracted EEG features saved at: {save_path}\")\nelse:\n    # 当FLAG为False时，加载之前保存的提取好的EEG特征\n    extracted_features = np.load('/kaggle/input/8-basic-feaatures-with-eeg/extracted_eeg_features (1).npy')\n\n    # 输出数组的形状\n    print(\"Shape of extracted features:\", extracted_features.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:39.265967Z","iopub.execute_input":"2024-01-27T12:31:39.266511Z","iopub.status.idle":"2024-01-27T12:31:39.563319Z","shell.execute_reply.started":"2024-01-27T12:31:39.266476Z","shell.execute_reply":"2024-01-27T12:31:39.562186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label","metadata":{}},{"cell_type":"code","source":"train_copy = train.copy()\nycol = [c for c in train_copy.columns if c in ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']]\ncd = {'Seizure':'seizure_vote', 'GPD':'gpd_vote', 'LRDA':'lrda_vote', 'Other':'other_vote', 'GRDA':'grda_vote', 'LPD':'lpd_vote'}\ntrain_copy['target'] = train_copy['target'].map(cd)\nfor i in range(len(train_copy)):\n    c = train_copy['target'][i]\n    train_copy[c][i] = train_copy[c][i]+10 #adding weight to expert consensus\n\nysum = train_copy[ycol].sum(axis=1) \nfor c in ycol:\n    train_copy[c] = (train_copy[c] / ysum).astype(np.float64)","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:39.567281Z","iopub.execute_input":"2024-01-27T12:31:39.567902Z","iopub.status.idle":"2024-01-27T12:31:44.747808Z","shell.execute_reply.started":"2024-01-27T12:31:39.567865Z","shell.execute_reply":"2024-01-27T12:31:44.746658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = train_copy[ycol]\nlabel.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:44.751798Z","iopub.execute_input":"2024-01-27T12:31:44.752197Z","iopub.status.idle":"2024-01-27T12:31:44.763461Z","shell.execute_reply.started":"2024-01-27T12:31:44.752157Z","shell.execute_reply":"2024-01-27T12:31:44.762176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom xgboost import XGBClassifier\n\n# 假设已有 extracted_features 和 label 数据\n\nX = extracted_features\ny_labels = np.argmax(label.values, axis=1)\n\n# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y_labels, test_size=0.2, random_state=42)\n\n# 初始化 XGBoost 分类器，并指定 tree_method 为 \"gpu_hist\"\nxgb_clf = XGBClassifier(n_estimators=1000,\n                        max_depth=6,\n                        learning_rate=0.1,\n                        objective='multi:softmax',\n                        num_class=len(np.unique(y_labels)),\n                        early_stopping_rounds=100,\n                        verbose=100,\n                        tree_method='gpu_hist')  # 指定使用 GPU\n\n# 训练 XGBoost 模型\nxgb_clf.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=100)\n\n# 获取模型的准确率\nacc_xgb = accuracy_score(y_test, xgb_clf.predict(X_test))\n\n# 预测结果\ny_pred_xgb = xgb_clf.predict(X_test)\n\n# 输出模型的准确率\naccuracy = accuracy_score(y_test, y_pred_xgb)\nprint(f\"Accuracy: {accuracy}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:44.764696Z","iopub.execute_input":"2024-01-27T12:31:44.765060Z","iopub.status.idle":"2024-01-27T12:31:51.433896Z","shell.execute_reply.started":"2024-01-27T12:31:44.765028Z","shell.execute_reply":"2024-01-27T12:31:51.432745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer Test and Create Submission CSV","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:51.435418Z","iopub.execute_input":"2024-01-27T12:31:51.435849Z","iopub.status.idle":"2024-01-27T12:31:51.451839Z","shell.execute_reply.started":"2024-01-27T12:31:51.435806Z","shell.execute_reply":"2024-01-27T12:31:51.450887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FEATURE ENGINEER TEST\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\ndata_test = np.zeros((len(test), extracted_features.shape[1]))\ntest_eeg = np.zeros((len(test), 10000, 19))\n\nfor k in range(len(test)):\n    row = test.iloc[k]\n    s = int(row.eeg_id)\n    eeg_test = pd.read_parquet(f'{PATH2}{s}.parquet')\n    eeg_test = eeg_test.iloc[:, :-1]\n    #eeg_test = eeg_test.iloc[:, [col for col in range(eeg_test.shape[1]) if col not in [8, 9, 10]]]\n    # 计算起始时间点\n    start_time_point = int((eeg_test.shape[0] - 10_000) // 2)\n    # 获取中间50秒的时间点数据\n    eeg_test_slice = eeg_test.iloc[start_time_point : start_time_point + 10_000, :]\n    \n    test_eeg[k, :, :] = eeg_test_slice\n\n    # 使用之前定义的 extract_eeg_features 函数进行特征提取\n    features_test = extract_eeg_features(test_eeg)\n    print(\"Shape of features_test:\", features_test.shape)\n    # 将提取的特征放入 data_test 中\n    data_test = features_test\n\n# 输出 data_test 的形状\nprint(\"Shape of data_test:\", data_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:51.453026Z","iopub.execute_input":"2024-01-27T12:31:51.453356Z","iopub.status.idle":"2024-01-27T12:31:51.509108Z","shell.execute_reply.started":"2024-01-27T12:31:51.453330Z","shell.execute_reply":"2024-01-27T12:31:51.508090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 使用训练好的模型进行预测\npredictions_proba = xgb_clf.predict_proba(data_test)\n\n# 类别的名称\nclass_names = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n\n# 创建一个DataFrame来存储概率值\nprobabilities_df = pd.DataFrame(predictions_proba, columns=class_names)\n\n# 显示DataFrame\nprint(probabilities_df.head(1))","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:51.510191Z","iopub.execute_input":"2024-01-27T12:31:51.510461Z","iopub.status.idle":"2024-01-27T12:31:51.545170Z","shell.execute_reply.started":"2024-01-27T12:31:51.510437Z","shell.execute_reply":"2024-01-27T12:31:51.544370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = probabilities_df\nsub.to_csv('submission.csv',index=False)\nprint('Submissionn shape',sub.shape)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:51.546266Z","iopub.execute_input":"2024-01-27T12:31:51.546575Z","iopub.status.idle":"2024-01-27T12:31:51.566827Z","shell.execute_reply.started":"2024-01-27T12:31:51.546548Z","shell.execute_reply":"2024-01-27T12:31:51.565920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsub.iloc[:,-6:].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-01-27T12:31:51.567829Z","iopub.execute_input":"2024-01-27T12:31:51.568091Z","iopub.status.idle":"2024-01-27T12:31:51.576434Z","shell.execute_reply.started":"2024-01-27T12:31:51.568069Z","shell.execute_reply":"2024-01-27T12:31:51.575482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}