{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://wooded-margin-2f4.notion.site/6e1d535b5bcc4b6caa1ce593e6db3cbe?pvs=4","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd, numpy as np\nfrom glob import glob\nimport matplotlib.pyplot as plt\nVER = 1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 설명\n    - **`os`**: 운영 체제 - 운영 체제에 종속된 기능을 사용\n    - **`pandas`**: 데이터 조작 및 분석\n    - **`numpy`**: 수치 계산\n    - **`glob`**: 파일 경로 확장\n    - **`matplotlib.pyplot`**: 데이터 시각화","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/hms-harmful-brain-activity-classification/'\n\ntrain_df = pd.read_csv(f'{BASE_PATH}/train.csv')\neeg_id = train_df.loc[0, 'eeg_id'] # 0번째 행, egg_id 열 선택\neeg_df = pd.read_parquet(f'{BASE_PATH}/train_eegs/{eeg_id}.parquet')\neeg_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first') \ntrain['patient_id'] = tmp\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum') \nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values \ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first') \ntrain['target'] = tmp\n\ntrain = train.reset_index() ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"READ_SPEC_FILES = False \nFEATURE_ENGINEER = True # 특성 공학 수행","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH) \nspectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item() ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nSPEC_COLS = pd.read_parquet(f'{PATH}1000086677.parquet').columns[1:]\n\nFEATURES = [f'{c}_mean_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_mean_20s' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_20s' for c in SPEC_COLS]\n\n# 데이터 행렬 초기화 및 특성 계산\nif FEATURE_ENGINEER:\n    data = np.zeros((len(train),len(FEATURES)))\n    for k in range(len(train)):\n        if k%100==0: print(k,', ',end='')\n        row = train.iloc[k] \n        r = int( (row['min'] + row['max'])//4 ) \n        x = np.nanmean(spectrograms[row.spec_id][r:r+300,:],axis=0)\n        data[k,:400] = x  \n        x = np.nanmin(spectrograms[row.spec_id][r:r+300,:],axis=0)\n        data[k,400:800] = x\n        \n        x = np.nanmean(spectrograms[row.spec_id][r+145:r+155,:],axis=0)\n        data[k,800:1200] = x \n        x = np.nanmin(spectrograms[row.spec_id][r+145:r+155,:],axis=0)\n\t\t\t\tdata[k,1200:1600] = x\n\n    train[FEATURES] = data \nelse: # FEATURE_ENGINEER 가 False 인 경우 아래 경로에 있는 미리 생성된 학습데이터를 불러옴\n    train = pd.read_parquet('/kaggle/input/brain-spectrograms/train.pqt')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import signal\nfrom sklearn.decomposition import PCA\n\ndef extract_frequency_band_features(segment):\n    eeg_bands = {'Delta': (0.5, 4), 'Theta': (4, 8), 'Alpha': (8, 12), 'Beta': (12, 30), 'Gamma': (30, 45)}\n    band_features = [] \n    for band in eeg_bands:\n        low, high = eeg_bands[band] \n        band_pass_filter = signal.butter(3, [low, high], btype='bandpass', fs=200, output='sos')\n        filtered = signal.sosfilt(band_pass_filter, segment)\n        band_features.extend([np.nanmean(filtered), np.nanstd(filtered), np.nanmax(filtered), np.nanmin(filtered)])\n    \n    return band_features","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nimport gc\nfrom sklearn.model_selection import KFold, GroupKFold\n\nall_oof = [] # 각 검증 세트에 대한 모델의 예측 확률\nall_true = [] # 각 검증 세트의 실제라벨\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train , train .target, train .patient_id)):   \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n   \n    model = xgb.XGBClassifier(\n        objective='multi:softprob', # 각 클래스에 대한 예측확률 출력\n        num_class=len(TARS), \n        learning_rate = 0.1,\n    )\n    \n    \n    X_train = train.loc[train_index, FEATURES] \n    y_train = train.loc[train_index, 'target'].map(TARS)\n    X_valid = train.loc[valid_index, FEATURES] #\n    y_valid = train.loc[valid_index, 'target'].map(TARS)\n    \n\t\t\n    model.fit(X_train, y_train, \n              eval_set=[(X_valid, y_valid)], \n              verbose=True, \n              early_stopping_rounds=10)\n    model.save_model(f'XGB_v{VER}_f{i}.model') \n    \n    oof = model.predict_proba(X_valid)\n    all_oof.append(oof)\n    all_true.append(train.loc[valid_index, TARGETS].values)\n\n\t\tdel X_train, y_train, X_valid, y_valid, oof\n    gc.collect() \n    \nall_oof = np.concatenate(all_oof) #예측결과\nall_true = np.concatenate(all_true) # 실제 ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOP = 30\n\nfeature_importance = model.feature_importances_\n\nfeature_names = train.columns\n\nsorted_idx = np.argsort(feature_importance)\n\nfig = plt.figure(figsize=(10, 8))\nplt.barh(np.arange(len(sorted_idx))[-TOP:], feature_importance[sorted_idx][-TOP:], align='center')\nplt.yticks(np.arange(len(sorted_idx))[-TOP:], feature_names[sorted_idx][-TOP:])\nplt.title(f'Feature Importance - Top {TOP}')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\ndata = np.zeros((len(test),len(FEATURES)))\n    \nfor k in range(len(test)):\n    row = test.iloc[k]\n    s = int( row.spectrogram_id )\n    spec = pd.read_parquet(f'{PATH2}{s}.parquet')\n   \n    x = np.nanmean( spec.iloc[:,1:].values, axis=0)\n    data[k,:400] = x\n    x = np.nanmin( spec.iloc[:,1:].values, axis=0)\n    data[k,400:800] = x\n    x = np.nanmean( spec.iloc[145:155,1:].values, axis=0)\n    data[k,800:1200] = x\n    x = np.nanmin( spec.iloc[145:155,1:].values, axis=0)\n    data[k,1200:1600] = x\n\ntest[FEATURES] = data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\nfor i in range(5):\n    print(i, ', ', end='')\n    model = xgb.XGBClassifier()\n    model.load_model(f'XGB_v{VER}_f{i}.model')\n\n    pred = model.predict_proba(test[FEATURES])\n    preds.append(pred)\n\npred = np.mean(preds, axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = pred\nsub.to_csv('submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.iloc[:,-6:].sum(axis=1)","metadata":{},"execution_count":null,"outputs":[]}]}