{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7659601,"sourceType":"datasetVersion","datasetId":4443737}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CatBoost on EEG and Spectrogram Data\n\nThe engineered features are from this <a href=https://www.kaggle.com/datasets/jacobsharples/engineered-features-for-hms-2024>Kaggle dataset</a> which is built upon three datasets:\n* Raw EEG Data\n* Raw Spectrogram Data (NAs filled by 0)\n* Spectrogram data built by EEGs (<a href=https://www.kaggle.com/datasets/cdeotte/brain-eeg-spectrograms/data)>Spectrogram Data Built by Chris Deotte</a>)\n\nFive classes utilized to build the features:\n* **ReadData**: Simple class to read eeg and spectrogram files\n* **FeatureEngineerData**: Class that turns data into summarized statistics\n* **EEGFeatures**: Feature engineer EEG data\n* **SpectrogramFeatures**: Feature engineer Spectrogram data\n* **EEGBuiltSpectrogramFeatures**: Feature engineer the EEG-built spectrograms\n\nThe EEGFeatures, SpectrogramFeatures, EEGBuiltSpectrogramFeatures classes all contain a method called **get_features()** which contains the engineered features I gather for each dataset.\n\n### Model Used?\nCurrently using CatBoostClassifier\n\n### Latest Version\n* Added a **FEATURES** variable to control which features to train on","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy.special import kl_div\nfrom sklearn.model_selection import GroupKFold\nimport xgboost as xgb\nimport catboost as cb\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n\nimport sys\nsys.path.append('/kaggle/input/kaggle-kl-div/')\nfrom kaggle_kl_div import score\n\nimport os\nimport gc\n#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n\nREGENERATING_DATA = False # Whether to regenerate the data, best to keep at false\nUSE_WEIGHTS = False # Whether or not to use the total votes to weigh the CatBoostClassifier\nFEATURES = 'drop' # either all, drop, or select","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-23T23:16:59.601124Z","iopub.execute_input":"2024-02-23T23:16:59.601344Z","iopub.status.idle":"2024-02-23T23:17:02.047540Z","shell.execute_reply.started":"2024-02-23T23:16:59.601323Z","shell.execute_reply":"2024-02-23T23:17:02.046782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom Functions and Classes <a class=\"anchor\"  id=\"classes\"></a>","metadata":{}},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:17:02.049243Z","iopub.execute_input":"2024-02-23T23:17:02.049897Z","iopub.status.idle":"2024-02-23T23:17:02.131548Z","shell.execute_reply.started":"2024-02-23T23:17:02.049867Z","shell.execute_reply":"2024-02-23T23:17:02.130828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def kl_div_score(model, X_test, true):\n    subm = pd.DataFrame(model.predict_proba(X_test), columns = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'])\n    subm = subm.set_axis(true.columns, axis=1)\n\n    subm['id'] = range(len(subm))\n    \n    true['id'] = range(len(true))\n    return score(true, subm, 'id')\n\ndef kl_div_score_reg(model, X_test, true):\n    \n    preds = model.predict(X_test).clip(0)\n    preds = preds/preds.sum(axis=1)[:,None]\n    \n    subm = pd.DataFrame(preds, columns = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'])\n    subm = subm.set_axis(true.columns, axis=1)\n\n    subm['id'] = range(len(subm))\n    \n    true['id'] = range(len(true))\n    return score(true, subm, 'id')\n\nclass ReadData():\n    def __init__(self, is_train = True):\n        self.is_train = is_train\n    \n    def _read_data(self, data_type, file_id):\n        if self.is_train:\n            PATH = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_{data_type}/{file_id}.parquet\"\n        else:\n            PATH = f\"/kaggle/input/hms-harmful-brain-activity-classification/test_{data_type}/{file_id}.parquet\"\n        \n        return pd.read_parquet(PATH)\n        \n    def read_spectrogram_data(self, spectrogram_id):\n        return self._read_data('spectrograms', spectrogram_id).set_index('time')\n    \n    def read_eeg_data(self, eeg_id) -> pd.DataFrame:\n        return self._read_data('eegs', eeg_id)\n    \n    def read_eeg_built_spectrogram_data(self, eeg_id) -> pd.DataFrame:\n        \n        \n        montages = ['LL', 'LP', 'RP', 'RR']\n        spec = pd.DataFrame()\n        \n        if self.is_train:\n            eeg_specs = np.load(f\"/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{eeg_id}.npy\")\n        else:\n            eeg_specs = spectrogram_from_eeg(f\"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/{eeg_id}.parquet\")\n\n\n        for i in range(len(montages)):\n            spec = pd.concat([spec, pd.DataFrame(eeg_specs[:,:,i]).T.add_prefix(f'{montages[i]}_')], axis=1)\n        \n        return spec\n    \n    def read_train_data(self):\n        return pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n    \n    def read_test_data(self):\n        return pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:02.468676Z","iopub.execute_input":"2024-02-23T23:19:02.468960Z","iopub.status.idle":"2024-02-23T23:19:02.480068Z","shell.execute_reply.started":"2024-02-23T23:19:02.468939Z","shell.execute_reply":"2024-02-23T23:19:02.479219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureEngineerData(ReadData):\n    def __init__(self, metadata, is_train=True, row_id='label_id'):\n        '''\n        \n        Params\n        ----------\n        metadata : dict\n            Contains the information on the eeg ids and labels\n        \n        '''\n        self.metadata = metadata\n        self.is_train = is_train\n        \n        self.row_id = metadata[row_id]    \n        \n    def get_mean(self, df) -> pd.DataFrame:\n        return (df\n                .mean()\n                .reset_index()\n                .set_axis(['var', 'mean'], axis=1)\n                .assign(row_id = self.row_id)\n                .pivot(columns='var', values='mean', index='row_id')\n                .add_prefix('mean_')\n        )\n     \n    def get_max(self, df) -> pd.DataFrame:\n        return (df\n                .max()\n                .reset_index()\n                .set_axis(['var', 'max'], axis=1)\n                .assign(row_id = self.row_id)\n                .pivot(columns='var', values='max', index='row_id')\n                .add_prefix('max_')\n        )\n    \n    def get_min(self, df) -> pd.DataFrame:\n        return (df\n                .max()\n                .reset_index()\n                .set_axis(['var', 'min'], axis=1)\n                .assign(row_id = self.row_id)\n                .pivot(columns='var', values='min', index='row_id')\n                .add_prefix('min_')\n        )   \n    \n    def get_corr(self, df) -> pd.DataFrame:\n        '''\n        Returns the correlation of an eeg file\n        '''\n        def apply_mask(df):\n            mask = np.triu(np.ones_like(df, dtype=bool))\n            return df.where(mask).unstack().dropna()\n\n        return (df\n             .corr()\n             .pipe(apply_mask)\n             .reset_index()\n             .set_axis(['var_1', 'var_2', 'corr'], axis=1)\n             .query(\"var_1 != var_2\")\n             .assign(\n                 row_id = self.row_id,\n                 label = lambda x: x.var_1 + \"_\" + x.var_2\n             )\n                .pivot(columns='label', values='corr', index='row_id')\n                .add_prefix('cor_')\n        )\n    \n    def filter_spectrogram_corr(self, corr_df) -> pd.DataFrame:\n        '''\n        Returns a dataframe with only the correlation across the same frequency\n        '''\n        return corr_df[[col for col in corr_df.columns if col.split('_')[2] == col.split('_')[4]]]\n    \n    def filter_eegspectrogram_corr(self, corr_df) -> pd.DataFrame:\n        pass\n        \n    \n    def get_std(self, df) -> pd.DataFrame:\n        return (df\n                .std()\n                .reset_index()\n                .set_axis(['var', 'std'], axis=1)\n                .assign(row_id = self.row_id)\n                .pivot(columns='var', values='std', index='row_id')\n                .add_prefix('std_')\n        )\n    \n    def get_range(self, df) -> pd.DataFrame:\n        return (\n            df\n            .max()\n            .sub(df.min())\n            .reset_index()\n            .set_axis(['var', 'range'], axis=1)\n            .assign(row_id = self.row_id)\n            .pivot(columns='var', values='range', index='row_id')\n            .add_prefix('range_')\n        )","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:04.006383Z","iopub.execute_input":"2024-02-23T23:19:04.006689Z","iopub.status.idle":"2024-02-23T23:19:04.019113Z","shell.execute_reply.started":"2024-02-23T23:19:04.006669Z","shell.execute_reply":"2024-02-23T23:19:04.018448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGFeatures(FeatureEngineerData):\n    \n    def get_offset(self):\n        if self.metadata.get('right_eeg_index') is None:\n            return [0, 10000]\n        else:\n            return [self.metadata['left_eeg_index'], self.metadata['right_eeg_index']]\n        \n    def format_eeg_data(self, window_sizes = {}):\n        \n        offset_range = self.get_offset()\n        \n        df = self.read_eeg_data(self.metadata['eeg_id']).iloc[offset_range[0]:offset_range[1]]\n        \n        eeg_df = pd.DataFrame()\n        for window in window_sizes:\n            left_index = window_sizes[window][0]\n            right_index = window_sizes[window][1]\n            \n            eeg_df = pd.concat([\n                eeg_df,\n                self.get_features(df.iloc[left_index:right_index], time_id = window)\n            ], axis=1)\n        \n        return eeg_df\n    \n    def get_features(self, df, time_id) -> pd.DataFrame():\n        return (\n            pd.concat([\n                self.get_mean(df),\n                self.get_std(df),\n                self.get_max(df),\n                self.get_range(df),\n                self.get_corr(df)\n            ], axis=1).add_prefix(f\"eeg_{time_id}_\")\n        )\n    \nclass SpectrogramFeatures(FeatureEngineerData):\n    \n    def get_offset(self):\n        if self.metadata.get('spectrogram_label_offset_seconds') is None:\n            return 0\n        else:\n            return self.metadata['spectrogram_label_offset_seconds']\n    \n    def format_spectrogram_data(self, window_sizes = {}):\n        \n        # Create a variable to make the code more readable\n        offset = self.get_offset()\n        \n        # Read specific spectrogram window\n        df = (self.read_spectrogram_data(self.metadata['spectrogram_id'])\n              .loc[offset:offset+600]\n              .fillna(0)\n             )\n                \n        # Creates the middle of the spectrogram\n        middle = (offset+(600+offset))/2\n\n        spec_df = pd.DataFrame()\n        for window in window_sizes:\n            left_index = window_sizes[window][0]\n            right_index = window_sizes[window][1]\n                        \n            spec_df = pd.concat([\n                spec_df,\n                self.get_features(df.loc[middle+left_index:middle+right_index], time_id = window)\n            ], axis=1)\n        \n        return spec_df\n    \n    def get_features(self, df, time_id) -> pd.DataFrame():\n        return (\n            pd.concat([\n                self.get_mean(df),\n                self.get_std(df),\n                self.get_max(df),\n                self.get_min(df),\n                self.get_range(df)\n            ], axis=1).add_prefix(f\"spec_{time_id}_\")\n        )\n    \nclass EEGBuiltSpectrogramFeatures(FeatureEngineerData):\n    def format_custom_spectrogram(self, window_sizes = {()} ):\n        \n        df = self.read_eeg_built_spectrogram_data(self.metadata['eeg_id']).copy()\n        \n        spec_df = pd.DataFrame()\n        for window in window_sizes:\n            left_index = window_sizes[window][0]\n            right_index = window_sizes[window][1]\n            \n            spec_df = pd.concat([\n                spec_df,\n                self.get_features(df.iloc[left_index:right_index], time_id = window)\n            ], axis=1)\n            \n        return spec_df\n    \n    def get_features(self, df, time_id) -> pd.DataFrame():\n        return (\n            pd.concat([\n                self.get_mean(df),\n                self.get_std(df),\n                self.get_max(df),\n                self.get_min(df),\n                self.get_range(df)\n            ], axis=1).add_prefix(f\"eegspec_{time_id}_\")\n        )","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:04.949459Z","iopub.execute_input":"2024-02-23T23:19:04.949754Z","iopub.status.idle":"2024-02-23T23:19:04.967039Z","shell.execute_reply.started":"2024-02-23T23:19:04.949733Z","shell.execute_reply":"2024-02-23T23:19:04.966455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"markdown","source":"## Window Functions","metadata":{}},{"cell_type":"code","source":"eeg_windows = {\n    '10s': (4000, 6000), # Middle 10s\n    '30s': (2000, 8000), # Middle 30s\n    '50s': (0, 10000) # Entire sample (50s)\n}\n\nspec_windows = {\n    '10m': (-300, 300), # Entire sample\n    '5m': (-150, 150),\n    '1m': (-30, 30),\n    '10s': (-5, 5),\n    '20s': (-10, 10),\n    '30s': (-15, 15),\n    'pre': (-300, -10),\n    'post': (10, 300)\n    \n}\n\neeg_built_spec_windows = {\n    '50s': (0, 256), # Entire sample\n    '10s': (100, -100), # 10s\n    'pre': (0, 100),\n    'post': (-100, 256)\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:06.925086Z","iopub.execute_input":"2024-02-23T23:19:06.925752Z","iopub.status.idle":"2024-02-23T23:19:06.931060Z","shell.execute_reply.started":"2024-02-23T23:19:06.925727Z","shell.execute_reply":"2024-02-23T23:19:06.930382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = ReadData()\n\ntrain_df = rd.read_train_data()\ntrain_df['left_eeg_index'] = train_df['eeg_label_offset_seconds'].multiply(200).astype('int')\ntrain_df['right_eeg_index'] = train_df['eeg_label_offset_seconds'].add(50).multiply(200).astype('int')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:09.068049Z","iopub.execute_input":"2024-02-23T23:19:09.068384Z","iopub.status.idle":"2024-02-23T23:19:09.244811Z","shell.execute_reply.started":"2024-02-23T23:19:09.068364Z","shell.execute_reply":"2024-02-23T23:19:09.244083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Training Data","metadata":{}},{"cell_type":"code","source":"if REGENERATING_DATA:\n    df = pd.DataFrame()\n    for index, row in tqdm(train_df.query(\"eeg_sub_id == 0\").iterrows()):\n        e = EEGFeatures(metadata=dict(row))\n        s = SpectrogramFeatures(metadata=dict(row))\n        es = EEGBuiltSpectrogramFeatures(metadata=dict(row))\n\n        feature_data = pd.concat([\n            e.format_eeg_data(eeg_windows),\n            s.format_spectrogram_data(spec_windows),\n            es.format_custom_spectrogram(eeg_built_spec_windows)\n        ], axis=1)\n\n\n        df = pd.concat([\n            df,\n            feature_data\n        ])\n    print('Finished creating training data...')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:11.164356Z","iopub.execute_input":"2024-02-23T23:19:11.164686Z","iopub.status.idle":"2024-02-23T23:19:11.169807Z","shell.execute_reply.started":"2024-02-23T23:19:11.164663Z","shell.execute_reply":"2024-02-23T23:19:11.169213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_prob = train_df.query(\"eeg_sub_id == 0\").filter(like = '_vote')\ny_prob = y_prob.divide(y_prob.sum(axis=1), axis=0)\nidx = y_prob[y_prob.max(axis=1) == 1].reset_index(drop=True).index","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:13.627183Z","iopub.execute_input":"2024-02-23T23:19:13.627644Z","iopub.status.idle":"2024-02-23T23:19:13.653554Z","shell.execute_reply.started":"2024-02-23T23:19:13.627620Z","shell.execute_reply":"2024-02-23T23:19:13.652673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = (pd.concat(\n        [\n        pd.read_parquet(\"/kaggle/input/engineered-features-for-hms-2024/eeg_data.parquet\"),\n        pd.read_parquet(\"/kaggle/input/engineered-features-for-hms-2024/spectrogram_data.parquet\"),\n        pd.read_parquet(\"/kaggle/input/engineered-features-for-hms-2024/eeg_built_spectrogram_data.parquet\"),\n        ], axis=1)\n      .sample(frac=1, random_state=42)\n     )#.iloc[idx]\n\nX_raw = df.reset_index(drop=True).copy()\ny_prob = df.reset_index().rename(columns={'row_id':'label_id'}).merge(train_df).filter(like = '_vote')\ny_prob = y_prob.divide(y_prob.sum(axis=1), axis=0)\n\nTARGETS = {\n    'Seizure': 0,\n    'LPD': 1,\n    'GPD': 2,\n    'LRDA': 3,\n    'GRDA': 4,\n    'Other': 5\n}\ny = df.merge(train_df, left_on='row_id', right_on='label_id')['expert_consensus'].map(TARGETS)\n\npatient_id = train_df.query(\"eeg_sub_id == 0\").iloc[0:len(df)].groupby('eeg_id')['patient_id'].first()\n\ndel df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:19:16.276508Z","iopub.execute_input":"2024-02-23T23:19:16.276782Z","iopub.status.idle":"2024-02-23T23:19:40.068594Z","shell.execute_reply.started":"2024-02-23T23:19:16.276762Z","shell.execute_reply":"2024-02-23T23:19:40.067770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove features that are not important in the model\n\nif FEATURES == 'drop':\n    drop_cols = []\n    drop_cols.extend(X_raw.filter(like='post_range').columns)\n    drop_cols.extend(X_raw.filter(like='pre_range').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_post_min').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_post_max').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_post_std').columns)\n    drop_cols.extend(X_raw.filter(like='eeg_min').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_pre_').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_post_mean_').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_10s_range').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_50s_range').columns)\n    #drop_cols.extend(X_raw.filter(like='eegspec_10s_min').columns)\n    #drop_cols.extend(X_raw.filter(like='eegspec_50s_min').columns)\n    drop_cols.extend(X_raw.filter(like='eegspec_10s_min_RR').columns)\n\n    X = X_raw[X_raw.columns[~X_raw.columns.isin(drop_cols)]].copy()\n    \nelif FEATURES == 'select':\n    X = X_raw[['eegspec_10s_min_RR_33', 'eegspec_10s_mean_RR_17', 'eegspec_10s_std_LP_65',\n        'spec_30s_min_LL_6.64', 'eegspec_10s_std_LL_59', 'spec_1m_max_LP_13.09', 'spec_30s_std_RL_6.45',\n      'eeg_50s_range_EKG', 'spec_10s_mean_RL_5.66', 'eegspec_10s_max_LP_31',\n      'spec_10s_max_RL_16.8', 'eegspec_50s_std_LP_58', 'spec_20s_max_LP_1.17',\n      'eegspec_10s_std_LP_35', 'eegspec_10s_std_RP_86', 'spec_10s_mean_LP_5.66',\n      'eegspec_10s_min_LL_29', 'spec_10s_max_RL_2.15', 'spec_10s_max_LP_2.54',\n      'spec_1m_max_RL_12.89', 'spec_20s_mean_RL_7.62', 'eegspec_10s_min_RR_50',\n      'eegspec_10s_std_LP_34', 'spec_1m_min_RL_11.13', 'eegspec_50s_mean_LP_16',\n      'eegspec_pre_range_LP_48', 'eegspec_pre_std_RR_49', 'eeg_10s_cor_T6_T5',\n          'eegspec_pre_std_LP_21', 'eeg_30s_cor_C4_C3', 'eegspec_10s_min_RR_41',\n          'eegspec_10s_std_LP_32', 'spec_20s_std_LP_1.95', 'eegspec_10s_mean_RR_18',\n          'spec_1m_max_LP_18.36', 'spec_5m_std_LP_7.62', 'eegspec_10s_std_LP_92',\n          'spec_1m_mean_LP_6.84', 'eegspec_10s_max_RR_35', 'spec_30s_min_RL_7.62',\n          'eegspec_50s_std_LL_45', 'eegspec_pre_mean_LL_17', 'eegspec_10s_std_RR_40',\n          'eeg_10s_cor_Fz_P3', 'eegspec_10s_std_LP_47', 'eegspec_50s_std_LP_75',\n              'spec_20s_mean_RL_8.98', 'spec_20s_min_LP_1.76', 'spec_5m_range_LP_13.28',\n              'eegspec_10s_max_RR_17', 'eegspec_10s_max_LL_24', 'eegspec_10s_std_LP_44',\n              'spec_1m_std_RP_7.03', 'eegspec_10s_range_RP_52', 'eegspec_10s_max_RR_28', \n               'eegspec_50s_std_LP_39', 'eegspec_10s_std_LP_77']].copy()\nelse:\n    X = X_raw.copy()\n\ndel X_raw\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:27:35.581969Z","iopub.execute_input":"2024-02-23T23:27:35.582713Z","iopub.status.idle":"2024-02-23T23:27:35.671280Z","shell.execute_reply.started":"2024-02-23T23:27:35.582673Z","shell.execute_reply":"2024-02-23T23:27:35.670437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Weigh the CatBoostClassifier by the number of votes\ntotal_votes = train_df.filter(like='vote').sum(axis=1).values\nmax_votes = train_df.filter(like='vote').max(axis=1).values\nmax_prob = y_prob.max(axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"folds = GroupKFold(n_splits=5)\nresult_dict = {}\n\nfor fold, (train_index, valid_index) in enumerate(folds.split(X, y, patient_id)): \n    X_train, y_train = X.iloc[train_index], y.iloc[train_index] \n    X_val, y_val = X.iloc[valid_index], y.iloc[valid_index] \n    \n    train_pool = cb.Pool(X_train, y_train,)\n    valid_pool = cb.Pool(X_val, y_val)\n    \n    if USE_WEIGHTS:\n        train_pool.set_weight(max_prob[train_index].astype('float'))\n        valid_pool.set_weight(max_prob[valid_index].astype('float'))\n    \n    print('Training: CatBoost...')\n    cb_mod = cb.CatBoostClassifier(\n        iterations=500,\n        max_depth=5,\n        objective='MultiClass'\n    )\n    \n    cb_mod.fit(\n        train_pool, \n        eval_set=[valid_pool], \n        verbose=50\n    )\n    \n    kl_score = kl_div_score(cb_mod, X_val, y_prob.iloc[valid_index].reset_index(drop=True))\n    print('KL-Divergence score:', kl_score)\n    \n    cb_mod.save_model(f'cb_v{fold}.cat')\n    \n    result_dict[fold] = {\n        'model': cb_mod,\n        'kl-score': kl_score,\n        'pred': cb_mod.predict_proba(X_val),\n        'true_prob':y_prob.iloc[valid_index],\n        'true': y_val\n    }","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:27:53.024139Z","iopub.execute_input":"2024-02-23T23:27:53.024442Z","iopub.status.idle":"2024-02-23T23:30:12.058223Z","shell.execute_reply.started":"2024-02-23T23:27:53.024420Z","shell.execute_reply":"2024-02-23T23:30:12.057074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_list = []\nfor key in result_dict:\n    score_list.append(result_dict[key]['kl-score'])\nnp.mean(score_list)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:30:32.224494Z","iopub.execute_input":"2024-02-23T23:30:32.224822Z","iopub.status.idle":"2024-02-23T23:30:32.230955Z","shell.execute_reply.started":"2024-02-23T23:30:32.224798Z","shell.execute_reply":"2024-02-23T23:30:32.230342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Explanation","metadata":{}},{"cell_type":"code","source":"cb_mod.get_feature_importance(prettified=True).head(n=50)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:30:36.138518Z","iopub.execute_input":"2024-02-23T23:30:36.139071Z","iopub.status.idle":"2024-02-23T23:30:36.155061Z","shell.execute_reply.started":"2024-02-23T23:30:36.139049Z","shell.execute_reply":"2024-02-23T23:30:36.154180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_mod.get_feature_importance(prettified=True).tail(n=50)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:30:38.158322Z","iopub.execute_input":"2024-02-23T23:30:38.158626Z","iopub.status.idle":"2024-02-23T23:30:38.175053Z","shell.execute_reply.started":"2024-02-23T23:30:38.158603Z","shell.execute_reply":"2024-02-23T23:30:38.174274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import shap","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:30:45.276131Z","iopub.execute_input":"2024-02-23T23:30:45.276458Z","iopub.status.idle":"2024-02-23T23:30:49.210587Z","shell.execute_reply.started":"2024-02-23T23:30:45.276434Z","shell.execute_reply":"2024-02-23T23:30:49.209558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#shap_values = shap.TreeExplainer(cb_mod).shap_values(X_train)\n#shap.summary_plot(shap_values, X_train, class_names = y_prob.columns)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T23:30:49.212166Z","iopub.execute_input":"2024-02-23T23:30:49.212814Z","iopub.status.idle":"2024-02-23T23:30:53.236931Z","shell.execute_reply.started":"2024-02-23T23:30:49.212787Z","shell.execute_reply":"2024-02-23T23:30:53.236090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"rd = ReadData(is_train=False)\n\ntest_df = rd.read_test_data()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T22:25:51.016615Z","iopub.execute_input":"2024-02-23T22:25:51.017022Z","iopub.status.idle":"2024-02-23T22:25:51.028907Z","shell.execute_reply.started":"2024-02-23T22:25:51.016992Z","shell.execute_reply":"2024-02-23T22:25:51.027661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = pd.DataFrame()\nfor index, row in tqdm(test_df.iterrows()):\n    e = EEGFeatures(metadata=dict(row), is_train=False, row_id='eeg_id')\n    s = SpectrogramFeatures(metadata=dict(row), is_train=False, row_id='eeg_id')\n    es = EEGBuiltSpectrogramFeatures(metadata=dict(row), is_train=False, row_id='eeg_id')\n    \n    feature_data = pd.concat([\n        e.format_eeg_data(eeg_windows),\n        s.format_spectrogram_data(spec_windows),\n        es.format_custom_spectrogram(eeg_built_spec_windows)\n    ], axis=1)\n    \n    \n    X_test = pd.concat([\n        X_test,\n        feature_data\n    ])\n    \n# Select same features from training dataset\nX_test = X_test[X_test.columns[X_test.columns.isin(X.columns)]]","metadata":{"execution":{"iopub.status.busy":"2024-02-23T22:25:53.275899Z","iopub.execute_input":"2024-02-23T22:25:53.276572Z","iopub.status.idle":"2024-02-23T22:26:02.553135Z","shell.execute_reply.started":"2024-02-23T22:25:53.276537Z","shell.execute_reply":"2024-02-23T22:26:02.551778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\nfor i in range(5):\n    print(i,', ',end='')\n    model = cb.CatBoostClassifier(\n        iterations=1000,\n        max_depth=5,\n        objective='MultiClass'\n    )\n    model.load_model(f'cb_v{i}.cat')\n    \n    test_pool = cb.Pool(\n        data = X_test\n    )\n    \n    pred = model.predict_proba(test_pool)\n    preds.append(pred)\n    \npred = np.mean(preds,axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-02-13T19:12:49.287159Z","iopub.execute_input":"2024-02-13T19:12:49.287545Z","iopub.status.idle":"2024-02-13T19:12:52.499550Z","shell.execute_reply.started":"2024-02-13T19:12:49.287520Z","shell.execute_reply":"2024-02-13T19:12:52.498564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = pd.DataFrame(pred, columns=['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'])\nsubm['eeg_id'] = test_df.eeg_id\n\nsubm[['eeg_id','seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-13T19:12:52.500703Z","iopub.execute_input":"2024-02-13T19:12:52.500999Z","iopub.status.idle":"2024-02-13T19:12:52.510473Z","shell.execute_reply.started":"2024-02-13T19:12:52.500974Z","shell.execute_reply":"2024-02-13T19:12:52.509442Z"},"trusted":true},"execution_count":null,"outputs":[]}]}