{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\ndf=pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:06.034081Z","iopub.execute_input":"2024-03-11T14:53:06.034802Z","iopub.status.idle":"2024-03-11T14:53:07.400847Z","shell.execute_reply.started":"2024-03-11T14:53:06.034770Z","shell.execute_reply":"2024-03-11T14:53:07.399873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.hist(df['eeg_label_offset_seconds'], bins='auto', log=True)\nplt.xlabel('Offset Seconds')\nplt.ylabel('Frequency (Log Scale)')\nplt.title('Histogram of Offset Seconds (Log Scale)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:07.402875Z","iopub.execute_input":"2024-03-11T14:53:07.403448Z","iopub.status.idle":"2024-03-11T14:53:09.940904Z","shell.execute_reply.started":"2024-03-11T14:53:07.403410Z","shell.execute_reply":"2024-03-11T14:53:09.939757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_eeg = pd.read_parquet(\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet\")\nsample_train_eeg","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:09.942366Z","iopub.execute_input":"2024-03-11T14:53:09.942700Z","iopub.status.idle":"2024-03-11T14:53:10.231004Z","shell.execute_reply.started":"2024-03-11T14:53:09.942673Z","shell.execute_reply":"2024-03-11T14:53:10.230014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(20, figsize=(10, 100))\n\n# Generate a line plot for each column in the DataFrame\nfor i, column in enumerate(sample_train_eeg.columns):\n    ax[i].plot(sample_train_eeg.index, sample_train_eeg[column], label=column)\n    ax[i].grid(True)\n    ax[i].set_title(str(column))\n\n# plt.legend()\n# plt.title('Simulated Data Line Chart')\n# plt.xlabel('Index')\n# plt.ylabel('Values')\n# plt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:10.233954Z","iopub.execute_input":"2024-03-11T14:53:10.234425Z","iopub.status.idle":"2024-03-11T14:53:17.644145Z","shell.execute_reply.started":"2024-03-11T14:53:10.234389Z","shell.execute_reply":"2024-03-11T14:53:17.643172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectrogram(spectrogram_path):\n    sample_spect = pd.read_parquet(spectrogram_path)\n    \n    split_spect = {\n        \"LL\": sample_spect.filter(regex='^LL', axis=1),\n        \"RL\": sample_spect.filter(regex='^RL', axis=1),\n        \"RP\": sample_spect.filter(regex='^RP', axis=1),\n        \"LP\": sample_spect.filter(regex='^LP', axis=1),\n    }\n    \n    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 12))\n    axes = axes.flatten()\n    label_interval = 5\n    for i, split_name in enumerate(split_spect.keys()):\n        ax = axes[i]\n        img = ax.imshow(np.log(split_spect[split_name]).T, cmap='viridis', aspect='auto', origin='lower')\n        cbar = fig.colorbar(img, ax=ax)\n        cbar.set_label('Log(Value)')\n        ax.set_title(split_name)\n        ax.set_ylabel(\"Frequency (Hz)\")\n        ax.set_xlabel(\"Time\")\n\n        ax.set_yticks(np.arange(len(split_spect[split_name].columns)))\n        ax.set_yticklabels([column_name[3:] for column_name in split_spect[split_name].columns])\n        frequencies = [column_name[3:] for column_name in split_spect[split_name].columns]\n        ax.set_yticks(np.arange(0, len(split_spect[split_name].columns), label_interval))\n        ax.set_yticklabels(frequencies[::label_interval])\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:17.645470Z","iopub.execute_input":"2024-03-11T14:53:17.645839Z","iopub.status.idle":"2024-03-11T14:53:17.660367Z","shell.execute_reply.started":"2024-03-11T14:53:17.645808Z","shell.execute_reply":"2024-03-11T14:53:17.659400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_spectrogram('/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/353733.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:17.661431Z","iopub.execute_input":"2024-03-11T14:53:17.661744Z","iopub.status.idle":"2024-03-11T14:53:20.487714Z","shell.execute_reply.started":"2024-03-11T14:53:17.661720Z","shell.execute_reply":"2024-03-11T14:53:20.486652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGETS = df.columns[-6:]","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:20.488747Z","iopub.execute_input":"2024-03-11T14:53:20.489029Z","iopub.status.idle":"2024-03-11T14:53:20.493411Z","shell.execute_reply.started":"2024-03-11T14:53:20.489005Z","shell.execute_reply":"2024-03-11T14:53:20.492501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a Unique EEG Segment per eeg_id:\n# The code groups (groupby) the EEG data (df) by eeg_id. Each eeg_id represents a different EEG recording.\n# It then picks the first spectrogram_id and the earliest (min) spectrogram_label_offset_seconds for each eeg_id. This helps in identifying the starting point of each EEG segment.\n# The resulting DataFrame train has columns spec_id (first spectrogram_id) and min (earliest spectrogram_label_offset_seconds).\ntrain = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\n\n# Finding the Latest Point in Each EEG Segment:\n# The code again groups the data by eeg_id and finds the latest (max) spectrogram_label_offset_seconds for each segment.\n# This max value is added to the train DataFrame, representing the end point of each EEG segment.\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first') # The code adds the patient_id for each eeg_id to the train DataFrame. This links each EEG segment to a specific patient.\ntrain['patient_id'] = tmp\n\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum') # The code sums up the target variable counts (like votes for seizure, LPD, etc.) for each eeg_id.\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values # It then normalizes these counts so that they sum up to 1. This step converts the counts into probabilities, which is a common practice in classification tasks.\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first') # For each eeg_id, the code includes the expert_consensus on the EEG segment's classification.\ntrain['target'] = tmp\n\ntrain = train.reset_index() # This makes eeg_id a regular column, making the DataFrame easier to work with.\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:20.494495Z","iopub.execute_input":"2024-03-11T14:53:20.494851Z","iopub.status.idle":"2024-03-11T14:53:20.588699Z","shell.execute_reply.started":"2024-03-11T14:53:20.494828Z","shell.execute_reply":"2024-03-11T14:53:20.587852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nREAD_SPEC_FILES = False\nFEATURE_ENGINEER = True","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:20.589794Z","iopub.execute_input":"2024-03-11T14:53:20.590156Z","iopub.status.idle":"2024-03-11T14:53:20.594538Z","shell.execute_reply.started":"2024-03-11T14:53:20.590112Z","shell.execute_reply":"2024-03-11T14:53:20.593535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:53:20.599369Z","iopub.execute_input":"2024-03-11T14:53:20.599783Z","iopub.status.idle":"2024-03-11T14:54:20.760030Z","shell.execute_reply.started":"2024-03-11T14:53:20.599758Z","shell.execute_reply":"2024-03-11T14:54:20.759044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n# ENGINEER FEATURES\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# The code generates features from the spectrogram data for use in a model \n# The features are derived by calculating the mean and minimum values over time for each of the 400 spectrogram frequencies.\n# Two types of windows are used for these calculations:\n# A 10-minute window (_mean_10m, _min_10m).\n# A 20-second window (_mean_20s, _min_20s).\n# This process results in 1600 features (400 features × 4 calculations) for each EEG ID.\n\nSPEC_COLS = pd.read_parquet(f'{PATH}1000086677.parquet').columns[1:]\nFEATURES = [f'{c}_mean_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_mean_20s' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_20s' for c in SPEC_COLS]\nprint(f'We are creating {len(FEATURES)} features for {len(train)} rows... ',end='')\n\n\n# A data matrix data is initialized to store the new features for each eeg_id in the train DataFrame.\n# For each row in train, the code calculates the mean and minimum values within the specified 10-minute and 20-second windows.\n# These calculated values are then stored in the data matrix.\n# Finally, the matrix is added to the train DataFrame as new columns.\n\nif FEATURE_ENGINEER:\n    data = np.zeros((len(train),len(FEATURES)))\n    for k in range(len(train)):\n        if k%100==0: print(k,', ',end='')\n        row = train.iloc[k]\n        r = int( (row['min'] + row['max'])//4 ) \n        \n        # 10 MINUTE WINDOW FEATURES (MEANS and MINS)\n        x = np.nanmean(spectrograms[row.spec_id][r:r+300,:],axis=0)\n        data[k,:400] = x\n        x = np.nanmin(spectrograms[row.spec_id][r:r+300,:],axis=0)\n        data[k,400:800] = x\n        \n        # 20 SECOND WINDOW FEATURES (MEANS and MINS)\n        x = np.nanmean(spectrograms[row.spec_id][r+145:r+155,:],axis=0)\n        data[k,800:1200] = x\n        x = np.nanmin(spectrograms[row.spec_id][r+145:r+155,:],axis=0)\n        data[k,1200:1600] = x\n\n    train[FEATURES] = data\nelse:\n    train = pd.read_parquet('/kaggle/input/brain-spectrograms/train.pqt')\nprint()\nprint('New train shape:',train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:20.761637Z","iopub.execute_input":"2024-03-11T14:54:20.761924Z","iopub.status.idle":"2024-03-11T14:54:37.863981Z","shell.execute_reply.started":"2024-03-11T14:54:20.761899Z","shell.execute_reply":"2024-03-11T14:54:37.863021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import signal\nfrom sklearn.decomposition import PCA","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:37.865228Z","iopub.execute_input":"2024-03-11T14:54:37.865564Z","iopub.status.idle":"2024-03-11T14:54:38.605412Z","shell.execute_reply.started":"2024-03-11T14:54:37.865533Z","shell.execute_reply":"2024-03-11T14:54:38.604601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_frequency_band_features(segment):\n    # Define EEG frequency bands\n    eeg_bands = {'Delta': (0.5, 4), 'Theta': (4, 8), 'Alpha': (8, 12), 'Beta': (12, 30), 'Gamma': (30, 45)}\n    \n    band_features = []\n    for band in eeg_bands:\n        low, high = eeg_bands[band]\n        # Filter signal for the specific band\n        band_pass_filter = signal.butter(3, [low, high], btype='bandpass', fs=200, output='sos')\n        filtered = signal.sosfilt(band_pass_filter, segment)\n        # Extract features like mean, standard deviation, etc.\n        band_features.extend([np.nanmean(filtered), np.nanstd(filtered), np.nanmax(filtered), np.nanmin(filtered)])\n    \n    return band_features","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:38.606492Z","iopub.execute_input":"2024-03-11T14:54:38.606962Z","iopub.status.idle":"2024-03-11T14:54:38.613775Z","shell.execute_reply.started":"2024-03-11T14:54:38.606936Z","shell.execute_reply":"2024-03-11T14:54:38.612840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:38.614877Z","iopub.execute_input":"2024-03-11T14:54:38.615180Z","iopub.status.idle":"2024-03-11T14:54:38.625768Z","shell.execute_reply.started":"2024-03-11T14:54:38.615151Z","shell.execute_reply":"2024-03-11T14:54:38.624916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/hms-harmful-brain-activity-classification/'\n\ndf = pd.DataFrame({'path': glob(BASE_PATH + '**/*.parquet')})\ndf['test_type'] = df['path'].str.split('/').str.get(-2).str.split('_').str.get(-1)\ndf['id'] = df['path'].str.split('/').str.get(-1).str.split('.').str.get(0)\n\ndf_eeg = pd.read_parquet(BASE_PATH + 'train_eegs/1000913311.parquet')\ndf_eeg.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:38.626961Z","iopub.execute_input":"2024-03-11T14:54:38.627493Z","iopub.status.idle":"2024-03-11T14:54:39.329775Z","shell.execute_reply.started":"2024-03-11T14:54:38.627470Z","shell.execute_reply":"2024-03-11T14:54:39.328631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_channels = df_eeg.shape[1]\nn_channels","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:39.330935Z","iopub.execute_input":"2024-03-11T14:54:39.331233Z","iopub.status.idle":"2024-03-11T14:54:39.337150Z","shell.execute_reply.started":"2024-03-11T14:54:39.331208Z","shell.execute_reply":"2024-03-11T14:54:39.336095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom sklearn.impute import SimpleImputer\n\n# Initialize a PCA model\npca = PCA(n_components=0.95)\nprint(\"PCA model initialized.\")\n\n# Initialize an array for original features\nnum_rows = len(train)\nnum_features = 20 * n_channels  # 20 features per channel\ndata_original = np.zeros((num_rows, num_features))\n\nprint(\"Starting feature extraction and PCA processing...\")\nstart_time = time.time()\n\nfor k in range(num_rows):\n    if k % 1000 == 0:\n        print(f\"Processing row {k} of {num_rows}...\")\n\n    row = train.iloc[k]\n    r = int((row['min'] + row['max']) // 4)\n    eeg_segment = spectrograms[row.spec_id][r:r+300, :]\n\n    # Apply the feature extraction function to each EEG channel\n    all_channel_features = []\n    for i in range(n_channels):\n        channel_features = extract_frequency_band_features(eeg_segment[:, i])\n        all_channel_features.extend(channel_features)\n    \n    data_original[k, :] = all_channel_features\n\nprint(\"Data matrix constructed\")\n\n# Impute NaN values in the data matrix\nimputer = SimpleImputer(strategy='mean')\ndata_imputed = imputer.fit_transform(data_original)\n\nprint(f\"NaN values handled. Imputed data matrix shape: {data_imputed.shape}\")\n\n# Apply PCA on the imputed data\npca.fit(data_imputed)\nprint(\"PCA fitting completed.\")\n\n# Transform data using PCA\ndata_pca = pca.transform(data_imputed)\n\n# Add PCA features to DataFrame\npca_feature_columns = [f'pca_feature_{i}' for i in range(data_pca.shape[1])]\ntrain[pca_feature_columns] = data_pca\n\n# Measure total processing time\ntotal_time = time.time() - start_time\nprint(f\"Total processing time: {total_time:.2f} seconds.\")","metadata":{"execution":{"iopub.status.busy":"2024-03-11T14:54:39.338456Z","iopub.execute_input":"2024-03-11T14:54:39.338811Z","iopub.status.idle":"2024-03-11T15:30:40.960316Z","shell.execute_reply.started":"2024-03-11T14:54:39.338774Z","shell.execute_reply":"2024-03-11T15:30:40.959033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:40.961979Z","iopub.execute_input":"2024-03-11T15:30:40.962608Z","iopub.status.idle":"2024-03-11T15:30:41.033598Z","shell.execute_reply.started":"2024-03-11T15:30:40.962554Z","shell.execute_reply":"2024-03-11T15:30:41.032359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Columns to be excluded from scaling\nexcluded_columns = ['eeg_id', 'spec_id', 'min', 'max', 'patient_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote','target']\n\n# Save the columns to be excluded\nexcluded_data = train[excluded_columns]\n\n# DataFrame with only the columns to be scaled\nfeatures = train.drop(columns=excluded_columns)\n\n# Initialize the StandardScaler\nscaler = StandardScaler()\n\n# Fit the scaler to the features and transform them\nfeatures_scaled = scaler.fit_transform(features)\n\n# Create a DataFrame from the scaled features\nfeatures_scaled_df = pd.DataFrame(features_scaled, columns=features.columns)\n\n# Concatenate the scaled features with the excluded columns\ntrain_scaled_df = pd.concat([excluded_data.reset_index(drop=True),features_scaled_df,], axis=1)\ntrain_scaled_df ","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:41.035430Z","iopub.execute_input":"2024-03-11T15:30:41.036427Z","iopub.status.idle":"2024-03-11T15:30:42.249326Z","shell.execute_reply.started":"2024-03-11T15:30:41.036379Z","shell.execute_reply":"2024-03-11T15:30:42.248330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_scaled_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:42.250537Z","iopub.execute_input":"2024-03-11T15:30:42.250862Z","iopub.status.idle":"2024-03-11T15:30:42.333163Z","shell.execute_reply.started":"2024-03-11T15:30:42.250836Z","shell.execute_reply":"2024-03-11T15:30:42.332165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nimport gc\nfrom sklearn.model_selection import KFold, GroupKFold\n\nprint('XGBoost version', xgb.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:42.334501Z","iopub.execute_input":"2024-03-11T15:30:42.334927Z","iopub.status.idle":"2024-03-11T15:30:42.610972Z","shell.execute_reply.started":"2024-03-11T15:30:42.334890Z","shell.execute_reply":"2024-03-11T15:30:42.610031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VER=1","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:42.612018Z","iopub.execute_input":"2024-03-11T15:30:42.612300Z","iopub.status.idle":"2024-03-11T15:30:42.616181Z","shell.execute_reply.started":"2024-03-11T15:30:42.612276Z","shell.execute_reply":"2024-03-11T15:30:42.615264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_oof = []\nall_true = []\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train , train .target, train .patient_id)):   \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    model = xgb.XGBClassifier(\n        objective='multi:softprob', \n        num_class=len(TARS),\n        learning_rate = 0.1, \n                      \n#         tree_method='gpu_hist',  #skip GPU acceleration\n    )\n    \n    # Prepare training and validation data\n    X_train = train.loc[train_index, FEATURES]\n    y_train = train.loc[train_index, 'target'].map(TARS)\n    X_valid = train.loc[valid_index, FEATURES]\n    y_valid = train.loc[valid_index, 'target'].map(TARS)\n    \n    model.fit(X_train, y_train, \n              eval_set=[(X_valid, y_valid)], \n              verbose=True, \n              early_stopping_rounds=10)\n    model.save_model(f'XGB_v{VER}_f{i}.model')\n    \n    oof = model.predict_proba(X_valid)\n    all_oof.append(oof)\n    all_true.append(train.loc[valid_index, TARGETS].values)\n    \n    del X_train, y_train, X_valid, y_valid, oof\n    gc.collect()\n    \nall_oof = np.concatenate(all_oof)\nall_true = np.concatenate(all_true)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:30:42.617289Z","iopub.execute_input":"2024-03-11T15:30:42.617589Z","iopub.status.idle":"2024-03-11T15:48:05.381049Z","shell.execute_reply.started":"2024-03-11T15:30:42.617552Z","shell.execute_reply":"2024-03-11T15:48:05.379982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\nfrom sklearn.metrics import log_loss\n\n\ndef objective(trial):\n    # Hyperparameters to be tuned by Optuna\n    param = {\n        'objective': 'multi:softprob',\n        'num_class': len(TARS),\n        'tree_method': 'gpu_hist',  # use 'gpu_hist' for GPU\n        'lambda': trial.suggest_loguniform('lambda', 1e-4, 10.0),\n        'alpha': trial.suggest_loguniform('alpha', 1e-4, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.6, 0.7, 0.8, 0.9, 1.0]),\n        'learning_rate': trial.suggest_categorical('learning_rate', [0.008, 0.01, 0.02, 0.05, 0.1]),\n        'n_estimators': 1000,\n        'max_depth': trial.suggest_categorical('max_depth', [5, 7, 9, 11, 13]),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),\n    }\n\n    gkf = GroupKFold(n_splits=5)\n    cv_scores = []\n\n    for train_index, valid_index in gkf.split(train, train.target, train.patient_id):\n        X_train, X_valid = train.loc[train_index, FEATURES], train.loc[valid_index, FEATURES]\n        y_train, y_valid = train.loc[train_index, 'target'].map(TARS), train.loc[valid_index, 'target'].map(TARS)\n\n        model = xgb.XGBClassifier(**param)\n        model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False, early_stopping_rounds=10)\n        preds = model.predict_proba(X_valid)\n        cv_scores.append(log_loss(y_valid, preds))\n\n    return np.mean(cv_scores)\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=10)  # Increase n_trials for more extensive search\n\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T15:48:05.382491Z","iopub.execute_input":"2024-03-11T15:48:05.382828Z","iopub.status.idle":"2024-03-11T17:08:40.058147Z","shell.execute_reply.started":"2024-03-11T15:48:05.382801Z","shell.execute_reply":"2024-03-11T17:08:40.057147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOP = 30\n\n# Assuming 'model' is your trained model\nfeature_importance = model.feature_importances_\n\n# Get the feature names from 'train'\nfeature_names = train.columns\n\n# Sort the feature importances and get the indices of the sorted array\nsorted_idx = np.argsort(feature_importance)\n\n# Plot only the top 'TOP' features\nfig = plt.figure(figsize=(10, 8))\nplt.barh(np.arange(len(sorted_idx))[-TOP:], feature_importance[sorted_idx][-TOP:], align='center')\nplt.yticks(np.arange(len(sorted_idx))[-TOP:], feature_names[sorted_idx][-TOP:])\nplt.title(f'Feature Importance - Top {TOP}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:40.059779Z","iopub.execute_input":"2024-03-11T17:08:40.060070Z","iopub.status.idle":"2024-03-11T17:08:40.524003Z","shell.execute_reply.started":"2024-03-11T17:08:40.060044Z","shell.execute_reply":"2024-03-11T17:08:40.523055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:40.525330Z","iopub.execute_input":"2024-03-11T17:08:40.525626Z","iopub.status.idle":"2024-03-11T17:08:40.544133Z","shell.execute_reply.started":"2024-03-11T17:08:40.525600Z","shell.execute_reply":"2024-03-11T17:08:40.543193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nspec = pd.read_parquet(f'{PATH2}853520.parquet')\nspec","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:40.545180Z","iopub.execute_input":"2024-03-11T17:08:40.545440Z","iopub.status.idle":"2024-03-11T17:08:40.622900Z","shell.execute_reply.started":"2024-03-11T17:08:40.545418Z","shell.execute_reply":"2024-03-11T17:08:40.622015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# READ ALL TEST SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles = os.listdir(PATH2)\nprint(f'There are {len(files)} spectrogram parquets')\n\nspectrograms_test = {}\nfor i,f in enumerate(files):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms_test[name] = tmp.iloc[:,1:].values","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:40.627549Z","iopub.execute_input":"2024-03-11T17:08:40.628080Z","iopub.status.idle":"2024-03-11T17:08:40.663418Z","shell.execute_reply.started":"2024-03-11T17:08:40.628056Z","shell.execute_reply":"2024-03-11T17:08:40.662499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n# ENGINEER FEATURES\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# The code generates features from the spectrogram data for use in a model \n# The features are derived by calculating the mean and minimum values over time for each of the 400 spectrogram frequencies.\n# Two types of windows are used for these calculations:\n# A 10-minute window (_mean_10m, _min_10m).\n# A 20-second window (_mean_20s, _min_20s).\n# This process results in 1600 features (400 features × 4 calculations) for each EEG ID.\n\nSPEC_COLS = pd.read_parquet(f'{PATH}1000086677.parquet').columns[1:]\nFEATURES = [f'{c}_mean_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_10m' for c in SPEC_COLS]\nFEATURES += [f'{c}_mean_20s' for c in SPEC_COLS]\nFEATURES += [f'{c}_min_20s' for c in SPEC_COLS]\nprint(f'We are creating {len(FEATURES)} features for {len(test)} rows... ',end='')\n\n\n# A data matrix data is initialized to store the new features for each eeg_id in the train DataFrame.\n# For each row in train, the code calculates the mean and minimum values within the specified 10-minute and 20-second windows.\n# These calculated values are then stored in the data matrix.\n# Finally, the matrix is added to the train DataFrame as new columns.\n\ndata = np.zeros((len(test),len(FEATURES)))\nfor k in range(len(test)):\n    if k%100==0: print(k,', ',end='')\n    row = test.iloc[k]\n            \n    # 10 MINUTE WINDOW FEATURES\n    x = np.nanmean( spec.iloc[:,1:].values, axis=0)\n    data[k,:400] = x\n    x = np.nanmin( spec.iloc[:,1:].values, axis=0)\n    data[k,400:800] = x\n\n    # 20 SECOND WINDOW FEATURES\n    x = np.nanmean( spec.iloc[145:155,1:].values, axis=0)\n    data[k,800:1200] = x\n    x = np.nanmin( spec.iloc[145:155,1:].values, axis=0)\n    data[k,1200:1600] = x\n\n    test[FEATURES] = data\n\n    \nprint()\nprint('New test shape:',test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:40.664752Z","iopub.execute_input":"2024-03-11T17:08:40.665082Z","iopub.status.idle":"2024-03-11T17:08:41.650035Z","shell.execute_reply.started":"2024-03-11T17:08:40.665053Z","shell.execute_reply":"2024-03-11T17:08:41.649076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.impute import SimpleImputer\n\n# # Initialize a PCA model\n# pca = PCA(n_components=0.95)\n# print(\"PCA model initialized.\")\n\n# # Initialize an array for original features\n# num_rows = len(test)\n# num_features = 20 * n_channels  # 20 features per channel\n# data_original = np.zeros((num_rows, num_features))\n\n# print(\"Starting feature extraction and PCA processing...\")\n# start_time = time.time()\n\n# for k in range(num_rows):\n#     if k % 1000 == 0:\n#         print(f\"Processing row {k} of {num_rows}...\")\n\n#     row = train.iloc[k]\n#     eeg_segment = spectrograms_test[853520][r:r+300, :]\n\n#     # Apply the feature extraction function to each EEG channel\n#     all_channel_features = []\n#     for i in range(n_channels):\n#         channel_features = extract_frequency_band_features(eeg_segment[:, i])\n#         all_channel_features.extend(channel_features)\n    \n#     data_original[k, :] = all_channel_features\n\n# print(\"Data matrix constructed\")\n\n# # Impute NaN values in the data matrix\n# imputer = SimpleImputer(strategy='mean')\n# data_imputed = imputer.fit_transform(data_original)\n\n# print(f\"NaN values handled. Imputed data matrix shape: {data_imputed.shape}\")\n\n# # Apply PCA on the imputed data\n# pca.fit(data_imputed)\n# print(\"PCA fitting completed.\")\n\n# # Transform data using PCA\n# data_pca = pca.transform(data_imputed)\n\n# # Add PCA features to DataFrame\n# pca_feature_columns = [f'pca_feature_{i}' for i in range(data_pca.shape[1])]\n# test[pca_feature_columns] = data_pca\n\n# # Measure total processing time\n# total_time = time.time() - start_time\n# print(f\"Total processing time: {total_time:.2f} seconds.\")\n\n# test.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:41.651420Z","iopub.execute_input":"2024-03-11T17:08:41.651787Z","iopub.status.idle":"2024-03-11T17:08:41.657891Z","shell.execute_reply.started":"2024-03-11T17:08:41.651754Z","shell.execute_reply":"2024-03-11T17:08:41.656984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Columns to be excluded from scaling\nexcluded_columns = ['eeg_id', 'spectrogram_id', 'patient_id']\n\n# Save the columns to be excluded\nexcluded_data = test[excluded_columns]\n\n# DataFrame with only the columns to be scaled\nfeatures = test.drop(columns=excluded_columns)\n\n# Initialize the StandardScaler\nscaler = StandardScaler()\n\n# Fit the scaler to the features and transform them\nfeatures_scaled = scaler.fit_transform(features)\n\n# Create a DataFrame from the scaled features\nfeatures_scaled_df = pd.DataFrame(features_scaled, columns=features.columns)\n\n# Concatenate the scaled features with the excluded columns\ntest_scaled_df = pd.concat([excluded_data.reset_index(drop=True),features_scaled_df,], axis=1)\ntest_scaled_df","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:41.658956Z","iopub.execute_input":"2024-03-11T17:08:41.659214Z","iopub.status.idle":"2024-03-11T17:08:41.990393Z","shell.execute_reply.started":"2024-03-11T17:08:41.659189Z","shell.execute_reply":"2024-03-11T17:08:41.989440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FEATURE ENGINEER TEST\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\ndata = np.zeros((len(test),len(FEATURES)))\n    \nfor k in range(len(test)):\n    row = test.iloc[k]\n    s = int( row.spectrogram_id )\n    spec = pd.read_parquet(f'{PATH2}{s}.parquet')\n    \n    # 10 MINUTE WINDOW FEATURES\n    x = np.nanmean( spec.iloc[:,1:].values, axis=0)\n    data[k,:400] = x\n    x = np.nanmin( spec.iloc[:,1:].values, axis=0)\n    data[k,400:800] = x\n\n    # 20 SECOND WINDOW FEATURES\n    x = np.nanmean( spec.iloc[145:155,1:].values, axis=0)\n    data[k,800:1200] = x\n    x = np.nanmin( spec.iloc[145:155,1:].values, axis=0)\n    data[k,1200:1600] = x\n\ntest[FEATURES] = data\nprint('New test shape',test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:41.991696Z","iopub.execute_input":"2024-03-11T17:08:41.991988Z","iopub.status.idle":"2024-03-11T17:08:42.146295Z","shell.execute_reply.started":"2024-03-11T17:08:41.991963Z","shell.execute_reply":"2024-03-11T17:08:42.145355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER XGBOOST ON TEST\npreds = []\n\nfor i in range(5):\n    print(i, ', ', end='')\n    \n    # Load the XGBoost model\n    model = xgb.XGBClassifier()\n    model.load_model(f'XGB_v{VER}_f{i}.model')\n    \n    # Make predictions\n    pred = model.predict_proba(test[FEATURES])\n    preds.append(pred)\n\n# Average the predictions from each fold\npred = np.mean(preds, axis=0)\nprint()\nprint('Test preds shape', pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:42.147678Z","iopub.execute_input":"2024-03-11T17:08:42.148367Z","iopub.status.idle":"2024-03-11T17:08:43.549444Z","shell.execute_reply.started":"2024-03-11T17:08:42.148332Z","shell.execute_reply":"2024-03-11T17:08:43.548466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = pred\nsub.to_csv('submission.csv',index=False)\nprint('Submission shape',sub.shape)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:08:43.552774Z","iopub.execute_input":"2024-03-11T17:08:43.553146Z","iopub.status.idle":"2024-03-11T17:08:43.574601Z","shell.execute_reply.started":"2024-03-11T17:08:43.553120Z","shell.execute_reply":"2024-03-11T17:08:43.573663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}