{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"def submit_simple_prediction():\n    try: pd.__version__\n    except NameError as e: import pandas as pd\n    \n    test_path = '/kaggle/input/hms-harmful-brain-activity-classification/test.csv'\n    test = pd.read_csv(test_path)\n    targets = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n    \n    preds = []\n    for i in range(len(test)):\n        preds.append([1/6 for _ in range(6)])\n    \n    submission = pd.DataFrame({'eeg_id': test.eeg_id.values})\n    submission[targets] = preds\n    submission.to_csv('submission.csv', index=False)\n    return test, submission\n\ntest, submission = submit_simple_prediction()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-02T21:53:30.161409Z","iopub.execute_input":"2024-02-02T21:53:30.162120Z","iopub.status.idle":"2024-02-02T21:53:31.318477Z","shell.execute_reply.started":"2024-02-02T21:53:30.162085Z","shell.execute_reply":"2024-02-02T21:53:31.317231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, random, warnings, time, sklearn\nimport pandas as pd, numpy as np, matplotlib.pyplot as plt\nimport fastai\nfrom fastai.tabular.all import *\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:31.322264Z","iopub.execute_input":"2024-02-02T21:53:31.323101Z","iopub.status.idle":"2024-02-02T21:53:38.522942Z","shell.execute_reply.started":"2024-02-02T21:53:31.323047Z","shell.execute_reply":"2024-02-02T21:53:38.521961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '/kaggle/input/hms-harmful-brain-activity-classification/train.csv'\ndf = pd.read_csv(train_path)\ntargets = df.columns[-6:]\nprint(targets)\nprint(len(df))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:38.527660Z","iopub.execute_input":"2024-02-02T21:53:38.528078Z","iopub.status.idle":"2024-02-02T21:53:38.809165Z","shell.execute_reply.started":"2024-02-02T21:53:38.528040Z","shell.execute_reply":"2024-02-02T21:53:38.808124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_df(df, targets):\n    agg_dict = {'spectrogram_id': 'first', 'spectrogram_label_offset_seconds': 'min'}\n    \n    train = df.groupby('eeg_id')[['spectrogram_id', 'spectrogram_label_offset_seconds']].agg(agg_dict)\n    train.columns = ['spec_id', 'min_offset']\n    \n    max_offset = df.groupby('eeg_id')['spectrogram_label_offset_seconds'].max()\n    train['max_offset'] = max_offset\n    \n    patient_id = df.groupby('eeg_id')['patient_id'].first()\n    train['patient_id'] = patient_id\n    \n    target_sums = df.groupby('eeg_id')[targets].sum()\n    for target in targets:\n        train[target] = target_sums[target].values\n    \n    y_data = train[targets].values\n    y_data_normalised = y_data/y_data.sum(axis=1, keepdims=True)\n    train[targets] = y_data_normalised\n    \n    consensus = df.groupby('eeg_id')['expert_consensus'].first()\n    train['target'] = consensus\n\n    train = train.reset_index()\n    return train\n\ntrain = preprocess_df(df, targets)\nprint(len(train))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:38.812263Z","iopub.execute_input":"2024-02-02T21:53:38.812628Z","iopub.status.idle":"2024-02-02T21:53:38.932707Z","shell.execute_reply.started":"2024-02-02T21:53:38.812593Z","shell.execute_reply":"2024-02-02T21:53:38.931655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:38.934292Z","iopub.execute_input":"2024-02-02T21:53:38.934669Z","iopub.status.idle":"2024-02-02T21:53:38.943273Z","shell.execute_reply.started":"2024-02-02T21:53:38.934640Z","shell.execute_reply":"2024-02-02T21:53:38.942070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_avg(train):\n    return [train[col].mean() for col in train.columns[-7:-1]]\navg = get_avg(train)\nprint(avg)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:38.945145Z","iopub.execute_input":"2024-02-02T21:53:38.945510Z","iopub.status.idle":"2024-02-02T21:53:38.954064Z","shell.execute_reply.started":"2024-02-02T21:53:38.945480Z","shell.execute_reply":"2024-02-02T21:53:38.952968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_kaggle_spectrograms(read_individual_files=False):\n    t0 = time.time()\n    \n    if read_individual_files:\n        kaggle_spectrograms_path = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\n        files = os.listdir(kaggle_spectrograms_path)\n        spectrograms = {}\n        for index, file in enumerate(files):\n            if index%i == 100: print(f'index = {index}; time elapsed = {time.time()-t0:.2f}s')\n            temp = pd.read_parquet(f'{kaggle_spectrometers_path}{file}')\n            name = int(file.split('.')[0])\n            spectrograms[name] = temp.iloc[:,1:].values\n    else:\n        spectrograms_file_path = '/kaggle/input/brain-spectrograms/specs.npy'\n        spectrograms = np.load(spectrograms_file_path,allow_pickle=True).item()\n        print(f'time to load all kaggle spectrograms from specs.npy: {time.time()-t0:.2f}s')\n    return spectrograms\n\nkaggle_spectrograms = load_kaggle_spectrograms()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:53:59.271415Z","iopub.execute_input":"2024-02-02T21:53:59.271848Z","iopub.status.idle":"2024-02-02T21:55:00.594360Z","shell.execute_reply.started":"2024-02-02T21:53:59.271801Z","shell.execute_reply":"2024-02-02T21:55:00.592678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_eeg_spectrograms(read_individual_files=False):\n    t0 = time.time()\n    if read_individual_files:\n        eeg_spectrograms = {}\n        for index, eeg_id in enumerate(train.eeg_id.values):\n            if i%100==0: print(f'index = {index}; time elapsed = {time.time()-t0:.2f}s')\n            eeg_spectrograms_path = '/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/'\n            eeg_spectrogram = np.load(f'{eeg_spectrograms_path}{eeg_index}.npy')\n            eeg_spectrograms[eeg_id] = eeg_spectrogram\n    else:\n        eeg_spectrograms_file_path = '/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy'\n        eeg_spectrograms = np.load(eeg_spectrograms_file_path, allow_pickle=True).item()\n        print(f'time to load all eeg spectrograms from specs.npy: {time.time()-t0:.2f}s')\n    return eeg_spectrograms\n\neeg_spectrograms = load_eeg_spectrograms()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T21:55:00.596985Z","iopub.execute_input":"2024-02-02T21:55:00.598195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nspec_cols = pd.read_parquet(f'{path}1000086677.parquet').columns[1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_features(row, kaggle_spectrograms, eeg_spectrograms, chain=None):\n    if chain == None: chain = 'all'\n    chain_map = {'LL': (0, 100), 'RL': (100, 200), 'LP': (200, 300), 'RP': (300, 400), 'all': (0, 400)}\n    cr = chain_map[chain]\n    \n    features, feature_names = [], []\n    \n    def kaggle_spectrogram_features(spectrogram, features, feature_names, t0):\n        t1 = t0 + 300\n        features.append(np.nanmean(spectrogram[t0:t1, cr[0]:cr[1]],axis=0))\n        features.append(np.nanmin(spectrogram[t0:t1, cr[0]:cr[1]],axis=0))\n        feature_names.append([f'{c}_mean_10m' for c in spec_cols])\n        feature_names.append([f'{c}_min_10m' for c in spec_cols])\n        \n        t0, t1 = t0+145, t0+155\n        features.append(np.nanmean(spectrogram[t0:t1, cr[0]: cr[1]], axis=0))\n        features.append(np.nanmin(spectrogram[t0:t1, cr[0]: cr[1]], axis=0))\n        feature_names.append([f'{c}_mean_20s' for c in spec_cols])\n        feature_names.append([f'{c}_min_20s' for c in spec_cols])\n        \n        return features, feature_names\n    \n    def eeg_spectrogram_features(spectrogram, features, feature_names):\n        features.append(np.nanmean(spectrogram.T[100:-100, cr[0]:cr[1]], axis=0))\n        features.append(np.nanmin(spectrogram.T[100:-100, cr[0]:cr[1]], axis=0))\n        features.append(np.nanmax(spectrogram.T[100:-100, cr[0]:cr[1]], axis=0))\n        features.append(np.nanstd(spectrogram.T[100:-100, cr[0]:cr[1]], axis=0))\n        \n        feature_names.append([f'{c}_mean_10s' for c in spec_cols])\n        feature_names.append([f'{c}_min_10s' for c in spec_cols])\n        feature_names.append([f'{c}_max_10s' for c in spec_cols])\n        feature_names.append([f'{c}_std_10s' for c in spec_cols])\n        \n        return features, feature_names\n    \n    t0 = int((row['min_offset'] + row['max_offset']) //4)\n    \n    features, feature_names = kaggle_spectrogram_features(kaggle_spectrograms[row.spec_id], features, feature_names, t0)\n    \n    # RESHAPE EEG SPECTROGRAMS\n    eeg_spectrogram = np.zeros((512, 256), dtype='float32')\n    xx = eeg_spectrograms[row.eeg_id]\n    for j in range(4):\n        eeg_spectrogram[128*j:128*(j+1),] = xx[:,:,j]\n        \n    features, feature_names = eeg_spectrogram_features(eeg_spectrogram, features, feature_names)\n\n    # Concatenate all features into a single array\n    return np.concatenate(features), np.concatenate(feature_names)\n\nn_chains = 4\nn_features = 8\nfeature_len = n_chains * 100 * n_features\ndata = np.zeros((len(train), feature_len))  # Update 4448 based on total number of features\n\ntime_zero = time.time()\n# for i in range(2):\nfor i in range(len(train)):\n    if i % 1000 == 0:\n        print(f'{(time.time()-time_zero):.2f}s; training row = {i}')\n    row = train.iloc[i]\n    data[i, :], feature_names = extract_features(row, kaggle_spectrograms, eeg_spectrograms, chain='all')\n    \ntrain[feature_names] = data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = train\n\ncolumn_means = train2.iloc[:, 5:11].mean()\ntrain2.iloc[:, 5:11] = train2.iloc[:, 5:11].fillna(column_means)\n\ncolumn_means = train2.iloc[:, 12:].mean()\ntrain2.iloc[:, 12:] = train2.iloc[:, 12:].fillna(column_means)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train2.drop(train2.columns[:12], axis=1)\ny = train2.iloc[:, 5:11]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cont_names = list(X.columns)\nsplits = RandomSplitter(valid_pct=0.2)(range_of(train2))\nto = TabularPandas(train2, procs=[Categorify, FillMissing, Normalize],\n                   cont_names = cont_names,\n                   y_names='seizure_vote',\n                   splits=splits)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = to.dataloaders(bs=64)\ndls.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner = tabular_learner(dls, metrics=accuracy)\nlearner.fit_one_cycle(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.show_results()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img\n\n# CREATE ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 0\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FEATURE ENGINEER TEST\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\n\nchain = 'all'\nspec_range_map = {'LL': (0, 100),\n                  'RL': (100, 200),\n                  'LP': (200, 300),\n                  'RP': (300, 400),\n                  'all': (0, 400)}\nspec_range = spec_range_map[chain]\n\npath = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nspec_cols = pd.read_parquet(f'{path}1000086677.parquet').columns[1:]\n\ndata = np.zeros((len(test), 3200))\nfor k in range(len(test)):\n    \n    features = []\n    feature_names = []\n    row = test.iloc[k]\n    s = int( row.spectrogram_id )\n    spec = pd.read_parquet(f'{PATH2}{s}.parquet')\n    \n    # 10 MINUTE WINDOW FEATURES\n    x = np.nanmean(spec.iloc[:,spec_range[0]:spec_range[1]].values, axis=0)\n    features.append(x)\n    print(len(np.concatenate(features)))\n    feature_names = [f'{c}_mean_10m' for c in spec_cols]\n    \n    x = np.nanmin(spec.iloc[:,spec_range[0]:spec_range[1]].values, axis=0)\n    features.append(x)\n    print(len(np.concatenate(features)))\n    feature_names += [f'{c}_min_10m' for c in spec_cols]\n\n#     # 20 SECOND WINDOW FEATURES\n    x = np.nanmean(spec.iloc[145:155,spec_range[0]:spec_range[1]].values, axis=0)\n    features.append(x)\n    print(len(np.concatenate(features)))\n    feature_names += [f'{c}_mean_20s' for c in spec_cols]\n    \n    x = np.nanmin( spec.iloc[145:155,spec_range[0]:spec_range[1]].values, axis=0)\n    features.append(x)\n    feature_names += [f'{c}_min_20s' for c in spec_cols]\n    \n    # RESHAPE EEG SPECTROGRAMS 128x256x4 => 512x256\n    eeg_spec = np.zeros((512,256),dtype='float32')\n    xx = all_eegs2[row.eeg_id]\n    for j in range(4): eeg_spec[128*j:128*(j+1),] = xx[:,:,j]\n\n    # 10 SECOND WINDOW FROM EEG SPECTROGRAMS \n    x = np.nanmean(eeg_spec.T[100:-100,spec_range[0]:spec_range[1]],axis=0)\n    features.append(x)\n    feature_names += [f'{c}_mean_10s' for c in spec_cols]\n    \n    x = np.nanmin(eeg_spec.T[100:-100,spec_range[0]:spec_range[1]],axis=0)\n    features.append(x)\n    feature_names += [f'{c}_min_10s' for c in spec_cols]\n    \n    x = np.nanmax(eeg_spec.T[100:-100,spec_range[0]:spec_range[1]],axis=0)\n    features.append(x)\n    feature_names += [f'{c}_max_10s' for c in spec_cols]\n    \n    x = np.nanstd(eeg_spec.T[100:-100,spec_range[0]:spec_range[1]],axis=0)\n    features.append(x)\n    feature_names += [f'{c}_std_10s' for c in spec_cols]\n\n    print(len(np.concatenate(features)))\n    data[k,:] = np.concatenate(features)\ntest[feature_names] = data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(feature_names))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.columns[0:20]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(data[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, col in enumerate(test.columns):\n    if (i)%100 == 0:\n        print(col)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test2 = test\n\ncolumn_means = test2.iloc[:, 3:].mean()\ntest2.iloc[:, 3:] = test2.iloc[:, 3:].fillna(column_means)\n\n# column_means = test2.iloc[:, 12:].mean()\n# test2.iloc[:, 12:] = test2.iloc[:, 12:].fillna(column_means)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = test2.drop(test2.columns[:3], axis=1)\n# y = test2.iloc[:, 5:11]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(x):\n    e_x = np.exp(x - np.max(x))\n    return e_x/e_x.sum()\n\nsub = pd.DataFrame({'eeg_id': test.eeg_id.values})\npreds_all = []\nfor i in range(len(X)):\n    preds = avg\n    _, _, probs = learner.predict(X.iloc[i])\n\n    preds[0] = float(probs)\n    preds = softmax(preds)\n    preds_all.append(preds)\n\nsub[targets] = preds_all\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}