{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7465251,"sourceType":"datasetVersion","datasetId":4317718},{"sourceId":158958765,"sourceType":"kernelVersion"},{"sourceId":159911317,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":7533.257933,"end_time":"2024-01-19T05:00:40.591450","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-19T02:55:07.333517","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Wavenet + EfficientNet = EEG + Spectrograms (LB: 0.48)\nIn this notebook I attempted to merge two most popular notebooks in this competition by @cdeotte, to make multi-input architecture that works on both - the 1D EEG signals and 2D spectrograms.\n<br>This is a baseline model, so feel free to experiment with it.\n<br>Resources:\n1. https://www.kaggle.com/code/cdeotte/wavenet-starter-lb-0-52\n2. https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43\n\nThe resulting architecture looks as follows:\n","metadata":{"papermill":{"duration":0.011598,"end_time":"2024-01-19T02:55:10.866974","exception":false,"start_time":"2024-01-19T02:55:10.855376","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#model = build_model()\n#tf.keras.utils.plot_model(model, \"multi_input_and_output_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-03T11:11:13.676709Z","iopub.execute_input":"2024-02-03T11:11:13.677151Z","iopub.status.idle":"2024-02-03T11:11:25.175063Z","shell.execute_reply.started":"2024-02-03T11:11:13.677117Z","shell.execute_reply":"2024-02-03T11:11:25.173750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data","metadata":{"papermill":{"duration":0.010667,"end_time":"2024-01-19T02:55:10.888626","exception":false,"start_time":"2024-01-19T02:55:10.877959","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt\n\ntrain = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nprint( train.shape )\ndisplay( train.head() )\n\n# CHOICE TO CREATE OR LOAD EEGS FROM NOTEBOOK VERSION 1\nCREATE_EEGS = False\nTRAIN_MODEL = True","metadata":{"papermill":{"duration":1.101164,"end_time":"2024-01-19T02:55:12.000613","exception":false,"start_time":"2024-01-19T02:55:10.899449","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:06:55.863786Z","iopub.execute_input":"2024-02-03T11:06:55.864755Z","iopub.status.idle":"2024-02-03T11:06:56.649874Z","shell.execute_reply.started":"2024-02-03T11:06:55.864708Z","shell.execute_reply":"2024-02-03T11:06:56.648573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Raw EEG Features","metadata":{"papermill":{"duration":0.012339,"end_time":"2024-01-19T02:55:12.024388","exception":false,"start_time":"2024-01-19T02:55:12.012049","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet')\nFEATS = df.columns\nprint(f'There are {len(FEATS)} raw eeg features')\nprint( list(FEATS) )","metadata":{"papermill":{"duration":0.205381,"end_time":"2024-01-19T02:55:12.240707","exception":false,"start_time":"2024-01-19T02:55:12.035326","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:06:57.850644Z","iopub.execute_input":"2024-02-03T11:06:57.851003Z","iopub.status.idle":"2024-02-03T11:06:58.044393Z","shell.execute_reply.started":"2024-02-03T11:06:57.850975Z","shell.execute_reply":"2024-02-03T11:06:58.043505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('We will use the following subset of raw EEG features:')\nFEATS = ['Fp1','O1','Fp2','O2']\nFEAT2IDX = {x:y for x,y in zip(FEATS,range(len(FEATS)))}\nprint( list(FEATS) )","metadata":{"papermill":{"duration":0.019409,"end_time":"2024-01-19T02:55:12.271361","exception":false,"start_time":"2024-01-19T02:55:12.251952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:06:58.355612Z","iopub.execute_input":"2024-02-03T11:06:58.356045Z","iopub.status.idle":"2024-02-03T11:06:58.363761Z","shell.execute_reply.started":"2024-02-03T11:06:58.356012Z","shell.execute_reply":"2024-02-03T11:06:58.362635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eeg_from_parquet(parquet_path, display=False):\n    \n    # EXTRACT MIDDLE 50 SECONDS\n    eeg = pd.read_parquet(parquet_path, columns=FEATS)\n    rows = len(eeg)\n    offset = (rows-10_000)//2\n    eeg = eeg.iloc[offset:offset+10_000]\n    \n    if display: \n        plt.figure(figsize=(10,5))\n        offset = 0\n    \n    # CONVERT TO NUMPY\n    data = np.zeros((10_000,len(FEATS)))\n    for j,col in enumerate(FEATS):\n        \n        # FILL NAN\n        x = eeg[col].values.astype('float32')\n        m = np.nanmean(x)\n        if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n        else: x[:] = 0\n            \n        data[:,j] = x\n        \n        if display: \n            if j!=0: offset += x.max()\n            plt.plot(range(10_000),x-offset,label=col)\n            offset -= x.min()\n            \n    if display:\n        plt.legend()\n        name = parquet_path.split('/')[-1]\n        name = name.split('.')[0]\n        plt.title(f'EEG {name}',size=16)\n        plt.show()\n        \n    return data","metadata":{"papermill":{"duration":0.023221,"end_time":"2024-01-19T02:55:12.305720","exception":false,"start_time":"2024-01-19T02:55:12.282499","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:06:58.872205Z","iopub.execute_input":"2024-02-03T11:06:58.872665Z","iopub.status.idle":"2024-02-03T11:06:58.887603Z","shell.execute_reply.started":"2024-02-03T11:06:58.872630Z","shell.execute_reply":"2024-02-03T11:06:58.886004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nall_eegs = {}\nDISPLAY = 4\nEEG_IDS = train.eeg_id.unique()\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\n\nfor i,eeg_id in enumerate(EEG_IDS):\n    if (i%100==0)&(i!=0): print(i,', ',end='') \n    \n    # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n    data = eeg_from_parquet(f'{PATH}{eeg_id}.parquet', display=i<DISPLAY)              \n    all_eegs[eeg_id] = data\n    \n    if i==DISPLAY:\n        if CREATE_EEGS:\n            print(f'Processing {train.eeg_id.nunique()} eeg parquets... ',end='')\n        else:\n            print(f'Reading {len(EEG_IDS)} eeg NumPys from disk.')\n            break\n            \nif CREATE_EEGS: \n    np.save('eegs',all_eegs)\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eegs/eegs.npy',allow_pickle=True).item()","metadata":{"papermill":{"duration":48.440875,"end_time":"2024-01-19T02:56:00.757834","exception":false,"start_time":"2024-01-19T02:55:12.316959","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:06:59.321905Z","iopub.execute_input":"2024-02-03T11:06:59.322346Z","iopub.status.idle":"2024-02-03T11:08:57.390597Z","shell.execute_reply.started":"2024-02-03T11:06:59.322312Z","shell.execute_reply":"2024-02-03T11:08:57.388330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deduplicate Train EEG Id","metadata":{"papermill":{"duration":0.016665,"end_time":"2024-01-19T02:56:00.791365","exception":false,"start_time":"2024-01-19T02:56:00.774700","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# LOAD TRAIN \ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\ntrain = df.groupby('eeg_id')[['patient_id']].agg('first')\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\ntrain = train.loc[train.eeg_id.isin(EEG_IDS)]\nprint('Train Data with unique eeg_id shape:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":0.263812,"end_time":"2024-01-19T02:56:01.071957","exception":false,"start_time":"2024-01-19T02:56:00.808145","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:08:57.394114Z","iopub.execute_input":"2024-02-03T11:08:57.395573Z","iopub.status.idle":"2024-02-03T11:08:57.902585Z","shell.execute_reply.started":"2024-02-03T11:08:57.395532Z","shell.execute_reply":"2024-02-03T11:08:57.901261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Butter Low-Pass Filter","metadata":{"papermill":{"duration":0.018601,"end_time":"2024-01-19T02:56:01.111099","exception":false,"start_time":"2024-01-19T02:56:01.092498","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from scipy.signal import butter, lfilter\n\ndef butter_lowpass_filter(data, cutoff_freq=20, sampling_rate=200, order=4):\n    nyquist = 0.5 * sampling_rate\n    normal_cutoff = cutoff_freq / nyquist\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    filtered_data = lfilter(b, a, data, axis=0)\n    return filtered_data","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:56:01.150195Z","iopub.status.busy":"2024-01-19T02:56:01.149885Z","iopub.status.idle":"2024-01-19T02:56:02.099248Z","shell.execute_reply":"2024-01-19T02:56:02.098375Z"},"papermill":{"duration":0.972857,"end_time":"2024-01-19T02:56:02.101527","exception":false,"start_time":"2024-01-19T02:56:01.128670","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FREQS = [1,2,4,8,16][::-1]\nx = [all_eegs[EEG_IDS[0]][:,0]]\nfor k in FREQS:\n    x.append( butter_lowpass_filter(x[0], cutoff_freq=k) )\n\nplt.figure(figsize=(20,20))\nplt.plot(range(10_000),x[0], label='without filter')\nfor k in range(1,len(x)):\n    plt.plot(range(10_000),x[k]-k*(x[0].max()-x[0].min()), label=f'with filter {FREQS[k-1]}Hz')\nplt.legend()\nplt.title('Butter Low-Pass Filter Examples',size=18)\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:56:02.182097Z","iopub.status.busy":"2024-01-19T02:56:02.181387Z","iopub.status.idle":"2024-01-19T02:56:03.065034Z","shell.execute_reply":"2024-01-19T02:56:03.064008Z"},"papermill":{"duration":0.951735,"end_time":"2024-01-19T02:56:03.072303","exception":false,"start_time":"2024-01-19T02:56:02.120568","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:56:03.122361Z","iopub.status.busy":"2024-01-19T02:56:03.122007Z","iopub.status.idle":"2024-01-19T02:56:03.207614Z","shell.execute_reply":"2024-01-19T02:56:03.206586Z"},"papermill":{"duration":0.112909,"end_time":"2024-01-19T02:56:03.209821","exception":false,"start_time":"2024-01-19T02:56:03.096912","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nREAD_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:56:03.258030Z","iopub.status.busy":"2024-01-19T02:56:03.257735Z","iopub.status.idle":"2024-01-19T02:57:02.754775Z","shell.execute_reply":"2024-01-19T02:57:02.753701Z"},"papermill":{"duration":59.548722,"end_time":"2024-01-19T02:57:02.782201","exception":false,"start_time":"2024-01-19T02:56:03.233479","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader with Butter Low-Pass Filter","metadata":{"papermill":{"duration":0.024209,"end_time":"2024-01-19T02:57:02.830693","exception":false,"start_time":"2024-01-19T02:57:02.806484","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\nimport albumentations as albu\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False,augment=False, eegs=all_eegs, specs = spectrograms,mode='train'): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.eegs = eegs\n        self.specs = specs\n        self.augment = False\n        self.mode = mode\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        eeg,spect, y = self.__data_generation(indexes)\n        if self.augment: spect = self.__augment_batch(spect) \n        return {'eeg':eeg, 'spect':spect}, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        spect = np.zeros((len(indexes),128,256,4),dtype='float32')\n        eeg = np.zeros((len(indexes),10_000,2),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        \n        sample = np.zeros((10_000,eeg.shape[-1]))\n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]      \n            data = self.eegs[row.eeg_id]\n            if self.mode=='test': \n                r = 0\n            elif self.mode=='valid': \n                r = int( (row['min'] + row['max'])//4 )\n            else:\n                # RANDOM CROPS FOR TRAIN\n                r = np.random.randint(row['min'], row['max']+1)//2  \n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                spect[j,14:-14,:,k] = img[:,22:-22]\n                \n            \n            \n            # FEATURE ENGINEER\n            sample[:,0] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['O1']]\n            sample[:,1] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['O2']]\n            \n            # STANDARDIZE\n            sample = np.clip(sample,-1024,1024)\n            sample = np.nan_to_num(sample, nan=0) / 32.0\n            \n            # BUTTER LOW-PASS FILTER\n            sample = butter_lowpass_filter(sample)\n            \n            eeg[j,] = sample\n            if self.mode!='test':\n                y[j] = row[TARGETS]\n            \n        return eeg,spect,y\n    ","metadata":{"papermill":{"duration":14.142003,"end_time":"2024-01-19T02:57:16.996695","exception":false,"start_time":"2024-01-19T02:57:02.854692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:10:34.538555Z","iopub.execute_input":"2024-02-03T11:10:34.539066Z","iopub.status.idle":"2024-02-03T11:10:36.455674Z","shell.execute_reply.started":"2024-02-03T11:10:34.539028Z","shell.execute_reply":"2024-02-03T11:10:36.454239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display DataLoader: Spects","metadata":{"papermill":{"duration":0.02343,"end_time":"2024-01-19T02:57:17.044233","exception":false,"start_time":"2024-01-19T02:57:17.020803","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gen = DataGenerator(train, shuffle=True)\nROWS=2; COLS=3; BATCHES=2\n\nfor i,(X,y) in enumerate(gen):\n    x = X['spect']\n    plt.figure(figsize=(20,8))\n    for j in range(ROWS):\n        for k in range(COLS):\n            plt.subplot(ROWS,COLS,j*COLS+k+1)\n            t = y[j*COLS+k]\n            img = x[j*COLS+k,:,:,0][::-1,]\n            mn = img.flatten().min()\n            mx = img.flatten().max()\n            img = (img-mn)/(mx-mn)\n            plt.imshow(img)\n            plt.title(f'Target = {TARS2[np.argmax(t)]}',size=16)\n            plt.yticks([])\n            plt.ylabel('Frequencies (Hz)',size=14)\n            plt.xlabel('Time (sec)',size=14)\n    plt.show()\n    if i==BATCHES-1: break","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:17.092755Z","iopub.status.busy":"2024-01-19T02:57:17.091824Z","iopub.status.idle":"2024-01-19T02:57:19.418378Z","shell.execute_reply":"2024-01-19T02:57:19.417534Z"},"papermill":{"duration":2.358184,"end_time":"2024-01-19T02:57:19.425647","exception":false,"start_time":"2024-01-19T02:57:17.067463","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Data Loader: EEGs","metadata":{"papermill":{"duration":0.038771,"end_time":"2024-01-19T02:57:19.503805","exception":false,"start_time":"2024-01-19T02:57:19.465034","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gen = DataGenerator(train, shuffle=False)\n\nfor x,y in gen:\n    x = x['eeg']\n    for k in range(4):\n        plt.figure(figsize=(20,4))\n        offset = 0\n        for j in range(x.shape[-1]):\n            if j!=0: offset -= x[k,:,j].min()\n            plt.plot(range(10_000),x[k,:,j]+offset,label=f'feature {j+1}')\n            offset += x[k,:,j].max()\n        tt = f'{y[k][0]:0.1f}'\n        for t in y[k][1:]:\n            tt += f', {t:0.1f}'\n        plt.title(f'EEG_Id = {EEG_IDS[k]}\\nTarget = {tt}',size=14)\n        plt.legend()\n        plt.show()\n    break","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:19.582979Z","iopub.status.busy":"2024-01-19T02:57:19.582613Z","iopub.status.idle":"2024-01-19T02:57:21.138360Z","shell.execute_reply":"2024-01-19T02:57:21.137370Z"},"papermill":{"duration":1.599046,"end_time":"2024-01-19T02:57:21.141581","exception":false,"start_time":"2024-01-19T02:57:19.542535","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initialize GPUs","metadata":{"papermill":{"duration":0.048114,"end_time":"2024-01-19T02:57:21.239501","exception":false,"start_time":"2024-01-19T02:57:21.191387","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nprint('TensorFlow version =',tf.__version__)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:21.339338Z","iopub.status.busy":"2024-01-19T02:57:21.338321Z","iopub.status.idle":"2024-01-19T02:57:22.714679Z","shell.execute_reply":"2024-01-19T02:57:22.713684Z"},"papermill":{"duration":1.428752,"end_time":"2024-01-19T02:57:22.716898","exception":false,"start_time":"2024-01-19T02:57:21.288146","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:22.819662Z","iopub.status.busy":"2024-01-19T02:57:22.819316Z","iopub.status.idle":"2024-01-19T02:57:22.824749Z","shell.execute_reply":"2024-01-19T02:57:22.823832Z"},"papermill":{"duration":0.059054,"end_time":"2024-01-19T02:57:22.826804","exception":false,"start_time":"2024-01-19T02:57:22.767750","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build WaveNet Model","metadata":{"papermill":{"duration":0.049395,"end_time":"2024-01-19T02:57:22.925254","exception":false,"start_time":"2024-01-19T02:57:22.875859","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TRAIN SCHEDULE\ndef lrfn(epoch):\n        return [1e-3,1e-3,1e-4,1e-4,1e-5][epoch]\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\nEPOCHS = 5","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:23.024454Z","iopub.status.busy":"2024-01-19T02:57:23.024121Z","iopub.status.idle":"2024-01-19T02:57:23.029248Z","shell.execute_reply":"2024-01-19T02:57:23.028352Z"},"papermill":{"duration":0.057034,"end_time":"2024-01-19T02:57:23.031134","exception":false,"start_time":"2024-01-19T02:57:22.974100","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate\n\ndef wave_block(x, filters, kernel_size, n):\n    dilation_rates = [2**i for i in range(n)]\n    x = Conv1D(filters = filters,\n               kernel_size = 1,\n               padding = 'same')(x)\n    res_x = x\n    for dilation_rate in dilation_rates:\n        tanh_out = Conv1D(filters = filters,\n                          kernel_size = kernel_size,\n                          padding = 'same', \n                          activation = 'tanh', \n                          dilation_rate = dilation_rate)(x)\n        sigm_out = Conv1D(filters = filters,\n                          kernel_size = kernel_size,\n                          padding = 'same',\n                          activation = 'sigmoid', \n                          dilation_rate = dilation_rate)(x)\n        x = Multiply()([tanh_out, sigm_out])\n        x = Conv1D(filters = filters,\n                   kernel_size = 1,\n                   padding = 'same')(x)\n        res_x = Add()([res_x, x])\n    return res_x","metadata":{"papermill":{"duration":0.061364,"end_time":"2024-01-19T02:57:23.142140","exception":false,"start_time":"2024-01-19T02:57:23.080776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:09:35.525784Z","iopub.execute_input":"2024-02-03T11:09:35.526199Z","iopub.status.idle":"2024-02-03T11:09:51.185564Z","shell.execute_reply.started":"2024-02-03T11:09:35.526154Z","shell.execute_reply":"2024-02-03T11:09:51.184046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{"papermill":{"duration":13.526393,"end_time":"2024-01-19T02:57:36.718094","exception":false,"start_time":"2024-01-19T02:57:23.191701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:09:51.187680Z","iopub.execute_input":"2024-02-03T11:09:51.188394Z","iopub.status.idle":"2024-02-03T11:10:07.827028Z","shell.execute_reply.started":"2024-02-03T11:09:51.188361Z","shell.execute_reply":"2024-02-03T11:10:07.825600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn","metadata":{"papermill":{"duration":0.068239,"end_time":"2024-01-19T02:57:36.838398","exception":false,"start_time":"2024-01-19T02:57:36.770159","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:10:07.829601Z","iopub.execute_input":"2024-02-03T11:10:07.830164Z","iopub.status.idle":"2024-02-03T11:10:07.957024Z","shell.execute_reply.started":"2024-02-03T11:10:07.830104Z","shell.execute_reply":"2024-02-03T11:10:07.955653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    \n        \n    # INPUT EEG #################################\n    inp_eeg = tf.keras.Input(shape=(10_000,2),name='eeg')\n    \n    x = wave_block(inp_eeg, 16, 3, 12)\n    x = wave_block(x, 32, 3, 8)\n    x = wave_block(x, 64, 3, 4)\n    x = wave_block(x, 128, 3, 1)\n    \n    # OUTPUT\n    x_eeg = tf.keras.layers.GlobalMaxPooling1D()(x)\n    x_eeg = tf.keras.layers.Flatten()(x_eeg)\n    \n    \n    \n    # INPUT Spectrogram #################################\n    \n    inp_spect = tf.keras.Input(shape=(128,256,4),name='spect')\n    base_model = efn.EfficientNetB2(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b2_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n    \n    # RESHAPE INPUT 128x256x4 => 512x256x3 MONOTONE IMAGE\n    x0 = inp_spect[:,:,:,:1]\n    x1 = inp_spect[:,:,:,1:2]\n    x2 = inp_spect[:,:,:,2:3]\n    x3 = inp_spect[:,:,:,3:4]\n    x = tf.keras.layers.Concatenate(axis=1)([x0,x1,x2,x3])\n    x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x = base_model(x)\n    x_spect = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x_spect = tf.keras.layers.Flatten()(x_spect)\n    # Concatenate #################################\n    x = tf.keras.layers.Concatenate(axis=1)([x_eeg,x_spect])\n    \n    x = tf.keras.layers.Dense(128,activation='elu')(x)\n    x = tf.keras.layers.Dense(64,activation='elu')(x)\n    \n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n    \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=[inp_eeg,inp_spect], outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n    model.compile(loss=loss, optimizer = opt)\n    \n    return model","metadata":{"papermill":{"duration":0.065348,"end_time":"2024-01-19T02:57:36.953861","exception":false,"start_time":"2024-01-19T02:57:36.888513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:10:07.959647Z","iopub.execute_input":"2024-02-03T11:10:07.960677Z","iopub.status.idle":"2024-02-03T11:10:07.977558Z","shell.execute_reply.started":"2024-02-03T11:10:07.960640Z","shell.execute_reply":"2024-02-03T11:10:07.976457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()\ntf.keras.utils.plot_model(model, \"multi_input_and_output_model.png\", show_shapes=True)","metadata":{"papermill":{"duration":9.36076,"end_time":"2024-01-19T02:57:46.365773","exception":false,"start_time":"2024-01-19T02:57:37.005013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-03T11:10:40.168371Z","iopub.execute_input":"2024-02-03T11:10:40.168853Z","iopub.status.idle":"2024-02-03T11:10:52.786691Z","shell.execute_reply.started":"2024-02-03T11:10:40.168820Z","shell.execute_reply":"2024-02-03T11:10:52.785414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Group KFold","metadata":{"papermill":{"duration":0.07572,"end_time":"2024-01-19T02:57:46.519821","exception":false,"start_time":"2024-01-19T02:57:46.444101","status":"completed"},"tags":[]}},{"cell_type":"code","source":"VERBOSE = 1\nFOLDS_TO_TRAIN = 5\nif not os.path.exists('WaveNet_Model'):\n    os.makedirs('WaveNet_Model')\n\nfrom sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\nall_oof = []; all_oof2 = []; all_true = []\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):   \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    # TRAIN MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n    if TRAIN_MODEL:\n        model.fit(train_gen, verbose=VERBOSE,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'WaveNet_Model/WaveNet_fold{i}.h5')\n    else:\n        model.load_weights(f'/kaggle/input/brain-eegs/WaveNet_Model/WaveNet_fold{i}.h5')\n    \n    # WAVENET OOF\n    oof = model.predict(valid_gen, verbose=VERBOSE)\n    all_oof.append(oof)\n    all_true.append(train.iloc[valid_index][TARGETS].values)\n    \n    # TRAIN MEAN OOF\n    y_train = train.iloc[train_index][TARGETS].values\n    y_valid = train.iloc[valid_index][TARGETS].values\n    oof = y_valid.copy()\n    for j in range(6):\n        oof[:,j] = y_train[:,j].mean()\n    oof = oof / oof.sum(axis=1,keepdims=True)\n    all_oof2.append(oof)\n    \n    del model, oof, y_train, y_valid\n    gc.collect()\n    \n    if i==FOLDS_TO_TRAIN-1: break\n    \nall_oof = np.concatenate(all_oof)\nall_oof2 = np.concatenate(all_oof2)\nall_true = np.concatenate(all_true)","metadata":{"execution":{"iopub.execute_input":"2024-01-19T02:57:46.673739Z","iopub.status.busy":"2024-01-19T02:57:46.673350Z","iopub.status.idle":"2024-01-19T04:59:57.438769Z","shell.execute_reply":"2024-01-19T04:59:57.437678Z"},"papermill":{"duration":7330.845858,"end_time":"2024-01-19T04:59:57.441106","exception":false,"start_time":"2024-01-19T02:57:46.595248","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV Score for WaveNet","metadata":{"papermill":{"duration":0.944583,"end_time":"2024-01-19T04:59:59.396082","exception":false,"start_time":"2024-01-19T04:59:58.451499","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\noof = pd.DataFrame(all_oof.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score with WaveNet Raw EEG =',cv)","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:01.343439Z","iopub.status.busy":"2024-01-19T05:00:01.342480Z","iopub.status.idle":"2024-01-19T05:00:01.423969Z","shell.execute_reply":"2024-01-19T05:00:01.422913Z"},"papermill":{"duration":1.023694,"end_time":"2024-01-19T05:00:01.425915","exception":false,"start_time":"2024-01-19T05:00:00.402221","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV Score using Train Means","metadata":{"papermill":{"duration":1.09236,"end_time":"2024-01-19T05:00:03.468639","exception":false,"start_time":"2024-01-19T05:00:02.376279","status":"completed"},"tags":[]}},{"cell_type":"code","source":"oof = pd.DataFrame(all_oof2.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score with Train Means =',cv)","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:05.407967Z","iopub.status.busy":"2024-01-19T05:00:05.407576Z","iopub.status.idle":"2024-01-19T05:00:05.463475Z","shell.execute_reply":"2024-01-19T05:00:05.462485Z"},"papermill":{"duration":0.997218,"end_time":"2024-01-19T05:00:05.465403","exception":false,"start_time":"2024-01-19T05:00:04.468185","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit to Kaggle LB","metadata":{"papermill":{"duration":0.991583,"end_time":"2024-01-19T05:00:07.385313","exception":false,"start_time":"2024-01-19T05:00:06.393730","status":"completed"},"tags":[]}},{"cell_type":"code","source":"del all_eegs, train; gc.collect()\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape:',test.shape)\ntest.head()","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:09.318480Z","iopub.status.busy":"2024-01-19T05:00:09.317292Z","iopub.status.idle":"2024-01-19T05:00:10.396365Z","shell.execute_reply":"2024-01-19T05:00:10.395396Z"},"papermill":{"duration":2.081982,"end_time":"2024-01-19T05:00:10.398354","exception":false,"start_time":"2024-01-19T05:00:08.316372","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:12.296945Z","iopub.status.busy":"2024-01-19T05:00:12.296512Z","iopub.status.idle":"2024-01-19T05:00:12.364281Z","shell.execute_reply":"2024-01-19T05:00:12.363280Z"},"papermill":{"duration":1.009087,"end_time":"2024-01-19T05:00:12.366401","exception":false,"start_time":"2024-01-19T05:00:11.357314","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_eegs2 = {}\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\n\nprint('Processing Test EEG parquets...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n    data = eeg_from_parquet(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = data","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:14.351397Z","iopub.status.busy":"2024-01-19T05:00:14.350719Z","iopub.status.idle":"2024-01-19T05:00:14.886561Z","shell.execute_reply":"2024-01-19T05:00:14.885608Z"},"papermill":{"duration":1.508269,"end_time":"2024-01-19T05:00:14.889267","exception":false,"start_time":"2024-01-19T05:00:13.380998","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER MLP ON TEST\npreds = []\nmodel = build_model()\ntest_gen = DataGenerator(test, shuffle=False, batch_size=64, eegs=all_eegs2,specs=spectrograms2, mode='test')\n\nprint('Inferring test... ',end='')\nfor i in range(5):\n    print(f'fold {i+1}, ',end='')\n    if TRAIN_MODEL:\n        model.load_weights(f'WaveNet_Model/WaveNet_fold{i}.h5')\n    else:\n        model.load_weights(f'/kaggle/input/brain-eegs/WaveNet_Model/WaveNet_fold{i}.h5')\n    pred = model.predict(test_gen, verbose=0)\n    preds.append(pred)\npred = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:16.763660Z","iopub.status.busy":"2024-01-19T05:00:16.762888Z","iopub.status.idle":"2024-01-19T05:00:31.186833Z","shell.execute_reply":"2024-01-19T05:00:31.185619Z"},"papermill":{"duration":15.360495,"end_time":"2024-01-19T05:00:31.188959","exception":false,"start_time":"2024-01-19T05:00:15.828464","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE SUBMISSION.CSV\nfrom IPython.display import display\n\nsub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = pred\nsub.to_csv('submission.csv',index=False)\nprint('Submission shape',sub.shape)\ndisplay( sub.head() )\n\n# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nprint('Sub row 0 sums to:',sub.iloc[0,-6:].sum())","metadata":{"execution":{"iopub.execute_input":"2024-01-19T05:00:33.130247Z","iopub.status.busy":"2024-01-19T05:00:33.129864Z","iopub.status.idle":"2024-01-19T05:00:33.155063Z","shell.execute_reply":"2024-01-19T05:00:33.154175Z"},"papermill":{"duration":0.965817,"end_time":"2024-01-19T05:00:33.157112","exception":false,"start_time":"2024-01-19T05:00:32.191295","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.96707,"end_time":"2024-01-19T05:00:35.323933","exception":false,"start_time":"2024-01-19T05:00:34.356863","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}