{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":158958765,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Grad Cam for Kaggle's Brain Comp\nThis notebook displays Grad Cam for the EfficientNetB0 model trained in version 5 of my EfficientNet starter notebook [here][1]. Grad Cam allows us to see where the model is giving its attention to when it makes a prediction. This helps us understand what is important in the spectrogram images. This knowledge helps us improve preprocessing, or improve model architecture, or engineer better features for our ML models like GBT CatBoost. There is a dicussion about this notebook [here][2]\n\n[1]: https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43\n[2]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/472976","metadata":{}},{"cell_type":"markdown","source":"# Initialize 2xT4 GPU","metadata":{}},{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n    \n# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')\n\n###############\n    \nVER = 5\n\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = '/kaggle/input/brain-efficientnet-models-v3-v4-v5/'\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:39:32.978795Z","iopub.execute_input":"2024-02-03T02:39:32.979096Z","iopub.status.idle":"2024-02-03T02:39:49.643481Z","shell.execute_reply.started":"2024-02-03T02:39:32.979056Z","shell.execute_reply":"2024-02-03T02:39:49.642422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data and Create Non-Overlapping Eeg Ids","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\n\n###########\n\ntrain = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:39:49.645351Z","iopub.execute_input":"2024-02-03T02:39:49.645931Z","iopub.status.idle":"2024-02-03T02:39:50.020166Z","shell.execute_reply.started":"2024-02-03T02:39:49.645900Z","shell.execute_reply":"2024-02-03T02:39:50.019206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Train Spectrograms and EEG Spectrograms","metadata":{}},{"cell_type":"code","source":"READ_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()\n    \n###########\n\nREAD_EEG_SPEC_FILES = False\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i%100==0: print(i,', ',end='')\n        x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()\n    \nprint(f'There are {len(all_eegs)} eeg parquets')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:39:50.021444Z","iopub.execute_input":"2024-02-03T02:39:50.021768Z","iopub.status.idle":"2024-02-03T02:41:55.370380Z","shell.execute_reply.started":"2024-02-03T02:39:50.021734Z","shell.execute_reply":"2024-02-03T02:41:55.369257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataLoader","metadata":{}},{"cell_type":"code","source":"import albumentations as albu\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs = spectrograms, eeg_specs = all_eegs): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X) \n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test': \n                r = 0\n            else: \n                r = int( (row['min'] + row['max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n                \n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n            \n        return X,y\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:41:55.373138Z","iopub.execute_input":"2024-02-03T02:41:55.373472Z","iopub.status.idle":"2024-02-03T02:41:57.344119Z","shell.execute_reply.started":"2024-02-03T02:41:55.373445Z","shell.execute_reply":"2024-02-03T02:41:57.342928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Grad Cam Model","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-03T02:41:57.345495Z","iopub.execute_input":"2024-02-03T02:41:57.346111Z","iopub.status.idle":"2024-02-03T02:42:12.995466Z","shell.execute_reply.started":"2024-02-03T02:41:57.346058Z","shell.execute_reply":"2024-02-03T02:42:12.994339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\ndef build_cam_model(pretrain=None):\n    \n    inp = tf.keras.Input(shape=(128,256,8))\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    if pretrain:\n        base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n    \n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n    # EEG SPECTROGRAMS\n    x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = tf.keras.layers.Concatenate(axis=2)([x1,x2])\n    elif USE_EEG_SPECTROGRAMS: x = x2\n    else: x = x1\n    x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x0 = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x0)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=[x,x0])\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt) \n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-02-03T02:42:12.997012Z","iopub.execute_input":"2024-02-03T02:42:12.997375Z","iopub.status.idle":"2024-02-03T02:42:13.038746Z","shell.execute_reply.started":"2024-02-03T02:42:12.997344Z","shell.execute_reply":"2024-02-03T02:42:13.037816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\n\ngkf = GroupKFold(n_splits=5)\nfor fold, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)): \n    # LOAD WEIGHTS INTO GRAD CAM MODEL\n    with strategy.scope():\n        model = build_cam_model()    \n    model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{fold}.h5')\n    layer_weights = model.layers[-1].get_weights()[0][:,0]\n    break\n    \nprint('Using fold 0 model and inferring fold 0 OOF (out of fold) samples...')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:42:13.040113Z","iopub.execute_input":"2024-02-03T02:42:13.041035Z","iopub.status.idle":"2024-02-03T02:42:19.550338Z","shell.execute_reply.started":"2024-02-03T02:42:13.041009Z","shell.execute_reply":"2024-02-03T02:42:19.549147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Grad Cam\nWith grad cam, given a specific OOF (out of fold) train sample, we can view both a model's prediction and where it looked to make this prediction. In the plots below we display the image that was fed into our image model, in my popular starter notebook, there are 8 spectrograms that have been tiled into one 1 input image. \n\nOn the left we have the 4 Kaggle spectrograms where each is 10 minutes long. Each represents one of the 4 montages LL, RL, LP, RP. (Montages explained [here][1]) On the right, we have the 4 EEG spectrograms where each is 50 seconds long. The EEG spectrograms are made from the Magic Formula [here][2]. The spectrograms are each `128x256x1`, so the final concatenation is `512x512x1`.\n\n![](https://raw.githubusercontent.com/cdeotte/Kaggle_Images/main/Feb-2024/key2.png)\n\nEach plot below has 3 subplots. The middle and right subplot use the KEY above. The left subplot is just the Grad Cam image where larger values (more yellow) indicates more attention. The middle subplot is the contours of the Grad Cam's 10% largest values superimposed over the image that we fed into our model. The right subplot is also Grad Cam contour superimposed over image but we add an emboss filter to the image to make the details more visible to humans. For explanations about specific grad cam examples, see discussion [here][3]\n\n[1]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/467877\n[2]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/469760\n[3]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/472976","metadata":{}},{"cell_type":"code","source":"import cv2\n\n# HELPER FUNCTION\ndef mask2contour(mask, width=5):\n    w = mask.shape[1]\n    h = mask.shape[0]\n    mask2 = np.concatenate([mask[:,width:],np.zeros((h,width))],axis=1)\n    mask2 = np.logical_xor(mask,mask2)\n    mask3 = np.concatenate([mask[width:,:],np.zeros((width,w))],axis=0)\n    mask3 = np.logical_xor(mask,mask3)\n    return np.logical_or(mask2,mask3) \n\nclahe = cv2.createCLAHE(clipLimit=16.0, tileGridSize=(8,8))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:42:19.551463Z","iopub.execute_input":"2024-02-03T02:42:19.551759Z","iopub.status.idle":"2024-02-03T02:42:19.561052Z","shell.execute_reply.started":"2024-02-03T02:42:19.551734Z","shell.execute_reply":"2024-02-03T02:42:19.560018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 128\n\nfor ii,tt in enumerate(TARGETS):\n    ttt = tt.split('_')[0].upper()\n    \n    print()\n    print('#'*25)\n    print('###',tt.upper())\n    print('#'*25)\n    \n    # FIND TRAIN SAMPLES IN OOF (OUT OF FOLD) WITH TARGET >= 0.5\n    IDX = train.loc[train.index.isin(valid_index) & (train[tt]>=0.5),TARGETS].index.values\n    print(f'Found {len(IDX)} samples in fold zero OOF for {tt} with true>0.5')\n    \n    # INFER TRAIN SAMPLES WITH MODEL (SAVE PREDS AND ACTIVATIONS)\n    valid_gen = DataGenerator(train.iloc[IDX[:128]], shuffle=False, batch_size=BATCH, mode='valid')\n    p,xx = model.predict(valid_gen,verbose=0)\n    #print(xx.shape)\n    \n    # DISPLAY GRAD CAM\n    for x,y in valid_gen:\n        ct = 0\n        for i in range(BATCH):\n            \n            # FIND SAMPLES WITH PRED >= 0.5 FOR TARGET\n            if i>=len(p): continue\n            pred = p[i]\n            if pred[ii]<0.5: continue\n                \n            # FORMAT PREDICTIONS AS STRING\n            pred2 = ''; true2 = ''\n            true = train.loc[IDX[i]][TARGETS].values\n            for j,t in enumerate(TARGETS):\n                n = t.split('_')[0]\n                pred2 += f' {n}={pred[j]:0.3f}'\n                true2 += f' {n}={true[j]:0.3f}'\n            print()\n            print('==> TRUE:',true2)\n            print('==> PRED:',pred2)\n\n            # PLOT GRAD CAM RESULTS\n            plt.figure(figsize=(20,8))\n\n            # PLOT GRAD CAM IMAGE (PLOT 1 OF 3)\n            plt.subplot(1,3,1)\n            img = np.sum(xx[i,] * layer_weights,axis=-1)\n            img = cv2.resize(img,(512,512))\n            plt.imshow(img[::-1,])\n            plt.title(f'{ttt} - Grad Cam',size=14)\n\n            # FIND GRAD CAM CONTOURS FOR AREAS OF INTEREST\n            cut = np.percentile(img.flatten(), [90])[0]\n            cntr = img.copy()\n            cntr[cntr>=cut] = 100\n            cntr[cntr<cut] = 0\n            cntr = mask2contour(cntr)\n\n            # PLOT EMBOSSED SPECTROGRAMS WITH GRADCAM CONTOURS (PLOT 3 OF 3)\n            plt.subplot(1,3,3)\n            x1 = [x[i,:,:,k:k+1] for k in range(4)] #KAGGLE-SPECS: LL RL LP RP\n            x1 = np.concatenate(x1,axis=0)  \n            x2 = [x[i,:,:,k+4:k+5] for k in range(4)] #EEG-SPECS: LL LP RL RP\n            x2 = np.concatenate(x2,axis=0)\n            x3 = np.concatenate([x1,x2],axis=1)\n            img = cv2.resize(x3,(512,512))\n            img0 = img.copy()\n\n            # EMBOSS IMAGE FOR IMAGE FEATURE VISIBILITY\n            img = img[1:,1:] - img[:-1,:-1] #emboss\n            img -= np.min(img)\n            img /= np.max(img)\n            img = (img*255).astype('uint8')\n            img = cv2.GaussianBlur(img,(5,5),0)\n            img = clahe.apply(img)\n            mx = np.max(img)\n\n            cntr2 = cntr[1:,1:]\n            img[cntr2>0] = mx\n            plt.imshow(img[::-1,])\n            plt.title(f'{ttt} - Embossed Spectrogram with Grad Cam Contours',size=14)\n\n            # PLOT SPECTROGRAMS WITH GRADCAM CONTOURS (PLOT 2 OF 3)\n            plt.subplot(1,3,2)        \n            mx = np.max(img0)\n            img0[cntr>0] = mx\n            plt.imshow(img0[::-1,])\n            plt.title(f'{ttt} - Spectrogram with Grad Cam Contours',size=14)\n\n            plt.show()\n            ct += 1\n            if ct==8: break\n\n        break","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-03T02:42:19.562522Z","iopub.execute_input":"2024-02-03T02:42:19.562809Z","iopub.status.idle":"2024-02-03T02:43:42.794885Z","shell.execute_reply.started":"2024-02-03T02:42:19.562785Z","shell.execute_reply":"2024-02-03T02:43:42.793849Z"},"trusted":true},"execution_count":null,"outputs":[]}]}