{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow.keras.models import Model\nimport tensorflow.keras.backend as K","metadata":{"_uuid":"e0aaef2d-bd48-4c4a-9495-8d28252f6ebe","_cell_guid":"275f6984-6560-46fc-b79e-9c6fc8602360","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:43:49.983430Z","iopub.execute_input":"2022-02-10T08:43:49.983811Z","iopub.status.idle":"2022-02-10T08:43:49.989760Z","shell.execute_reply.started":"2022-02-10T08:43:49.983742Z","shell.execute_reply":"2022-02-10T08:43:49.988541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# the function responsible for generating the CAMs (Class Activation Maps)","metadata":{"_uuid":"4569ba63-855e-432b-a104-ffd23251a196","_cell_guid":"2a150ee9-e6ca-4446-ab1e-041df029b29a","trusted":true}},{"cell_type":"code","source":"def gen_heatmap_img(img, model0, layer_name='last_conv_layer',viz_img=None,orig_img=None):\n    preds_raw = model0.predict(img[np.newaxis])\n    preds = preds_raw > 0.5\n    class_idx = (preds.astype(int).sum(axis=1) - 1)[0]\n    class_output_tensor = model0.output[:, class_idx]\n    \n    viz_layer = model0.get_layer(layer_name)\n    grads = K.gradients(\n                        class_output_tensor ,\n                        viz_layer.output\n                        )[0] \n    \n    pooled_grads=K.mean(grads,axis=(0,1,2))\n    iterate=K.function([model0.input],[pooled_grads, viz_layer.output[0]])\n    \n    pooled_grad_value, viz_layer_out_value = iterate([img[np.newaxis]])\n    \n    for i in range(pooled_grad_value.shape[0]):\n        viz_layer_out_value[:,:,i] *= pooled_grad_value[i]\n    \n    heatmap = np.mean(viz_layer_out_value, axis=-1)\n    heatmap = np.maximum(heatmap,0)\n    heatmap /= np.max(heatmap)\n\n    viz_img=cv2.resize(viz_img,(img.shape[1],img.shape[0]))\n    heatmap=cv2.resize(heatmap,(viz_img.shape[1],viz_img.shape[0]))\n    \n    heatmap_color = cv2.applyColorMap(np.uint8(heatmap*255), cv2.COLORMAP_HOT)/255\n    heated_img = heatmap_color*0.5 + viz_img*0.5\n    \n\n    print('raw output from model : ')\n    print_pred(preds_raw)\n    \n    if orig_img is None:\n        show_Nimages([img,viz_img,heatmap_color,heated_img])\n    else:\n        show_Nimages([orig_img,img,viz_img,heatmap_color,heated_img])\n\n    plt.show()\n    return heated_img,heatmap_color,heatmap","metadata":{"_uuid":"2b8e2cee-25c9-43bb-91c3-461cd9f6bdb4","_cell_guid":"a9b116b3-4624-488b-912a-73c6db5b77d4","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:43:49.991936Z","iopub.execute_input":"2022-02-10T08:43:49.992585Z","iopub.status.idle":"2022-02-10T08:43:50.012283Z","shell.execute_reply.started":"2022-02-10T08:43:49.992526Z","shell.execute_reply":"2022-02-10T08:43:50.011479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here are simple tools to easily show images and prediction","metadata":{"_uuid":"8486b8c9-316e-4bb3-8415-8a9bfd96caab","_cell_guid":"9d21afc6-2717-42e3-ae83-9d42239b8064","trusted":true}},{"cell_type":"code","source":"def show_image(image,figsize=None,title=None):\n    \n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n\n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    else:\n        plt.imshow(image)\n        \n    if title is not None:\n        plt.title(title)\n\ndef show_Nimages(imgs,scale=1):\n\n    N=len(imgs)\n    fig = plt.figure(figsize=(25/scale, 16/scale))\n    for i, img in enumerate(imgs):\n        ax = fig.add_subplot(1, N, i + 1, xticks=[], yticks=[])\n        show_image(img)\n        \ndef print_pred(array_of_classes):\n    xx = array_of_classes\n    s1,s2 = xx.shape\n    for i in range(s1):\n        for j in range(s2):\n            print('%.3f ' % xx[i,j],end='')\n        print('')","metadata":{"_uuid":"ac94e23c-7bd5-4a78-a61f-cdcebcaab5e0","_cell_guid":"556c5f6d-a318-4d87-bdc5-07b925d93ecc","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:43:50.014243Z","iopub.execute_input":"2022-02-10T08:43:50.014831Z","iopub.status.idle":"2022-02-10T08:43:50.028032Z","shell.execute_reply.started":"2022-02-10T08:43:50.014678Z","shell.execute_reply":"2022-02-10T08:43:50.027226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First let us test the first 5 test examples. For each test example, I show original input, a preprocessed input, heatmap and combined heatmap respectively.","metadata":{"_uuid":"9a6325e3-6dca-4d18-bb52-731abdf535a5","_cell_guid":"1f6c43c8-3b03-413b-a2d7-1f90fb45b051","trusted":true}},{"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.callbacks import Callback, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nfrom tqdm import tqdm\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-02-10T08:43:50.029902Z","iopub.execute_input":"2022-02-10T08:43:50.030274Z","iopub.status.idle":"2022-02-10T08:43:50.044336Z","shell.execute_reply.started":"2022-02-10T08:43:50.030193Z","shell.execute_reply":"2022-02-10T08:43:50.043506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n#shutil.rmtree(\"/kaggle/working/cropped\")","metadata":{"execution":{"iopub.status.busy":"2022-02-10T08:43:50.047045Z","iopub.execute_input":"2022-02-10T08:43:50.047740Z","iopub.status.idle":"2022-02-10T08:43:50.053282Z","shell.execute_reply.started":"2022-02-10T08:43:50.047682Z","shell.execute_reply":"2022-02-10T08:43:50.052629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/cropped')","metadata":{"execution":{"iopub.status.busy":"2022-02-10T08:43:50.056765Z","iopub.execute_input":"2022-02-10T08:43:50.057012Z","iopub.status.idle":"2022-02-10T08:43:50.063412Z","shell.execute_reply.started":"2022-02-10T08:43:50.056958Z","shell.execute_reply":"2022-02-10T08:43:50.062668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')","metadata":{"_uuid":"8f54e55c-fcdf-4675-9a14-b9ddc58e3670","_cell_guid":"33498596-1c1a-4920-b339-36163ff71bea","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:43:50.064960Z","iopub.execute_input":"2022-02-10T08:43:50.065249Z","iopub.status.idle":"2022-02-10T08:43:50.085635Z","shell.execute_reply.started":"2022-02-10T08:43:50.065200Z","shell.execute_reply":"2022-02-10T08:43:50.084956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nmodel = tf.keras.models.load_model('../input/aptos-densenet-model/desenet_aptos_final.h5')","metadata":{"_uuid":"a365535f-4d05-4deb-ba31-cce0130cf977","_cell_guid":"8b41b5e5-6dff-4789-9034-531845ac51c4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:43:50.087740Z","iopub.execute_input":"2022-02-10T08:43:50.088054Z","iopub.status.idle":"2022-02-10T08:44:33.175795Z","shell.execute_reply.started":"2022-02-10T08:43:50.088006Z","shell.execute_reply":"2022-02-10T08:44:33.174989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_image(image_path, desired_size=224):\n    im = Image.open(image_path)\n    im = im.resize((desired_size, desired_size ), resample=Image.LANCZOS)\n    \n    return im","metadata":{"_uuid":"b69d6702-b149-491b-ba0e-4c6ef25b274f","_cell_guid":"ac16b97c-0dc8-466b-913e-5dc77fb848f0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:44:33.178304Z","iopub.execute_input":"2022-02-10T08:44:33.178550Z","iopub.status.idle":"2022-02-10T08:44:33.187393Z","shell.execute_reply.started":"2022-02-10T08:44:33.178507Z","shell.execute_reply":"2022-02-10T08:44:33.186660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_heatmap_img(img, model0, layer_name='last_conv_layer',viz_img=None,orig_img=None):\n    preds_raw = model0.predict(img[np.newaxis])\n    preds = preds_raw > 0.5\n    class_idx = (preds.astype(int).sum(axis=1) - 1)[0]\n    class_output_tensor = model0.output[:, class_idx]\n    \n    viz_layer = model0.get_layer(layer_name)\n    grads = K.gradients(\n                        class_output_tensor ,\n                        viz_layer.output\n                        )[0] \n    \n    pooled_grads=K.mean(grads,axis=(0,1,2))\n    iterate=K.function([model0.input],[pooled_grads, viz_layer.output[0]])\n    \n    pooled_grad_value, viz_layer_out_value = iterate([img[np.newaxis]])\n    \n    for i in range(pooled_grad_value.shape[0]):\n        viz_layer_out_value[:,:,i] *= pooled_grad_value[i]\n    \n    heatmap = np.mean(viz_layer_out_value, axis=-1)\n    heatmap = np.maximum(heatmap,0)\n    heatmap /= np.max(heatmap)\n\n    viz_img=cv2.resize(viz_img,(img.shape[1],img.shape[0]))\n    heatmap=cv2.resize(heatmap,(viz_img.shape[1],viz_img.shape[0]))\n    \n    heatmap_color = cv2.applyColorMap(np.uint8(heatmap*255), cv2.COLORMAP_HOT)/255\n    heated_img = heatmap_color*0.5 + viz_img*0.5\n    \n    return heated_img,heatmap_color,heatmap","metadata":{"_uuid":"7b964d61-0f76-46d2-9fa9-83016017b934","_cell_guid":"8e6f0f34-c983-4103-804b-a591ec67f2de","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:44:33.190287Z","iopub.execute_input":"2022-02-10T08:44:33.191868Z","iopub.status.idle":"2022-02-10T08:44:33.204230Z","shell.execute_reply.started":"2022-02-10T08:44:33.191818Z","shell.execute_reply":"2022-02-10T08:44:33.203218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image_ben_orig(path,resize=True,crop=False,norm255=True,keras=False):\n    image = cv2.imread(path)\n    \n    \n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        \n    image=cv2.addWeighted( image,4, cv2.GaussianBlur( image , (0,0) ,  10) ,-4 ,128)\n    \n    if norm255:\n        return image/255\n    elif keras:\n        \n        image = np.expand_dims(image, axis=0)\n        return preprocess_input(image)[0]\n    else:\n        return image.astype(np.int16)\n    \n    return image\n\ndef transform_image_ben(img,resize=True,crop=False,norm255=True,keras=False):  \n    image=cv2.addWeighted( img,4, cv2.GaussianBlur( img , (0,0) ,  10) ,-4 ,128)\n    \n    if norm255:\n        return image/255\n    elif keras:\n        image = np.expand_dims(image, axis=0)\n        return preprocess_input(image)[0]\n    else:\n        return image.astype(np.int16)\n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2022-02-10T08:44:33.205742Z","iopub.execute_input":"2022-02-10T08:44:33.206315Z","iopub.status.idle":"2022-02-10T08:44:33.219328Z","shell.execute_reply.started":"2022-02-10T08:44:33.206199Z","shell.execute_reply":"2022-02-10T08:44:33.218317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"layer_name = 'relu' #'conv5_block16_concat'\n","metadata":{"execution":{"iopub.status.busy":"2022-02-10T08:44:33.220906Z","iopub.execute_input":"2022-02-10T08:44:33.221428Z","iopub.status.idle":"2022-02-10T08:44:33.230677Z","shell.execute_reply.started":"2022-02-10T08:44:33.221261Z","shell.execute_reply":"2022-02-10T08:44:33.229871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nNUM_SAMP=3662\nstart_index=0\nDATA_PATH = '../input/aptos2019-blindness-detection'\nTRAIN_IMG_PATH = os.path.join(DATA_PATH, 'train_images')\nlayer_name = 'relu' #'conv5_block16_concat'\nfor i, (idx, row) in enumerate(train_df[:NUM_SAMP].iterrows()):\n    index = idx\n    index += start_index\n    path = os.path.join(TRAIN_IMG_PATH, train_df['id_code'][index]+'.png')\n    ben_img = load_image_ben_orig(path)\n    input_img = np.empty((1,224, 224, 3), dtype=np.uint8)\n    input_img[0,:,:,:] = resize_image(path)\n        \n    [heatmaped, heatmap_col,heatmap] = gen_heatmap_img(input_img[0],\n                            model, layer_name=layer_name,viz_img=ben_img)\n    img_rect = cv2.imread(path)\n    img_rect = cv2.resize(img_rect, (224,224))\n    img_rect = cv2.cvtColor(img_rect, cv2.COLOR_BGR2RGB)\n\n\n    img_test = heatmap_col\n    img_test=img_test.astype(np.uint8)\n\n    img_t = cv2.cvtColor(img_test, cv2.COLOR_RGB2GRAY)\n\n    ret, binary = cv2.threshold(img_t, 0, 255, cv2.THRESH_BINARY)\n\n\n    contours, h = cv2.findContours(\n         binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n    \n    a = np.zeros(len(contours)*2)\n    b= np.zeros(len(contours)*2)\n    j=0\n    for c in contours:\n        if len(contours)<2:\n            x, y, w, h = cv2.boundingRect(c)\n     # draw a green rectangle to visualize the bounding rect\n            cv2.rectangle( img_test, (x, y), (x+w, y+h), (0, 255, 0), 1)\n            cv2.rectangle( img_rect, (x, y), (x+w, y+h), (0, 255, 0), 1)\n            cropped_image = img_rect[y:y+h, x:x+w]\n            cropped_image = cv2.resize(cropped_image, (224,224))\n        else:\n            x, y, w, h = cv2.boundingRect(c)\n     # draw a green rectangle to visualize the bounding rec\n            a[j]=x\n            b[j]=y\n            j=j+1\n            a[j]=x+w\n            b[j]=y+h\n        j=j+1\n\n    if len(contours)<2:\n        cropped_image = img_rect[y:y+h, x:x+w]\n        cropped_image = cv2.resize(cropped_image, (224,224))\n    else:\n        x1=int(max(a))\n        y1=int(max(b))\n        x2=int(min(a))\n        y2=int(min(b))\n        w1=x1-x2\n        h1=y1-y2\n        cv2.rectangle( img_rect, (x2, y2), (x2+w1, y2+h1), (0, 255, 0), 1)\n\n        cropped_image = img_rect[y2:y2+h1, x2:x2+w1]\n        cropped_image = cv2.resize(cropped_image, (224,224))\n        \n        \n    filename = '/kaggle/working/cropped/'+'cropped' + str(i) + '.jpg'\n\n    cv2.imwrite(filename, cropped_image)\n    ","metadata":{"_uuid":"af308c08-969a-479a-9183-83b68bc6b189","_cell_guid":"bbce61ba-605d-478e-9035-1cae67fc29bc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-10T08:44:33.233584Z","iopub.execute_input":"2022-02-10T08:44:33.233874Z","iopub.status.idle":"2022-02-10T11:56:18.689522Z","shell.execute_reply.started":"2022-02-10T08:44:33.233828Z","shell.execute_reply":"2022-02-10T11:56:18.688270Z"},"trusted":true},"execution_count":null,"outputs":[]}]}