{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tf_keras_vis tensorflow lime","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:05:33.044186Z","iopub.execute_input":"2022-11-15T00:05:33.044624Z","iopub.status.idle":"2022-11-15T00:05:49.866193Z","shell.execute_reply.started":"2022-11-15T00:05:33.044545Z","shell.execute_reply":"2022-11-15T00:05:49.864072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%rm gradCAM/* Lime/*","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom tf_keras_vis.saliency import Saliency\nfrom tf_keras_vis.utils.model_modifiers import ReplaceToLinear\nfrom tf_keras_vis.utils.scores import CategoricalScore, BinaryScore\nfrom tf_keras_vis.gradcam_plus_plus import GradcamPlusPlus\nfrom sklearn.model_selection import train_test_split\nfrom skimage.transform import resize\nimport matplotlib.pyplot as plt\nimport imageio.v3 as iio\nimport pandas as pd\nimport numpy as np\nimport time\nimport cv2\nimport os\n# Plot libs\nfrom lime import lime_image\nfrom skimage.segmentation import felzenszwalb, slic, mark_boundaries\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nfrom mpl_toolkits.axes_grid1 import make_axes_locatable\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:05:49.869082Z","iopub.execute_input":"2022-11-15T00:05:49.869545Z","iopub.status.idle":"2022-11-15T00:05:53.191993Z","shell.execute_reply.started":"2022-11-15T00:05:49.869504Z","shell.execute_reply":"2022-11-15T00:05:53.190746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.read_csv(\"./metadata.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:13:47.860401Z","iopub.execute_input":"2022-11-15T00:13:47.862296Z","iopub.status.idle":"2022-11-15T00:13:47.886700Z","shell.execute_reply.started":"2022-11-15T00:13:47.862243Z","shell.execute_reply":"2022-11-15T00:13:47.885509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:13:50.575837Z","iopub.execute_input":"2022-11-15T00:13:50.576297Z","iopub.status.idle":"2022-11-15T00:13:50.587079Z","shell.execute_reply.started":"2022-11-15T00:13:50.576257Z","shell.execute_reply":"2022-11-15T00:13:50.585737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, test_df=train_test_split(final_df, train_size=0.8, shuffle=True, random_state=123)\ndummy_df, test_df = train_test_split(test_df, train_size=0.98, shuffle=True, random_state=123)\nprint(' train_df length: ', len(train_df), '  test_df length: ', len(test_df))  \nprint (train_df.head())\ntrain_samples = train_df['label'].shape[0]\ntest_samples = test_df['label'].shape[0]\ntrain_samples, test_samples","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:13:56.929962Z","iopub.execute_input":"2022-11-15T00:13:56.930407Z","iopub.status.idle":"2022-11-15T00:13:56.950390Z","shell.execute_reply.started":"2022-11-15T00:13:56.930366Z","shell.execute_reply":"2022-11-15T00:13:56.949320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:00.612119Z","iopub.execute_input":"2022-11-15T00:14:00.612538Z","iopub.status.idle":"2022-11-15T00:14:00.622105Z","shell.execute_reply.started":"2022-11-15T00:14:00.612505Z","shell.execute_reply":"2022-11-15T00:14:00.620785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen = ImageDataGenerator(rescale =1./255)\n\ntest_ds = gen.flow_from_dataframe( test_df, x_col='image', y_col='label', target_size=(224, 224), class_mode='binary',\n                                    color_mode='rgb', shuffle=True, batch_size=32)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:02.512187Z","iopub.execute_input":"2022-11-15T00:14:02.513193Z","iopub.status.idle":"2022-11-15T00:14:02.545642Z","shell.execute_reply.started":"2022-11-15T00:14:02.513149Z","shell.execute_reply":"2022-11-15T00:14:02.544345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hit_miss(image_gen, preds):\n    \"\"\"\n    Função utilizada para pegar os erros e acertos das predições.\n\n    Parametros\n    ----------\n        image_gen : DirectoryIterator, DataFrameIterator\n            Iterador das imagens usadas na predição\n        preds : np.ndarray\n            Predições geradas pelo modelo\n    Returns\n    -------\n    dict\n        Um dicionário contendo todos os erros\n    dict\n        Outro dicionário contendo todos os acertos\n    \"\"\"\n\n    labels = image_gen.classes\n    file_paths = image_gen.filepaths\n    map_class = {v: k for k, v in image_gen.class_indices.items()}\n    misses_list = []\n    misses_true_class = []\n    misses_pred_class = []\n    hit_list = []\n    hit_pred_class = []\n\n    for i, p in enumerate(preds):\n        pred_index = 1 if p[0] >= 0.5 else 0\n        true_index = labels[i]\n        if pred_index != true_index:\n            misses_list.append(file_paths[i])\n            misses_true_class.append(map_class[true_index])\n            misses_pred_class.append(map_class[pred_index])\n        else:\n            hit_list.append(file_paths[i])\n            hit_pred_class.append(map_class[pred_index])\n\n    return (\n        {\n            \"misses\": misses_list,\n            \"misses_pred\": misses_pred_class,\n            \"misses_true\": misses_true_class,\n        },\n        {\n            \"hits\": hit_list,\n            \"hits_pred\": hit_pred_class,\n        },\n    )\n\n\ndef get_data_iterator(df, img_size=(224, 224), mode=\"binary\"):\n    \"\"\"\n    Função utilizada para gerar um DataFrameIterator com as instâncias a serem explicadas.\n\n    Parametros\n    ----------\n        df : pd.DataFrame\n            DataFrame contendo as instâncias.\n\n\n        img_size : tuple\n            Tupla contendo as dimensões das imagens\n\n        mode: str\n            String contendo o class_mode para gerar o DataFrameIterator\n    Returns\n    -------\n        DataFrameIterator\n            Um iterador contendo tanto as instâncias.\n    \"\"\"\n\n    gen = ImageDataGenerator(rescale=1.0 / 255)\n    ds = gen.flow_from_dataframe(\n        df,\n        x_col=\"image\",\n        y_col=\"label\",\n        target_size=img_size,\n        class_mode=mode,\n        color_mode=\"rgb\",\n        shuffle=False,\n        batch_size=df.shape[0],\n    )\n\n    return ds\n\n\ndef make_lime_vis(\n    vis_ds,\n    predict_fn,\n    explainer=None,\n    per_batch=None,\n    segmentation_fn=None,\n    prefix=\"Lime\",\n    out_dir=None,\n):\n    \"\"\"\n    Função utilizada para gerar explicações usando LIME.\n\n    Parametros\n    ----------\n        vis_ds: DirectoryIterator, DataFrameIterator\n            Iterador das instâncias a serem explicadas.\n\n        predict_fn: function\n            Função utilizada para classificar as instâncias.\n\n        explainer: LimeImageExplainer\n            Objeto para gerar explicações.\n\n        per_batch: int\n            Quantidade de explicações a serem geradas por batch.\n\n        segmentation_fn: function\n            Função de segmentação a ser utilizada na explicação.\n\n        prefix: str\n            Nome que será colocado antes do nome da instância ao salvar\n\n        out_dir:\n            Diretório onde será armazenado as explicações.\n    \"\"\"\n\n    map_class = {v: k for k, v in vis_ds.class_indices.items()}\n\n    if explainer is None:\n        explainer = lime_image.LimeImageExplainer()\n\n    if per_batch is None:\n        per_batch = vis_ds.batch_size\n\n    if segmentation_fn is None:\n        segmentation_fn = lambda x: felzenszwalb(x, scale=50, sigma=0.5, min_size=50)\n\n    for i in range(len(vis_ds)):\n        images, labels = vis_ds.next()\n        for j in range(per_batch if per_batch <= len(images) else len(images)):\n            explanation = explainer.explain_instance(\n                images[j],\n                predict_fn,\n                top_labels=2,\n                hide_color=0,\n                segmentation_fn=segmentation_fn,\n                num_samples=1000,\n            )\n\n            ind = explanation.top_labels[0]\n            dict_heatmap = dict(explanation.local_exp[ind])\n            heatmap = np.vectorize(dict_heatmap.get)(explanation.segments)\n\n            fig, ax = plt.subplots(1, 2, figsize=(8, 8))\n\n            ax[0].imshow(mark_boundaries(images[j], explanation.segments))\n            img = ax[1].imshow(\n                heatmap, cmap=\"RdBu\", vmin=-heatmap.max(), vmax=heatmap.max()\n            )\n            divider = make_axes_locatable(ax[1])\n            cax = divider.append_axes(\"right\", size=\"5%\", pad=0.15)\n            fig.colorbar(img, cax=cax)\n            fig.suptitle(f\"Predicted:{map_class[labels[j]]}\")\n\n            plt.tight_layout()\n#             fig.savefig(\n#                 f\"./{out_dir}/{prefix}_{vis_ds.filenames[i*vis_ds.batch_size+j].split('/')[-1].split('.')[0]}\"\n#             )\n            plt.show()\n            plt.close(fig)\n\n\ndef make_gradCAM_vis(\n    vis_ds,\n    model,\n    score=\"binary\",\n    gradcam=None,\n    per_batch=None,\n    prefix=\"GradCAM\",\n    out_dir=None,\n):\n    \"\"\"\n    Função utilizada para gerar explicações usando GradCAM.\n\n    Parametros\n    ----------\n        vis_ds: DirectoryIterator, DataFrameIterator\n            Iterador das instâncias a serem explicadas.\n\n        model: tf.keras.Model\n            Modelo treinado a ser explicado\n\n        gradcam: GradcamPlusPlus\n            Gerador das explicações\n\n        per_batch: int\n            Quantidade de explicações a serem geradas por batch.\n\n        prefix: str\n            Nome que será colocado antes do nome da instância ao salvar\n\n        out_dir:\n            Diretório onde será armazenado as explicações.\n    \"\"\"\n    map_class = {v: k for k, v in vis_ds.class_indices.items()}\n    if per_batch is None:\n        per_batch = vis_ds.batch_size\n\n    if gradcam is None:\n        gradcam = GradcamPlusPlus(model, clone=True)\n\n    for i in range(len(vis_ds)):\n        images, labels = vis_ds.next()\n        if score == \"binary\":\n            score = BinaryScore(list(labels))\n        else:\n            score = CategoricalScore(list(labels))\n\n        cam = gradcam(score, images)\n\n        for j in range(per_batch if per_batch <= len(images) else len(images)):\n            heatmap = np.uint8(cm.jet(cam[j])[..., :3] * 255)\n            fig, ax = plt.subplots(1, 2, figsize=(8, 8))\n            ax[0].imshow(images[j])\n            ax[1].imshow(images[j])\n            ax[1].imshow(heatmap, cmap=\"jet\", alpha=0.6)\n\n            fig.suptitle(f\"Predicted:{map_class[labels[j]]}\")\n            plt.tight_layout()\n#             fig.savefig(\n#                 f\"./{out_dir}/{prefix}_{vis_ds.filenames[i*vis_ds.batch_size+j].split('/')[-1].split('.')[0]}\"\n#             )\n            plt.show()\n            plt.close(fig)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:17:02.502257Z","iopub.execute_input":"2022-11-15T00:17:02.502699Z","iopub.status.idle":"2022-11-15T00:17:02.533911Z","shell.execute_reply.started":"2022-11-15T00:17:02.502665Z","shell.execute_reply":"2022-11-15T00:17:02.532393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%mkdir gradCAM Lime","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-15T00:14:11.661310Z","iopub.execute_input":"2022-11-15T00:14:11.661748Z","iopub.status.idle":"2022-11-15T00:14:12.814682Z","shell.execute_reply.started":"2022-11-15T00:14:11.661711Z","shell.execute_reply":"2022-11-15T00:14:12.813527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model(\"./resnet50_model.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:14.410951Z","iopub.execute_input":"2022-11-15T00:14:14.411418Z","iopub.status.idle":"2022-11-15T00:14:16.090653Z","shell.execute_reply.started":"2022-11-15T00:14:14.411374Z","shell.execute_reply":"2022-11-15T00:14:16.089565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_ds)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:16.092782Z","iopub.execute_input":"2022-11-15T00:14:16.093952Z","iopub.status.idle":"2022-11-15T00:14:27.130050Z","shell.execute_reply.started":"2022-11-15T00:14:16.093903Z","shell.execute_reply":"2022-11-15T00:14:27.129082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"misses_dict, hit_dict = get_hit_miss(test_ds, preds)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:27.132156Z","iopub.execute_input":"2022-11-15T00:14:27.132747Z","iopub.status.idle":"2022-11-15T00:14:27.138119Z","shell.execute_reply.started":"2022-11-15T00:14:27.132713Z","shell.execute_reply":"2022-11-15T00:14:27.137029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(zip(hit_dict[\"hits\"], hit_dict[\"hits_pred\"]), columns=[\"image\", \"label\"])\ndf['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:27.139650Z","iopub.execute_input":"2022-11-15T00:14:27.140005Z","iopub.status.idle":"2022-11-15T00:14:27.157709Z","shell.execute_reply.started":"2022-11-15T00:14:27.139974Z","shell.execute_reply":"2022-11-15T00:14:27.156435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nhits_ds = get_data_iterator(df)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:14:33.114647Z","iopub.execute_input":"2022-11-15T00:14:33.115205Z","iopub.status.idle":"2022-11-15T00:14:33.129971Z","shell.execute_reply.started":"2022-11-15T00:14:33.115149Z","shell.execute_reply":"2022-11-15T00:14:33.128639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hits_ds.reset()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:17:08.871717Z","iopub.execute_input":"2022-11-15T00:17:08.872155Z","iopub.status.idle":"2022-11-15T00:17:08.878200Z","shell.execute_reply.started":"2022-11-15T00:17:08.872119Z","shell.execute_reply":"2022-11-15T00:17:08.876905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_gradCAM_vis(hits_ds, model, out_dir=\"./gradCAM\", per_batch=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:17:11.037124Z","iopub.execute_input":"2022-11-15T00:17:11.037601Z","iopub.status.idle":"2022-11-15T00:17:23.222461Z","shell.execute_reply.started":"2022-11-15T00:17:11.037564Z","shell.execute_reply":"2022-11-15T00:17:23.220885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hits_ds.reset()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:17:23.224955Z","iopub.execute_input":"2022-11-15T00:17:23.225419Z","iopub.status.idle":"2022-11-15T00:17:23.233208Z","shell.execute_reply.started":"2022-11-15T00:17:23.225372Z","shell.execute_reply":"2022-11-15T00:17:23.231078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_lime_vis(hits_ds, model.predict, per_batch=1, out_dir=\"./Lime\")","metadata":{"execution":{"iopub.status.busy":"2022-11-15T00:17:31.566568Z","iopub.execute_input":"2022-11-15T00:17:31.567064Z","iopub.status.idle":"2022-11-15T00:18:59.305718Z","shell.execute_reply.started":"2022-11-15T00:17:31.567023Z","shell.execute_reply":"2022-11-15T00:18:59.304273Z"},"trusted":true},"execution_count":null,"outputs":[]}]}