{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

CheXNet fine-tuned + model Interpretation

\n
\n\n**CheXNet** [[1]](https://arxiv.org/pdf/1711.05225.pdf) is a 121 layer **DenseNet** developed by Stanford researchers that can detect pneumonia from chest X-rays at a level exceeding practicing radiologists. The weights of the model are uploaded into this notebook and used to train on our data to classify normal vs opacity (typical, atypical, indeterminate) cases. Contrast Limited Adaptive Histogram Equalization (**CLAHE**) is used for preprocessing and some augmentation techniques are applied. For interpretability, **GRAD-CAM** is used to see if the model is paying attention to the opacities (comparing to the groundtruth bounding boxes).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tensorflow.keras.applications import DenseNet121\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import models\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nimport cv2\nimport os\nfrom skimage import exposure\nimport matplotlib\nmatplotlib.rcParams.update({'font.size': 16})\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\nfrom tensorflow.math import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom seaborn import heatmap\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom ast import literal_eval\nfrom matplotlib.patches import Rectangle","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:37:43.287985Z","iopub.execute_input":"2021-06-18T06:37:43.288423Z","iopub.status.idle":"2021-06-18T06:37:43.297229Z","shell.execute_reply.started":"2021-06-18T06:37:43.288376Z","shell.execute_reply":"2021-06-18T06:37:43.296114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ndf_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ndf_study['id'] = df_study['id'].str.replace('_study',\"\")\ndf_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\ndf_train = df_image.merge(df_study, on='StudyInstanceUID')\ndf_train.loc[df_train['Negative for Pneumonia']==1, 'study_label'] = 'negative'\ndf_train.loc[df_train['Typical Appearance']==1, 'study_label'] = 'typical'\ndf_train.loc[df_train['Indeterminate Appearance']==1, 'study_label'] = 'indeterminate'\ndf_train.loc[df_train['Atypical Appearance']==1, 'study_label'] = 'atypical'\ndf_train.drop(['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'], axis=1, inplace=True)\ndf_train['id'] = df_train['id'].str.replace('_image', '.jpg')\ndf_train['image_label'] = df_train['label'].str.split().apply(lambda x : x[0])\ndf_size = pd.read_csv('../input/covid-jpg-512/size.csv')\ndf_train = df_train.merge(df_size, on='id')\ndf_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:37:43.381726Z","iopub.execute_input":"2021-06-18T06:37:43.382093Z","iopub.status.idle":"2021-06-18T06:37:43.49847Z","shell.execute_reply.started":"2021-06-18T06:37:43.38206Z","shell.execute_reply":"2021-06-18T06:37:43.497307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing\n\nSimply using Contrast Limited Adaptive Histogram Equalization (CLAHE) after passing them to the generator","metadata":{}},{"cell_type":"code","source":"train_dir = '../input/covid-jpg-512/train'\n\ndef preprocess_image(img):\n equ_img = exposure.equalize_adapthist(img/255, clip_limit=0.05, kernel_size=24)\n return equ_img\n\ndf_opa = df_train[df_train['image_label']=='opacity'].reset_index()\nfig, axs = plt.subplots(5, 2, figsize=(10,20))\nfig.subplots_adjust(hspace=.2, wspace=.2)\nn=5\nfor i in range(n):\n img = cv2.imread(os.path.join(train_dir, df_opa['id'][i]))\n img_proc = preprocess_image(img)\n axs[i, 0].imshow(img)\n axs[i, 1].imshow(img_proc)\n axs[i, 0].axis('off')\n axs[i, 1].axis('off')\n boxes = literal_eval(df_opa['boxes'][i])\n for box in boxes:\n axs[i, 0].add_patch(Rectangle((box['x']*(512/df_opa['dim1'][i]), box['y']*(512/df_opa['dim0'][i])), box['width']*(512/df_opa['dim1'][i]), box['height']*(512/df_opa['dim0'][i]), fill=0, color='y', linewidth=3))\n axs[i, 0].set_title(df_opa['study_label'][i])\n axs[i, 1].add_patch(Rectangle((box['x']*(512/df_opa['dim1'][i]), box['y']*(512/df_opa['dim0'][i])), box['width']*(512/df_opa['dim1'][i]), box['height']*(512/df_opa['dim0'][i]), fill=0, color='r', linewidth=3))\n axs[i, 1].set_title('After CLAHE')\n \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:37:43.499938Z","iopub.execute_input":"2021-06-18T06:37:43.500288Z","iopub.status.idle":"2021-06-18T06:37:45.508194Z","shell.execute_reply.started":"2021-06-18T06:37:43.500234Z","shell.execute_reply":"2021-06-18T06:37:45.507122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ImageGenerators and Augmentations","metadata":{}},{"cell_type":"code","source":"img_size = 224\nbatch_size = 16\n\nimage_generator = ImageDataGenerator(\n validation_split=0.2,\n horizontal_flip = True,\n zoom_range = 0.15,\n brightness_range = [0.8, 1.2],\n fill_mode='nearest',\n preprocessing_function=preprocess_image\n)\n\nimage_generator_valid = ImageDataGenerator(validation_split=0.2,preprocessing_function=preprocess_image)\n\ntrain_generator = image_generator.flow_from_dataframe(\n dataframe = df_train,\n directory='../input/covid-jpg-512/train',\n x_col = 'id',\n y_col = 'image_label', \n target_size=(img_size, img_size),\n batch_size=batch_size,\n subset='training', seed = 23) \n\nvalid_generator=image_generator_valid.flow_from_dataframe(\n dataframe = df_train,\n directory='../input/covid-jpg-512/train',\n x_col = 'id',\n y_col = 'image_label',\n target_size=(img_size, img_size),\n batch_size=batch_size,\n subset='validation', shuffle=False, seed=23) \n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:37:45.509927Z","iopub.execute_input":"2021-06-18T06:37:45.510244Z","iopub.status.idle":"2021-06-18T06:37:47.975222Z","shell.execute_reply.started":"2021-06-18T06:37:45.510211Z","shell.execute_reply":"2021-06-18T06:37:47.974058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(4):\n aug_images = [train_generator[0][0][j] for i in range(5)]\n fig, axes = plt.subplots(1, 5, figsize=(24,24))\n axes = axes.flatten()\n for img, ax in zip(aug_images, axes):\n ax.imshow(img)\n ax.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:37:47.977369Z","iopub.execute_input":"2021-06-18T06:37:47.97782Z","iopub.status.idle":"2021-06-18T06:38:05.322434Z","shell.execute_reply.started":"2021-06-18T06:37:47.977765Z","shell.execute_reply":"2021-06-18T06:38:05.32133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_process(img, img_size):\n img = load_img(img, target_size = (img_size, img_size))\n img = img_to_array(img)\n img = img.reshape((1, img.shape[0], img.shape[1], img.shape[2]))\n img = preprocess_image(img)\n return img","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:38:05.323893Z","iopub.execute_input":"2021-06-18T06:38:05.324195Z","iopub.status.idle":"2021-06-18T06:38:05.33052Z","shell.execute_reply.started":"2021-06-18T06:38:05.324159Z","shell.execute_reply":"2021-06-18T06:38:05.329461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Architecture","metadata":{}},{"cell_type":"code","source":"chex_weights_path = '../input/chexnet-weights/brucechou1983_CheXNet_Keras_0.3.0_weights.h5'\n\npre_model = DenseNet121(weights=None,\n include_top=False,\n input_shape=(img_size,img_size,3)\n )\nout = Dense(14, activation='sigmoid')(pre_model.output)\npre_model = Model(inputs=pre_model.input, outputs=out) \npre_model.load_weights(chex_weights_path)\npre_model.trainable = False\nx = pre_model.layers[-2].output\nx = GlobalAveragePooling2D()(x)\nx = Dropout(0.1)(x)\noutput = Dense(2, activation='softmax')(x)\nmodel = Model(pre_model.input, output)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:38:05.331976Z","iopub.execute_input":"2021-06-18T06:38:05.332362Z","iopub.status.idle":"2021-06-18T06:38:09.619486Z","shell.execute_reply.started":"2021-06-18T06:38:05.332314Z","shell.execute_reply":"2021-06-18T06:38:09.618311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(Adam(lr=1e-3),loss='binary_crossentropy',metrics='accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:38:09.621072Z","iopub.execute_input":"2021-06-18T06:38:09.6215Z","iopub.status.idle":"2021-06-18T06:38:09.642159Z","shell.execute_reply.started":"2021-06-18T06:38:09.621457Z","shell.execute_reply":"2021-06-18T06:38:09.641239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rlr = ReduceLROnPlateau(monitor = 'val_acc', factor = 0.2, patience = 2, verbose = 1, \n min_delta = 1e-4, min_lr = 1e-4, mode = 'max')\nes = EarlyStopping(monitor = 'val_acc', min_delta = 1e-4, patience = 5, mode = 'max', \n restore_best_weights = True, verbose = 1)\n\nckp = ModelCheckpoint('model.h5',monitor = 'val_acc',\n verbose = 0, save_best_only = True, mode = 'max')\n\nhistory = model.fit(\n train_generator,\n epochs=1,\n validation_data=valid_generator,\n callbacks=[es,rlr, ckp],\n verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:38:09.645492Z","iopub.execute_input":"2021-06-18T06:38:09.645924Z","iopub.status.idle":"2021-06-18T06:47:08.177746Z","shell.execute_reply.started":"2021-06-18T06:38:09.645878Z","shell.execute_reply":"2021-06-18T06:47:08.176626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fine-tuning","metadata":{}},{"cell_type":"code","source":"pre_model.trainable = True\n\nmodel.compile(Adam(lr=1e-5),loss='binary_crossentropy',metrics='accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:08.17961Z","iopub.execute_input":"2021-06-18T06:47:08.180048Z","iopub.status.idle":"2021-06-18T06:47:08.217869Z","shell.execute_reply.started":"2021-06-18T06:47:08.180015Z","shell.execute_reply":"2021-06-18T06:47:08.216741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rlr2 = ReduceLROnPlateau(monitor = 'val_acc', factor = 0.1, patience = 3, verbose = 1, \n min_delta = 1e-4, min_lr = 1e-7, mode = 'max')\n\nes2 = EarlyStopping(monitor = 'val_acc', min_delta = 1e-4, patience = 7, mode = 'max', \n restore_best_weights = True, verbose = 1)\n\nhistory2 = model.fit(\n train_generator,\n epochs=1,\n validation_data=valid_generator,\n callbacks=[es2,rlr2, ckp],\n verbose=1)\n\nK.clear_session()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T06:47:08.219748Z","iopub.execute_input":"2021-06-18T06:47:08.220284Z","iopub.status.idle":"2021-06-18T07:23:16.583361Z","shell.execute_reply.started":"2021-06-18T06:47:08.220219Z","shell.execute_reply":"2021-06-18T07:23:16.581783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model performance","metadata":{}},{"cell_type":"code","source":"actual = valid_generator.labels\npreds = np.argmax(model.predict(valid_generator), axis=1)\ncfmx = confusion_matrix(actual, preds)\nacc = accuracy_score(actual, preds)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:23:16.58618Z","iopub.execute_input":"2021-06-18T07:23:16.586744Z","iopub.status.idle":"2021-06-18T07:24:52.934695Z","shell.execute_reply.started":"2021-06-18T07:23:16.586678Z","shell.execute_reply":"2021-06-18T07:24:52.933597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print ('Test Accuracy:', acc )\nheatmap(cfmx, annot=True, cmap='plasma',\n xticklabels=['Normal','Opacity'],fmt='.0f', yticklabels=['Normal', 'Opacity'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:52.936547Z","iopub.execute_input":"2021-06-18T07:24:52.937308Z","iopub.status.idle":"2021-06-18T07:24:53.153082Z","shell.execute_reply.started":"2021-06-18T07:24:52.937238Z","shell.execute_reply":"2021-06-18T07:24:53.15193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = pd.DataFrame(history.history)\nfig, (ax1, ax2) = plt.subplots(figsize=(12,12),nrows=2, ncols=1)\nhist['loss'].plot(ax=ax1,c='k',label='training loss')\nhist['val_loss'].plot(ax=ax1,c='r',linestyle='--', label='validation loss')\nax1.legend()\nhist['accuracy'].plot(ax=ax2,c='k',label='training accuracy')\nhist['val_accuracy'].plot(ax=ax2,c='r',linestyle='--',label='validation accuracy')\nax2.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:53.154694Z","iopub.execute_input":"2021-06-18T07:24:53.154997Z","iopub.status.idle":"2021-06-18T07:24:53.572198Z","shell.execute_reply.started":"2021-06-18T07:24:53.154968Z","shell.execute_reply":"2021-06-18T07:24:53.571052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = pd.DataFrame(history2.history)\nfig, (ax1, ax2) = plt.subplots(figsize=(12,12),nrows=2, ncols=1)\nhist['loss'].plot(ax=ax1,c='k',label='training loss')\nhist['val_loss'].plot(ax=ax1,c='r',linestyle='--', label='validation loss')\nax1.legend()\nhist['accuracy'].plot(ax=ax2,c='k',label='training accuracy')\nhist['val_accuracy'].plot(ax=ax2,c='r',linestyle='--',label='validation accuracy')\nax2.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:53.573838Z","iopub.execute_input":"2021-06-18T07:24:53.574246Z","iopub.status.idle":"2021-06-18T07:24:53.990023Z","shell.execute_reply.started":"2021-06-18T07:24:53.574201Z","shell.execute_reply":"2021-06-18T07:24:53.98878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Interpretation ","metadata":{}},{"cell_type":"code","source":"def grad_cam(input_image, model, layer_name):\n\n desired_layer = model.get_layer(layer_name)\n grad_model = Model(model.inputs, [desired_layer.output, model.output])\n\n with tf.GradientTape() as tape:\n layer_output, preds = grad_model(input_image)\n ix = (np.argsort(preds, axis=1)[:, -1]).item()\n output_idx = preds[:, ix]\n\n gradient = tape.gradient(output_idx, layer_output)\n alpha_kc = np.mean(gradient, axis=(0,1,2))\n L_gradCam = tf.nn.relu(np.dot(layer_output, alpha_kc)[0])\n L_gradCam = (L_gradCam - np.min(L_gradCam)) / (np.max(L_gradCam) - np.min(L_gradCam)) \n return L_gradCam.numpy()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:53.991921Z","iopub.execute_input":"2021-06-18T07:24:53.99239Z","iopub.status.idle":"2021-06-18T07:24:54.00171Z","shell.execute_reply.started":"2021-06-18T07:24:53.99234Z","shell.execute_reply":"2021-06-18T07:24:54.0001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def blend(img_path, gradCam_img, alpha, colormap = cv2.COLORMAP_JET):\n origin_img = img_to_array(load_img(img_path))\n gradCam_resized = cv2.resize(gradCam_img, (origin_img.shape[1], origin_img.shape[0]), interpolation = cv2.INTER_LINEAR)\n heatmap = cv2.applyColorMap(np.uint8(gradCam_resized*255), colormap)\n superimposed_image = cv2.cvtColor(origin_img.astype('uint8'), cv2.COLOR_RGB2BGR) + heatmap * alpha\n return heatmap, superimposed_image","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:54.003926Z","iopub.execute_input":"2021-06-18T07:24:54.004503Z","iopub.status.idle":"2021-06-18T07:24:54.021073Z","shell.execute_reply.started":"2021-06-18T07:24:54.00445Z","shell.execute_reply":"2021-06-18T07:24:54.019877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_results(model, gen, label=0):\n n = 50\n fig, axs = plt.subplots(10, 5, figsize=(20,60))\n fig.subplots_adjust(hspace=.5, wspace=.1)\n axs = axs.ravel()\n gen.next()\n classes = list(gen.class_indices.keys()) \n if label==0:\n idx = np.array(np.where(np.array(gen.labels) ==0)).ravel()\n else:\n idx = np.array(np.where(np.array(gen.labels) ==1)).ravel()\n \n layer_name = 'relu'\n for i in range(n):\n sample_img_path = os.path.join(train_dir, df_train['id'][idx[i]])\n img = load_process(sample_img_path, img_size)\n pred = model.predict(img)\n grad_cam_img = grad_cam(img, model, layer_name)\n heatmap_img, result_img = blend(sample_img_path, grad_cam_img, 0.5)\n axs[i].imshow(result_img[:,:,::-1]/255)\n axs[i].set_xticklabels([])\n axs[i].set_yticklabels([])\n if type(df_train['boxes'][idx[i]])==str:\n boxes = literal_eval(df_train['boxes'][idx[i]])\n for box in boxes:\n axs[i].add_patch(Rectangle((box['x']*(512/df_train['dim1'][idx[i]]), box['y']*(512/df_train['dim0'][idx[i]])), box['width']*(512/df_train['dim1'][idx[i]]), box['height']*(512/df_train['dim0'][idx[i]]), fill=0, color='y', linewidth=2))\n axs[i].set_title(f\"{df_train['study_label'][idx[i]]}, {df_train['image_label'][idx[i]]}\")\n else:\n axs[i].set_title(df_train['study_label'][idx[i]])\n \n axs[i].set_xlabel(f\"{classes[np.argmax(pred)]}, {round(pred[0][np.argmax(pred)]*100, 2)}%\")","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:54.022633Z","iopub.execute_input":"2021-06-18T07:24:54.023087Z","iopub.status.idle":"2021-06-18T07:24:54.043904Z","shell.execute_reply.started":"2021-06-18T07:24:54.023052Z","shell.execute_reply":"2021-06-18T07:24:54.041886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_results(model, valid_generator,label=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:24:54.045986Z","iopub.execute_input":"2021-06-18T07:24:54.046422Z","iopub.status.idle":"2021-06-18T07:54:07.28254Z","shell.execute_reply.started":"2021-06-18T07:24:54.046374Z","shell.execute_reply":"2021-06-18T07:54:07.281745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_results(model, valid_generator,label=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T07:54:07.283724Z","iopub.execute_input":"2021-06-18T07:54:07.284117Z","iopub.status.idle":"2021-06-18T08:23:38.875068Z","shell.execute_reply.started":"2021-06-18T07:54:07.284086Z","shell.execute_reply":"2021-06-18T08:23:38.873509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_results(model, valid_generator,label=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T08:23:38.877023Z","iopub.execute_input":"2021-06-18T08:23:38.877762Z","iopub.status.idle":"2021-06-18T08:53:07.46268Z","shell.execute_reply.started":"2021-06-18T08:23:38.877703Z","shell.execute_reply":"2021-06-18T08:53:07.461573Z"},"trusted":true},"execution_count":null,"outputs":[]}]}