{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\n### Histopathologic Cancer Detection. \n\nIn this competition, we had to build and train a Convolutional Neural Network nodel which can accurately identify metastatic cancer in small image patches taken from larger digital pathology scans. \n\nThe data for this competition is a slightly modified version of the PatchCamelyon (PCam) benchmark dataset. In this dataset, we are provided with a large number of small pathology images to classify. Files are named with an image id. The train_labels.csv file provides the ground truth for the images in the train folder. We are predicting the labels for the images in the test folder. A positive label indicates that the center 32x32px region of a patch contains at least one pixel of tumor tissue. Tumor tissue in the outer region of the patch does not influence the label. This outer region is provided to enable fully-convolutional models that do not use zero-padding, to ensure consistent behavior when applied to a whole-slide image.","metadata":{}},{"cell_type":"markdown","source":"# Import namespaces","metadata":{}},{"cell_type":"code","source":"# Import Numpy for array operations, panda for dataframe and matplotlib for plotting.\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n\n# SciKitLearn for Confusion Matrix and other statistics\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score, roc_auc_score, plot_confusion_matrix\n\n# Tensorflow and Keras to build CNN Model Architechture\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# Pickle and os for saving the files\nimport pickle\nimport os\nimport itertools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-12T14:16:28.991504Z","iopub.execute_input":"2021-12-12T14:16:28.991890Z","iopub.status.idle":"2021-12-12T14:16:34.554290Z","shell.execute_reply.started":"2021-12-12T14:16:28.991819Z","shell.execute_reply":"2021-12-12T14:16:34.553538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters\nThis param will be used to reduce the test and validation datasets so that the notebook can run faster.","metadata":{}},{"cell_type":"code","source":"SAMPLE_SIZE = 0.8","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:34.557103Z","iopub.execute_input":"2021-12-12T14:16:34.557676Z","iopub.status.idle":"2021-12-12T14:16:34.561585Z","shell.execute_reply.started":"2021-12-12T14:16:34.557626Z","shell.execute_reply":"2021-12-12T14:16:34.560946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\nIMG_SIZE = 96\nRANDOM_SEED = 1","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:34.563767Z","iopub.execute_input":"2021-12-12T14:16:34.564029Z","iopub.status.idle":"2021-12-12T14:16:34.579860Z","shell.execute_reply.started":"2021-12-12T14:16:34.563992Z","shell.execute_reply":"2021-12-12T14:16:34.579186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET = \"../input/histopathologic-cancer-detection/\"\ntrain_path = DATASET+\"train\"\ntest_path = DATASET+\"test\"","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:34.582212Z","iopub.execute_input":"2021-12-12T14:16:34.582852Z","iopub.status.idle":"2021-12-12T14:16:34.589092Z","shell.execute_reply.started":"2021-12-12T14:16:34.582791Z","shell.execute_reply":"2021-12-12T14:16:34.588374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation dataset\nSince the validation dataset has the true label column which is missing in test dataset, We will use it to evaluate the final model. \nWe will find out accuracy, confusion matrix, classification reports etc","metadata":{}},{"cell_type":"code","source":"valid = pd.read_csv(DATASET+'train_labels.csv', dtype=str)\nvalid['path'] = valid.id+'.tif'\nvalid.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:34.590546Z","iopub.execute_input":"2021-12-12T14:16:34.591181Z","iopub.status.idle":"2021-12-12T14:16:35.124379Z","shell.execute_reply.started":"2021-12-12T14:16:34.591143Z","shell.execute_reply":"2021-12-12T14:16:35.123704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation Dataset Size:', valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.125636Z","iopub.execute_input":"2021-12-12T14:16:35.126040Z","iopub.status.idle":"2021-12-12T14:16:35.131108Z","shell.execute_reply.started":"2021-12-12T14:16:35.126004Z","shell.execute_reply":"2021-12-12T14:16:35.130333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reduce the validation dataset to run the notebook faster.","metadata":{}},{"cell_type":"code","source":"# Reduce the Validation Dataset size to run the predictions faster\nvalid, ignore = train_test_split(valid, test_size=SAMPLE_SIZE, random_state=RANDOM_SEED, stratify=valid.label)\nprint('Validation Dataset Size:', valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.132618Z","iopub.execute_input":"2021-12-12T14:16:35.133060Z","iopub.status.idle":"2021-12-12T14:16:35.513701Z","shell.execute_reply.started":"2021-12-12T14:16:35.133004Z","shell.execute_reply":"2021-12-12T14:16:35.512924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution\nIn the validation image dataset, close to 60% are benign and and 40% are malignant. ","metadata":{}},{"cell_type":"code","source":"categories = [\"Benign\", \"Malignant\"]\nplt.figure(figsize=(10,10));\nplt.pie(valid.label.value_counts(), labels=categories, startangle=55, \n        autopct='%1.2f%%', colors=sns.color_palette('flare')[0:2], shadow=True);\nplt.axis('off')\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.515013Z","iopub.execute_input":"2021-12-12T14:16:35.515682Z","iopub.status.idle":"2021-12-12T14:16:35.652288Z","shell.execute_reply.started":"2021-12-12T14:16:35.515650Z","shell.execute_reply":"2021-12-12T14:16:35.651613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Dataset\nWe will use the test dataset to find out the probability distribution of the predictions.","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(DATASET+'sample_submission.csv', dtype=str)\ntest['path'] = test.id+'.tif'\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.653671Z","iopub.execute_input":"2021-12-12T14:16:35.654147Z","iopub.status.idle":"2021-12-12T14:16:35.831247Z","shell.execute_reply.started":"2021-12-12T14:16:35.654107Z","shell.execute_reply":"2021-12-12T14:16:35.830439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation Dataset Size:', valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.834537Z","iopub.execute_input":"2021-12-12T14:16:35.834783Z","iopub.status.idle":"2021-12-12T14:16:35.838793Z","shell.execute_reply.started":"2021-12-12T14:16:35.834750Z","shell.execute_reply":"2021-12-12T14:16:35.838009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reduce the test dataset to run the notebook faster.","metadata":{}},{"cell_type":"code","source":"# Reduce the test Dataset size to run the predictions faster\ntest, ignore = train_test_split(test, test_size=SAMPLE_SIZE, random_state=RANDOM_SEED, stratify=test.label)\nprint('Test Dataset Size:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.840387Z","iopub.execute_input":"2021-12-12T14:16:35.840655Z","iopub.status.idle":"2021-12-12T14:16:35.940556Z","shell.execute_reply.started":"2021-12-12T14:16:35.840613Z","shell.execute_reply":"2021-12-12T14:16:35.939873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Model (VGG16)\nWe have built so many models thru this past six weeks. We selected VGG16 as the final model for submission because of the private score of 0.93 and and public score of 0.94. Though the PyTorch model achieved higher public score it is not able to matchup in terms of private score. Even the Ensemble model has a better public score but the time it takes to train so many models and then ensemble them is huge compared to the score gains.\n\n| Model Architecture  \t| Image Size  | Epochs  \t| Private Score  \t| Public Score   | Notebook  | \n|---\t|---\t|---\t|---\t|---\t|---\t|\n| Simple Model  | 96x96  \t| 30  \t| [0.8288](https://www.kaggle.com/leopoldtchomgwi/lt-cancer-detection-v01-submission-revised)  \t| 0.8669  \t|[[LT] Cancer Detection Simple Model](https://www.kaggle.com/leopoldtchomgwi/lt-cancerdetection-v01-with-balanced-target-dist)   \t|\n| VGG16  \t| 96x96  | 60  \t| [0.9376](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-models-submission?scriptVersionId=81777457)  \t| 0.9449  \t| [[LP] Cancer Detection VGG16 Model](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-vgg16-model)  |\n| ResNet50  \t| 96x96  | 40  \t| [0.9029](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-models-submission?scriptVersionId=81778571)  \t| 0.8798  \t| [[LP] Cancer Detection ResNet50 Model](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-resnet50-model)  |\n| Ensemble VGG16, ResNet50, Simple Model <br/> with Logistic Regression | 96x96  | 40   | [0.8370](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-ensemble-model-with-lr?scriptVersionId=81787013) | 0.8569  \t|  [[LP] Cancer Detection Ensemble Model with LR](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-ensemble-model-with-lr)\t|\n| Ensemble VGG16, ResNet50, Simple Model <br/> with Weighted Average | 96x96  | 40  |  [0.9298](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-models-submission?scriptVersionId=81797079) \t|  0.9537 \t|  [[LP] Cancer Detection Models Submission](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-models-submission?scriptVersionId=81797079)  \t|\n| PyTorch  \t| 96x96  | 40  |  [0.9263](https://www.kaggle.com/lokanathpatro/lp-cancer-detection-cnn-model-with-pytorch?scriptVersionId=81689603) \t| 0.9642  \t| [[LP] Cancer Detection CNN Model with PyTorch]( https://www.kaggle.com/lokanathpatro/lp-cancer-detection-cnn-model-with-pytorch)  |\n| Cropping2D Layer Model  \t| 96x96  | 25  |  [0.8368](https://www.kaggle.com/gauravsamudra/gs-cancersubmission-v1?scriptVersionId=79504395) \t| 0.8958  \t| [[GS] Cancer Detection with Cropping2D Layer](https://www.kaggle.com/gauravsamudra/gs-cancerdetection-v01?scriptVersionId=79421784)  |\n| Model With Pre-Cropped Images  \t| 32x32  | 25  |  [0.7224](https://www.kaggle.com/leopoldtchomgwi/lt-cancer-detection-cropped-images-submission?scriptVersionId=79658223) \t| 0.7813  \t| [[LT] Cancer Detection with Pre-Cropped Images](https://www.kaggle.com/leopoldtchomgwi/lt-cancer-detection-cropped-im)  |\n| Model With Augmented Images  \t| 96x96  | 25  |  [0.8585](https://www.kaggle.com/jennaward6/team-4-cancer-detection-submit-jw-edit?scriptVersionId=80319158) \t| 0.9124  \t| [[JW] Cancer Detection with Augmented Images](https://www.kaggle.com/jennaward6/jw-cancerdetection-v01-with-visualizations?scriptVersionId=80269876)  |\n| Simple Model2  \t| 96x96  | 20  |  [0.9231](https://www.kaggle.com/leopoldtchomgwi/cancerdetection-submission-final2?scriptVersionId=81039203) \t| 0.9397  \t| [[LT] Cancer Detection_Simple Model](https://colab.research.google.com/drive/1_PvSVhBlYdwXEAqbb7a8RABTXgcU14ME?usp=sharing#scrollTo=yYkbG_C6KJ1Y)  |","metadata":{}},{"cell_type":"code","source":"cnn = keras.models.load_model('../input/lp-cancer-detection-vgg16-model-after-40epochs/LP_HCD_VGG16_Model.h5')\nkeras.utils.plot_model(cnn,show_shapes=True,show_dtype=True,show_layer_names=True,dpi=60)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:35.941803Z","iopub.execute_input":"2021-12-12T14:16:35.942184Z","iopub.status.idle":"2021-12-12T14:16:43.136453Z","shell.execute_reply.started":"2021-12-12T14:16:35.942141Z","shell.execute_reply":"2021-12-12T14:16:43.135572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training history","metadata":{}},{"cell_type":"code","source":"pickle_file = open(\"../input/lp-cancer-detection-vgg16-model-after-40epochs/LP_HCD_VGG16_Model_History.pkl\", \"rb\")\nhistory = pickle.load(pickle_file)\npickle_file.close()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:43.138201Z","iopub.execute_input":"2021-12-12T14:16:43.138617Z","iopub.status.idle":"2021-12-12T14:16:43.157408Z","shell.execute_reply.started":"2021-12-12T14:16:43.138577Z","shell.execute_reply":"2021-12-12T14:16:43.156782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[20,5])\nsns.set_style(\"darkgrid\")\nplt.subplot(1,3,1)\nsns.lineplot(x=epoch_range,y=history['loss'], label='Training')\nsns.lineplot(x=epoch_range,y=history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nsns.set_style(\"darkgrid\")\nplt.subplot(1,3,2)\nsns.lineplot(x=epoch_range,y=history['accuracy'], label='Training')\nsns.lineplot(x=epoch_range,y=history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nsns.set_style(\"darkgrid\")\nplt.subplot(1,3,3)\nsns.lineplot(x=epoch_range,y=history['auc'], label='Training')\nsns.lineplot(x=epoch_range,y=history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:43.158715Z","iopub.execute_input":"2021-12-12T14:16:43.159117Z","iopub.status.idle":"2021-12-12T14:16:44.036639Z","shell.execute_reply.started":"2021-12-12T14:16:43.159083Z","shell.execute_reply":"2021-12-12T14:16:44.035984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Generator","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1/255)\n\nvalid_loader = datagen.flow_from_dataframe(\n    dataframe = valid,\n    directory = train_path,\n    x_col = 'path',\n    batch_size = BATCH_SIZE,\n    shuffle = False,\n    class_mode = None,\n    target_size = (IMG_SIZE,IMG_SIZE)\n)\n\ntest_loader = datagen.flow_from_dataframe(\n    dataframe = test,\n    directory = test_path,\n    x_col = 'path',\n    batch_size = BATCH_SIZE,\n    shuffle = False,\n    class_mode = None,\n    target_size = (IMG_SIZE,IMG_SIZE)\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:44.037917Z","iopub.execute_input":"2021-12-12T14:16:44.038279Z","iopub.status.idle":"2021-12-12T14:16:46.204755Z","shell.execute_reply.started":"2021-12-12T14:16:44.038244Z","shell.execute_reply":"2021-12-12T14:16:46.204010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test prdictions","metadata":{}},{"cell_type":"code","source":"test_probs = cnn.predict(test_loader)\nprint(test_probs.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:46.206269Z","iopub.execute_input":"2021-12-12T14:16:46.206530Z","iopub.status.idle":"2021-12-12T14:16:57.056264Z","shell.execute_reply.started":"2021-12-12T14:16:46.206495Z","shell.execute_reply":"2021-12-12T14:16:57.055502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Probability Distribution plot","metadata":{}},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nsns.displot(x=test_probs[:,1], height=7, stat='percent', kde=True)\nplt.xlabel('Probability')\nplt.title(\"Probability Distribution Plot (Test Dataset)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:28:20.139771Z","iopub.execute_input":"2021-12-12T14:28:20.140456Z","iopub.status.idle":"2021-12-12T14:28:20.523016Z","shell.execute_reply.started":"2021-12-12T14:28:20.140420Z","shell.execute_reply":"2021-12-12T14:28:20.522318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation predictions","metadata":{}},{"cell_type":"code","source":"valid_probs = cnn.predict(valid_loader)\nprint(valid_probs.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:16:57.434316Z","iopub.execute_input":"2021-12-12T14:16:57.434557Z","iopub.status.idle":"2021-12-12T14:17:06.983848Z","shell.execute_reply.started":"2021-12-12T14:16:57.434524Z","shell.execute_reply":"2021-12-12T14:17:06.982517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid['label'] = pd.to_numeric(valid['label'])\nvalid['prediction'] = np.argmax(valid_probs, axis=1)\nvalid['probability'] = valid_probs[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:06.986521Z","iopub.execute_input":"2021-12-12T14:17:06.987307Z","iopub.status.idle":"2021-12-12T14:17:06.996106Z","shell.execute_reply.started":"2021-12-12T14:17:06.987240Z","shell.execute_reply":"2021-12-12T14:17:06.995045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:24:23.282050Z","iopub.execute_input":"2021-12-12T14:24:23.282506Z","iopub.status.idle":"2021-12-12T14:24:23.315130Z","shell.execute_reply.started":"2021-12-12T14:24:23.282458Z","shell.execute_reply":"2021-12-12T14:24:23.313967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Probability Distribution Plot","metadata":{}},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nsns.displot(data=valid, x='probability', height=7, stat='percent', kde=True)\nplt.xlabel('Probability')\nplt.title(\"Probability Distribution Plot (Validation Dataset)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:35:26.817450Z","iopub.execute_input":"2021-12-12T14:35:26.818050Z","iopub.status.idle":"2021-12-12T14:35:27.447081Z","shell.execute_reply.started":"2021-12-12T14:35:26.818011Z","shell.execute_reply":"2021-12-12T14:35:27.446393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test vs Validation Probability Distribution Plot ","metadata":{}},{"cell_type":"code","source":"sns.set(rc = {'figure.figsize':(15,8)})\nsns.set_style(\"darkgrid\")\nsns.histplot(data=valid, x='probability', color=\"skyblue\", label=\"Sepal Length\", kde=True)\nsns.histplot(x=test_probs[:,1], color=\"teal\", label=\"Sepal Length\", kde=True)\nplt.xlabel('Probability')\nplt.title(\"Probability Distribution Plot (Test vs Validation)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:34:52.466868Z","iopub.execute_input":"2021-12-12T14:34:52.467383Z","iopub.status.idle":"2021-12-12T14:34:52.818878Z","shell.execute_reply.started":"2021-12-12T14:34:52.467341Z","shell.execute_reply":"2021-12-12T14:34:52.817807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Accuracy\n\nAccuracy is one metric for evaluating classification models. Informally, accuracy is the fraction of predictions our model got right. Formally, accuracy has the following definition. \n\n$\nAccuracy=\\frac{Number of correct predictions}{Number of total predictons}\n$","metadata":{}},{"cell_type":"code","source":"accuracy = accuracy_score(valid.label, valid.prediction)\nprint('For our model, the accuracy is %f' % accuracy)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:07.413543Z","iopub.execute_input":"2021-12-12T14:17:07.413886Z","iopub.status.idle":"2021-12-12T14:17:07.420962Z","shell.execute_reply.started":"2021-12-12T14:17:07.413849Z","shell.execute_reply":"2021-12-12T14:17:07.419957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix w/o Bias\n\nA confusion matrix tell us the percentage of examples from each class in our test set that our model predicted correctly. In the case of an imbalanced dataset like the one we're dealing with, this is a better measure of our model's performance than overall accuracy.","metadata":{}},{"cell_type":"code","source":"def getLabels(cm):    \n    group_names = [\"True Benign\",\"False Malignant\",\"False Benign\",\"True Malignant\"]\n    group_counts = [\"{0:0.0f}\".format(value) for value in cm.flatten()]\n    group_percentages = [\"{0:.2%}\".format(value) for value in cm.flatten()/np.sum(cm)]\n    labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\n    return np.asarray(labels).reshape(2,2)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:07.422794Z","iopub.execute_input":"2021-12-12T14:17:07.423167Z","iopub.status.idle":"2021-12-12T14:17:07.432714Z","shell.execute_reply.started":"2021-12-12T14:17:07.423112Z","shell.execute_reply":"2021-12-12T14:17:07.431347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (21,7))\nplt.subplot(1,3,1)\ncm=confusion_matrix(valid.label, valid.prediction)\nnp.set_printoptions(precision=2)\nsns.heatmap(cm/np.sum(cm), fmt='', cbar=False, annot=getLabels(cm), annot_kws={\"fontsize\":20},\n            xticklabels=categories, yticklabels=categories, cmap='flare')\nplt.title(\"Confusion Matrix without bias\")\n\nplt.subplot(1,3,2)\ncm=confusion_matrix(valid.label, [1 if prob>=0.4 else 0 for prob in valid.probability])\nnp.set_printoptions(precision=2)\nsns.heatmap(cm/np.sum(cm), fmt='', cbar=False, annot=getLabels(cm), annot_kws={\"fontsize\":20},\n            xticklabels=categories, yticklabels=categories, cmap='flare')\nplt.title(\"Confusion Matrix with 10% positive bias\")\n\nplt.subplot(1,3,3)\ncm=confusion_matrix(valid.label, [1 if prob>=0.3 else 0 for prob in valid.probability])\nnp.set_printoptions(precision=2)\nsns.heatmap(cm/np.sum(cm), fmt='', cbar=False, annot=getLabels(cm), annot_kws={\"fontsize\":20},\n            xticklabels=categories, yticklabels=categories, cmap='flare')\nplt.title(\"Confusion Matrix with 20% positive bias\")\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:45:21.128294Z","iopub.execute_input":"2021-12-12T14:45:21.129116Z","iopub.status.idle":"2021-12-12T14:45:21.553697Z","shell.execute_reply.started":"2021-12-12T14:45:21.129078Z","shell.execute_reply":"2021-12-12T14:45:21.552983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Special Note\nWhen you see the first Confusion Matrix, Accuracy is about 94% but the False benign rate is around 3.45% which is two times higher than False Malignant. When a trusted model predicts Benign the case doesn't go for further review and hence False Benign rate is a very important parameter of trust after Accuracy. And our final model is NOT upto the mark. \n\nFor our submissions, anything more than 50% probability is predicted a Malignant and less than 50% is predicted and Benign. \n\nThe sceond confusion matrix shows that when we add a 10 basis point positive bias to our probabilities and then derive the predictions The accuracy stays same but the False Benign percentage drops significantly. If we stretch further and add 20 basis points Accuracy goes down along with False benign percentage.   ","metadata":{}},{"cell_type":"markdown","source":"# Classification Report\n\nClassification report allows us to look at Precision and Recall.\n\nPrecision is defined as follows\n\n$\nPrecision\\ =\\ \\frac{TP}{TP+FP}\n$\n\nPrecision helps us answer the question _\"What proportion of positive identifications was actually correct?\"_\n\nRecall is is defined as follows\n\n$\nRecall\\ =\\ \\frac{TP}{TP+FN}\n$\n\nRecall helps us answers the question _\"What proportion of actual positives was identified correctly?\"_\n","metadata":{}},{"cell_type":"code","source":"print(classification_report(valid.label, valid.prediction, target_names=categories))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:07.854543Z","iopub.execute_input":"2021-12-12T14:17:07.854936Z","iopub.status.idle":"2021-12-12T14:17:07.869627Z","shell.execute_reply.started":"2021-12-12T14:17:07.854904Z","shell.execute_reply":"2021-12-12T14:17:07.868966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Benign Sample Images (True & False)","metadata":{}},{"cell_type":"code","source":"temp = valid[valid.label==0]\ntrueBenign = temp[temp.prediction==0]['path'][:4].tolist()\nfalseBenign = temp[temp.prediction==1]['path'][:4].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:07.870844Z","iopub.execute_input":"2021-12-12T14:17:07.871234Z","iopub.status.idle":"2021-12-12T14:17:07.879157Z","shell.execute_reply.started":"2021-12-12T14:17:07.871199Z","shell.execute_reply":"2021-12-12T14:17:07.878464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,10))\n\nfor i in range(4):\n    plt.subplot(2,4,i+1)\n    plt.imshow(mpimg.imread(train_path+\"/\"+trueBenign[i]))\n    plt.text(0, -5, f'True Benign', color='k')\n    plt.axis('off')\n    \n    plt.subplot(2,4,i+5)\n    plt.imshow(mpimg.imread(train_path+\"/\"+falseBenign[i]))\n    plt.text(0, -5, f'False Benign', color='k')\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:07.880515Z","iopub.execute_input":"2021-12-12T14:17:07.880792Z","iopub.status.idle":"2021-12-12T14:17:09.120717Z","shell.execute_reply.started":"2021-12-12T14:17:07.880755Z","shell.execute_reply":"2021-12-12T14:17:09.119992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Malignant Sample Images (True & False) ","metadata":{}},{"cell_type":"code","source":"temp = valid[valid.label==1]\ntrueMalignant = temp[temp.prediction==1]['path'][:4].tolist()\nfalseMalignant = temp[temp.prediction==0]['path'][:4].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:09.124063Z","iopub.execute_input":"2021-12-12T14:17:09.124679Z","iopub.status.idle":"2021-12-12T14:17:09.133619Z","shell.execute_reply.started":"2021-12-12T14:17:09.124643Z","shell.execute_reply":"2021-12-12T14:17:09.132791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,10))\n\nfor i in range(4):\n    plt.subplot(2,4,i+1)\n    plt.imshow(mpimg.imread(train_path+\"/\"+trueMalignant[i]))\n    plt.text(0, -5, f'True Malignant', color='k')\n    plt.axis('off')\n    \n    plt.subplot(2,4,i+5)\n    plt.imshow(mpimg.imread(train_path+\"/\"+falseMalignant[i]))\n    plt.text(0, -5, f'False Malignant', color='k')\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:17:09.135039Z","iopub.execute_input":"2021-12-12T14:17:09.135289Z","iopub.status.idle":"2021-12-12T14:17:10.508600Z","shell.execute_reply.started":"2021-12-12T14:17:09.135259Z","shell.execute_reply":"2021-12-12T14:17:10.507851Z"},"trusted":true},"execution_count":null,"outputs":[]}]}