{"cells":[{"metadata":{},"cell_type":"markdown","source":"

Detect diabetic retinopathy to stop blindness before it's too late

\n
\n##### Image source: http://cceyemd.com/diabetes-and-eye-exams/"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper libraries\nimport tensorflow\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport cv2\nimport os\nimport random\nfrom tqdm import tqdm_notebook as tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set random seeds to try and get consistent results"},{"metadata":{"trusted":true},"cell_type":"code","source":"RANDOM_SEED = 2019\nnp.random.seed(RANDOM_SEED)\ntensorflow.set_random_seed(RANDOM_SEED)\nrandom.seed(RANDOM_SEED) # Python\nos.environ['PYTHONHASHSEED'] = str(RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install keras_efficientnets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read in the training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntrain_df['id_code'] = train_df['id_code'].apply(lambda x:'../input/aptos2019-blindness-detection/train_images/' + x + '.png')\ntrain_df['diagnosis'] = train_df['diagnosis'].astype(str)\nnum_classes = train_df['diagnosis'].nunique()\ndiag_text = ['Normal', 'Mild', 'Moderate', 'Severe', 'Proliferative']\n\ntrain_df.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"old_train_df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\nold_train_df = old_train_df[['image','level']]\nold_train_df.columns = train_df.columns\nold_train_df['id_code'] = old_train_df['id_code'].apply(lambda x:'../input/diabetic-retinopathy-resized/resized_train/resized_train/' + x + '.jpeg')\nold_train_df['diagnosis'] = old_train_df['diagnosis'].astype(str)\n\nold_train_df.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Look at some raw images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_raw_images(df, columns = 4, rows = 2):\n fig=plt.figure(figsize = (5 * columns, 4 * rows))\n for i in range(columns * rows):\n image_name = df.loc[i,'id_code']\n image_id = df.loc[i,'diagnosis']\n img = cv2.imread(image_name)[...,[2, 1, 0]]\n fig.add_subplot(rows, columns, i + 1)\n plt.title(diag_text[int(image_id)])\n plt.imshow(img)\n plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2019 Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_raw_images(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Old Competion Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_raw_images(old_train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Look at the class frequency"},{"metadata":{"trusted":true},"cell_type":"code","source":"unique, counts = np.unique(train_df['diagnosis'], return_counts=True)\nplt.bar(unique, counts)\nplt.title('Class Frequency')\nplt.xlabel('Class')\nplt.ylabel('Frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique, counts = np.unique(old_train_df['diagnosis'], return_counts = True)\nplt.bar(unique, counts)\nplt.title('Class Frequency')\nplt.xlabel('Class')\nplt.ylabel('Frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Divide by class\nold_train_df_class_0 = old_train_df[old_train_df['diagnosis'] == '0']\nold_train_df_class_1 = old_train_df[old_train_df['diagnosis'] == '1']\nold_train_df_class_2 = old_train_df[old_train_df['diagnosis'] == '2']\nold_train_df_class_3 = old_train_df[old_train_df['diagnosis'] == '3']\nold_train_df_class_4 = old_train_df[old_train_df['diagnosis'] == '4']\n\n\ntrain_df_plus = pd.concat([train_df,\n old_train_df_class_0.sample(2000),\n old_train_df_class_1.sample(2000),\n old_train_df_class_2.sample(2000),\n old_train_df_class_3.sample(873),\n old_train_df_class_4.sample(708)],\n axis=0)\n\ntrain_df_plus = train_df_plus.sample(frac = 1).reset_index(drop = True)\n\nprint('After random under-sampling: ')\ntrain_df_plus.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique, counts = np.unique(train_df_plus['diagnosis'], return_counts = True)\nplt.bar(unique, counts)\nplt.title('Class Frequency')\nplt.xlabel('Class')\nplt.ylabel('Frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Calculate class weights to help with training on the unbalanced data set.[](http://) "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\n\nsklearn_class_weights = class_weight.compute_class_weight(\n 'balanced',\n np.unique(train_df['diagnosis']), \n train_df['diagnosis'])\n\nprint(sklearn_class_weights)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load a model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, Callback\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, LeakyReLU\nfrom keras.models import Model, Sequential\nfrom keras.optimizers import Adam \n\nfrom keras_efficientnets import EfficientNetB5\n\ndef create_effnetB5_model(input_shape, n_out):\n model = Sequential()\n base_model = EfficientNetB5(weights = 'imagenet', \n include_top = False,\n input_shape = input_shape)\n base_model.name = 'base_model'\n model.add(base_model)\n model.add(Dropout(0.25))\n model.add(Dense(1024))\n model.add(LeakyReLU())\n model.add(GlobalAveragePooling2D())\n model.add(Dropout(0.5)) \n model.add(Dense(n_out, activation = 'sigmoid'))\n return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_HEIGHT = 340\nIMAGE_WIDTH = 340\nmodel = create_effnetB5_model(input_shape = (IMAGE_HEIGHT, IMAGE_WIDTH, 3), n_out = num_classes)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Change target to a multi-label problem so a class encompasses all the classes before it.\nsee: https://arxiv.org/abs/0704.1028"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = pd.get_dummies(train_df['diagnosis']).values\ny_train_multi = np.empty(y_train.shape, dtype = y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i + 1])\n \nx_train = train_df['id_code']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_old = pd.get_dummies(old_train_df['diagnosis']).values\ny_train_multi_old = np.empty(y_train_old.shape, dtype = y_train_old.dtype)\ny_train_multi_old[:, 4] = y_train_old[:, 4]\n\nfor i in range(3, -1, -1):\n y_train_multi_old[:, i] = np.logical_or(y_train_old[:, i], y_train_multi_old[:, i + 1])\n \nx_train_old = old_train_df['id_code']\ny_train_old = y_train_multi_old","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Split into training and validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train, x_val, y_train, y_val = train_test_split(\n x_train, y_train_multi, \n test_size = 0.50, \n random_state = RANDOM_SEED\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentation Recepie"},{"metadata":{"trusted":true},"cell_type":"code","source":"import imgaug as ia\nfrom imgaug import augmenters as iaa\n\nsometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential(\n [\n # apply the following augmenters to most images\n iaa.Fliplr(0.1), # horizontally flip 10% of all images\n iaa.Flipud(0.1), # vertically flip 10% of all images\n sometimes(iaa.Affine(\n scale={\"x\": (0.95, 1.05), \"y\": (0.95, 1.05)}, # scale images to 95-105% of their size, individually per axis\n translate_percent={\"x\": (-0.05, 0.05), \"y\": (-0.05, 0.05)}, # translate by -5 to +5 percent (per axis)\n rotate=(-180, 180), # rotate by -180 to +180 degrees\n shear=(-3, 3), # shear by -3 to +3 degrees\n order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n )),\n # execute 0 to 5 of the following (less important) augmenters per image\n # don't execute all of them, as that would often be way too strong\n iaa.SomeOf((0, 3),\n [\n sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n iaa.OneOf([\n iaa.GaussianBlur((0, 0.5)), # blur images with a sigma between 0 and 0.5\n iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7\n iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7\n ]),\n iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images\n iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n # search either for all edges or for directed edges,\n # blend the result with the original image using a blobby mask\n iaa.SimplexNoiseAlpha(iaa.OneOf([\n iaa.EdgeDetect(alpha=(0.5, 1.0)),\n iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n ])),\n iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), # add gaussian noise to images\n iaa.OneOf([\n iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 5% of the pixels\n iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),\n ]),\n iaa.Invert(0.01, per_channel = True), # invert color channels\n iaa.Add((-2, 2), per_channel = 0.5), # change brightness of images (by -5 to 5 of original value)\n iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation\n # either change the brightness of the whole image (sometimes\n # per channel) or change the brightness of subareas\n iaa.OneOf([\n iaa.Multiply((0.9, 1.1), per_channel = 0.5),\n iaa.FrequencyNoiseAlpha(\n exponent = (-1, 0),\n first = iaa.Multiply((0.9, 1.1), per_channel = True),\n second = iaa.ContrastNormalization((0.9, 1.1))\n )\n ]),\n sometimes(iaa.ElasticTransformation(alpha = (0.5, 3.5), sigma = 0.25)), # move pixels locally around (with random strengths)\n sometimes(iaa.PiecewiseAffine(scale = (0.01, 0.05))), # sometimes move parts of the image around\n sometimes(iaa.PerspectiveTransform(scale = (0.01, 0.1)))\n ],\n random_order = True\n )\n ],\n random_order = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image_from_gray(img, tol = 7):\n if img.ndim ==2:\n mask = img>tol\n return img[np.ix_(mask.any(1),mask.any(0))]\n elif img.ndim==3:\n gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n mask = gray_img>tol\n \n check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n if (check_shape == 0): # image is too dark so that we crop out everything,\n return img # return original image\n else:\n img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n # print(img1.shape,img2.shape,img3.shape)\n img = np.stack([img1,img2,img3],axis = -1)\n # print(img.shape)\n return img\n \ndef process_image(image):\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n image = crop_image_from_gray(image)\n image = cv2.resize(image, (IMAGE_HEIGHT, IMAGE_WIDTH))\n image=cv2.addWeighted (image,4, cv2.GaussianBlur( image , (0,0) , 10) ,-4 ,128)\n return image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import Sequence\nfrom sklearn.utils import shuffle\n\nclass Aptos2019Generator(Sequence):\n\n def __init__(self, image_filenames, labels,\n batch_size, is_train = True,\n mix = False, augment = False):\n self.image_filenames = image_filenames\n self.labels = labels\n self.batch_size = batch_size\n self.is_train = is_train\n self.is_augment = augment\n if(self.is_train):\n self.on_epoch_end()\n self.is_mix = mix\n\n def __len__(self):\n return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n def __getitem__(self, idx):\n batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n if(self.is_train):\n return self.train_generate(batch_x, batch_y)\n else:\n return self.valid_generate(batch_x, batch_y)\n\n def on_epoch_end(self):\n if(self.is_train):\n self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n else:\n pass\n \n def mix_up(self, x, y):\n lam = np.random.beta(0.2, 0.4)\n ori_index = np.arange(int(len(x)))\n index_array = np.arange(int(len(x)))\n np.random.shuffle(index_array) \n mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n return mixed_x, mixed_y\n\n def train_generate(self, batch_x, batch_y):\n batch_images = []\n for (sample, label) in zip(batch_x, batch_y):\n img = cv2.imread(sample)\n# img = cv2.resize(img, (IMAGE_HEIGHT, IMAGE_WIDTH))\n img = process_image(img)\n if(self.is_augment):\n img = seq.augment_image(img)\n batch_images.append(img)\n batch_images = np.array(batch_images, np.float32) / 255\n batch_y = np.array(batch_y, np.float32)\n if(self.is_mix):\n batch_images, batch_y = self.mix_up(batch_images, batch_y)\n return batch_images, batch_y\n\n def valid_generate(self, batch_x, batch_y):\n batch_images = []\n for (sample, label) in zip(batch_x, batch_y):\n img = cv2.imread(sample)\n# img = cv2.resize(img, (IMAGE_HEIGHT, IMAGE_WIDTH))\n img = process_image(img)\n batch_images.append(img)\n batch_images = np.array(batch_images, np.float32) / 255\n batch_y = np.array(batch_y, np.float32)\n return batch_images, batch_y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup training data generator with augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8\ntrain_generator = Aptos2019Generator(x_train, y_train, BATCH_SIZE, is_train = True, augment = False, mix = False)\nvalid_generator = Aptos2019Generator(x_val, y_val, BATCH_SIZE, is_train = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nfrom sklearn.metrics import cohen_kappa_score\n\nclass QWK(Callback):\n def __init__(self, validation_data = (), batch_size = 64, interval=1):\n super(Callback, self).__init__()\n self.interval = interval\n self.batch_size = batch_size\n self.valid_generator, self.y_val = validation_data\n self.history = []\n self.max_score = float(\"-inf\")\n\n def on_epoch_end(self, epoch, logs = {}):\n if epoch % self.interval == 0:\n validation_predictions_raw = self.model.predict_generator(generator=self.valid_generator,\n steps = np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n workers = 1, use_multiprocessing=False,\n verbose = 1) \n validation_predictions = validation_predictions_raw > 0.5\n validation_predictions = validation_predictions.astype(int).sum(axis=1) - 1\n validation_truth = y_val.sum(axis=1) - 1 \n score = cohen_kappa_score(validation_predictions, validation_truth, weights = 'quadratic')\n self.history.append(score)\n print(\"epoch: %d - qwk_score: %.6f\" % (epoch + 1, score))\n if score >= self.max_score:\n print('qwk_score improved from %.6f to %.6f, saving model to blindness_detector_best_qwk.h5' % (self.max_score, score)) \n self.model.save('../working/blindness_detector_best_qwk.h5')\n self.max_score = score\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qwk = QWK(\n validation_data = (valid_generator, y_val), \n batch_size = BATCH_SIZE, \n interval = 1)\n\ncheckpoint = ModelCheckpoint(\n 'blindness_detector_best.h5', \n monitor = 'val_acc', \n save_best_only = True, \n save_weights_only = False,\n verbose = 1)\n\nrlrop = ReduceLROnPlateau(\n monitor = 'val_loss', \n patience = 3, \n factor = 0.5, \n min_lr = 1e-6, \n verbose = 1)\n\nstopping = EarlyStopping(\n monitor = 'val_acc', \n patience = 8, \n restore_best_weights = True, \n verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the clasifier head"},{"metadata":{"trusted":true},"cell_type":"code","source":"WARMUP_EPOCHS = 3\nWARMUP_LEARNING_RATE = 1e-3\n \nfor layer in model.layers:\n if layer.name == 'base_model':\n layer.trainable = False \n else:\n layer.trainable = True \n\nmodel.compile(optimizer = Adam(lr = WARMUP_LEARNING_RATE),\n loss = 'binary_crossentropy', \n metrics = ['accuracy'])\n\nwarmup_history = model.fit_generator(generator = train_generator,\n steps_per_epoch = len(train_generator),\n epochs = WARMUP_EPOCHS,\n validation_data = valid_generator,\n validation_steps = len(valid_generator),\n callbacks = [qwk],\n use_multiprocessing = True,\n verbose = 1).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fine-tune the whole model"},{"metadata":{"trusted":true},"cell_type":"code","source":"FINETUNING_EPOCHS = 20\nFINETUNING_LEARNING_RATE = 1e-4\n\nfor layer in model.layers:\n layer.trainable = True\n # if layer.name == 'base_model': \n # set_trainable = False\n # for sub_layer in layer.layers:\n # if sub_layer.name == 'multiply_16':\n # set_trainable = True\n # if set_trainable:\n # sub_layer.trainable = True\n # else:\n # sub_layer.trainable = False \n \nmodel.compile(optimizer = Adam(lr = FINETUNING_LEARNING_RATE), \n loss = 'binary_crossentropy',\n metrics = ['accuracy'])\n\n\n\ntrain_generator_augmented = Aptos2019Generator(x_train, y_train, BATCH_SIZE, is_train = True, mix = True, augment = True)\n\nfinetune_history = model.fit_generator(\n generator = train_generator_augmented,\n# class_weight = sklearn_class_weights,\n steps_per_epoch = len(train_generator_augmented),\n validation_data = valid_generator,\n validation_steps = len(valid_generator),\n epochs = FINETUNING_EPOCHS,\n callbacks = [rlrop, qwk], \n use_multiprocessing = True,\n# workers = 2,\n verbose = 1).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot learning curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_accuracy = warmup_history['acc'] + finetune_history['acc'] \nvalidation_accuracy = warmup_history['val_acc'] + finetune_history['val_acc']\ntraining_loss = warmup_history['loss'] + finetune_history['loss'] \nvalidation_loss = warmup_history['val_loss'] + finetune_history['val_loss'] \n\nplt.figure(figsize = (8, 8))\nplt.subplot(2, 1, 1)\nplt.plot(training_accuracy, label = 'Training Accuracy')\nplt.plot(validation_accuracy, label = 'Validation Accuracy')\nplt.legend(loc = 'lower right')\nplt.ylabel('Accuracy')\nplt.ylim([min(plt.ylim()), 1])\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(2, 1, 2)\nplt.plot(training_loss, label = 'Training Loss')\nplt.plot(validation_loss, label = 'Validation Loss')\nplt.legend(loc = 'upper right')\nplt.ylabel('Cross Entropy')\nplt.ylim([0, 1.0])\nplt.title('Training and Validation Loss')\nplt.xlabel('epoch')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluate the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"../working/blindness_detector_best_qwk.h5\")\n\nfor layer in model.layers:\n layer.trainable = True\n if layer.name == 'base_model': \n for sub_layer in layer.layers:\n sub_layer.trainable = True\n \nmodel.save('../working/blindness_detector_best_qwk.h5') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Get validation predictions from the final model"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_predictions_raw = model.predict_generator(\n valid_generator,\n steps = np.ceil(float(len(x_val)) / float(BATCH_SIZE)))\nvalidation_predictions = validation_predictions_raw > 0.5\nvalidation_predictions = validation_predictions.astype(int).sum(axis = 1) - 1\nvalidation_truth = y_val.sum(axis = 1) - 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix, cohen_kappa_score\n\ndef plot_confusion_matrix(cm, target_names, title = 'Confusion matrix', cmap = plt.cm.Blues):\n plt.grid(False)\n plt.imshow(cm, interpolation = 'nearest', cmap = cmap)\n plt.title(title)\n plt.colorbar()\n tick_marks = np.arange(len(target_names))\n plt.xticks(tick_marks, target_names, rotation = 90)\n plt.yticks(tick_marks, target_names)\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')\n\nnp.set_printoptions(precision = 2)\ncm = confusion_matrix(validation_truth, validation_predictions)\ncm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\nplot_confusion_matrix(cm = cm, target_names = diag_text)\nplt.show()\n\nprint('Confusion Matrix')\nprint(cm)\n\nprint('Classification Report')\nprint(classification_report(validation_truth, validation_predictions, target_names = diag_text))\n\nprint(\"Validation Cohen Kappa Score: %.3f\" % cohen_kappa_score(validation_predictions, validation_truth, weights = 'quadratic'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"1139da673271489bbd4f774119c192df":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"15845a0b5a584a969cb4ab3da4d074fd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c0cb60a31d6a4a488e8830522fcaadd0","placeholder":"​","style":"IPY_MODEL_c4921b9420a749c4af557f6aecedf968","value":"100% 3662/3662 [09:24<00:00, 5.90it/s]"}},"1958495068d14391b164121ec8fdf7b2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d5c89e8efcbd446da04aa2d341a26057","IPY_MODEL_15845a0b5a584a969cb4ab3da4d074fd"],"layout":"IPY_MODEL_553a91b436d54045a338b3d81cf436cc"}},"23ea6c0ef2e445a1a5768ee878f24228":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"330daeddbd094e12939e7fb13d94511e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"36d75e09c3df4c8d86eb6bc29472e938":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"40b84d1570f24bb89604360c32c2bdcc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_23ea6c0ef2e445a1a5768ee878f24228","placeholder":"​","style":"IPY_MODEL_36d75e09c3df4c8d86eb6bc29472e938","value":"100% 1928/1928 [02:09<00:00, 14.85it/s]"}},"553a91b436d54045a338b3d81cf436cc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"72be950808414ae1a5923136868fbab7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_88c2f868de0e4091b430a66a9c6f7561","max":1928,"min":0,"orientation":"horizontal","style":"IPY_MODEL_1139da673271489bbd4f774119c192df","value":1928}},"88c2f868de0e4091b430a66a9c6f7561":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9e1345f5a1704adea45bae7cdb042a90":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9e531c9e2b1145799e982054eedc791b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_72be950808414ae1a5923136868fbab7","IPY_MODEL_40b84d1570f24bb89604360c32c2bdcc"],"layout":"IPY_MODEL_9e1345f5a1704adea45bae7cdb042a90"}},"c0cb60a31d6a4a488e8830522fcaadd0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c4921b9420a749c4af557f6aecedf968":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"d19e80b6c0f44cf2a9268ae2f28ff0e0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d5c89e8efcbd446da04aa2d341a26057":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d19e80b6c0f44cf2a9268ae2f28ff0e0","max":3662,"min":0,"orientation":"horizontal","style":"IPY_MODEL_330daeddbd094e12939e7fb13d94511e","value":3662}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}