{"cells":[{"metadata":{"_cell_guid":"c67e806c-e0e6-415b-8cf0-ed92eba5ed37","_uuid":"34b6997bb115a11f47a7f54ce9d9052791b2e707"},"cell_type":"markdown","source":"# Overview\nThe goal is to make a nice retinopathy model by using a pretrained inception v3 as a base and retraining some modified final layers with attention\n\nThis can be massively improved with \n* high-resolution images\n* better data sampling\n* ensuring there is no leaking between training and validation sets, ```sample(replace = True)``` is real dangerous\n* better target variable (age) normalization\n* pretrained models\n* attention/related techniques to focus on areas"},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\".\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the directories for the pretrained models\n!ls ../input/inceptionv3/   # just to be sure the data is here\n\nfrom os import makedirs\nfrom os.path import expanduser, exists, join\n\ncache_dir = expanduser(join('~', '.keras'))\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models')\nif not exists(models_dir):\n    makedirs(models_dir)\n!cp ../input/inceptionv3/* ~/.keras/models/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ~/.keras/models -a","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c3cc4285-bfa4-4612-ac5f-13d10678c09a","_uuid":"725d378daf5f836d4885d67240fc7955f113309d","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # showing and rendering figures\n# io related\nfrom skimage.io import imread\nimport os\nfrom glob import glob\n# not needed in Kaggle, but required in Jupyter\n%matplotlib inline ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c4b38df6-ffa1-4847-b605-511e72b68231","_uuid":"346da81db6ee7a34af8da8af245b42e681f2ba48","scrolled":true,"trusted":true},"cell_type":"code","source":"base_image_dir = os.path.join('..', \"input\",'aptos2019-blindness-detection','train_images')\nretina_df = pd.read_csv(os.path.join('..', \"input\",'aptos2019-blindness-detection','train.csv'))\nretina_df['path'] = retina_df['id_code'].map(lambda x: os.path.join(base_image_dir,\n                                                         '{}.png'.format(x)))\nfrom keras.utils.np_utils import to_categorical\nretina_df['level_cat'] = retina_df['diagnosis'].map(lambda x: to_categorical(x, 1+retina_df['diagnosis'].max()))\nretina_df.dropna(inplace = True)\nretina_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"818da6ca-bbff-4ca0-ad57-ef3a145ae863","_uuid":"688e4340238e013b8459b6f6470993c7de492d83"},"cell_type":"markdown","source":"# Examine the distribution of eye and severity"},{"metadata":{"_cell_guid":"5c8bd288-8261-4cbe-a954-e62ac795cc3e","_uuid":"60a8111c4093ca6f69d27a4499442ba7dd750839","scrolled":true,"trusted":true},"cell_type":"code","source":"retina_df[['diagnosis']].hist(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0ba697ed-85bb-4e9a-9765-4c367db078d1","_uuid":"4df45776bae0b8a1bf9d3eb4eaaebce6e24d726d"},"cell_type":"markdown","source":"# Split Data into Training and Validation"},{"metadata":{"_cell_guid":"1192c6b3-a940-4fa0-a498-d7e0d400a796","_uuid":"a48b300ca4d37a6e8b39f82e3c172739635e4baa","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nrr_df = retina_df[['id_code', 'diagnosis']].drop_duplicates()\ntrain_ids, valid_ids = train_test_split(rr_df['id_code'], \n                                   test_size = 0.25, \n                                   random_state = 2018,\n                                   stratify = rr_df['diagnosis'])\nraw_train_df = retina_df[retina_df['id_code'].isin(train_ids)]\nvalid_df = retina_df[retina_df['id_code'].isin(valid_ids)]\nprint('train', raw_train_df.shape[0], 'validation', valid_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f8060459-da1e-4293-8f61-c7f99de1de9f","_uuid":"26e566d6cec5bd41f9afe392f456ddf7ceb306ea"},"cell_type":"markdown","source":"# Balance the distribution in the training set"},{"metadata":{"_cell_guid":"7a130199-fbf6-4c60-95f5-0797b2f3eaf1","_uuid":"ba7befa238b8c11f9672e3539ac58f3da6955bd9","trusted":true},"cell_type":"code","source":"train_df = raw_train_df.groupby(['diagnosis']).apply(lambda x: x.sample(75, replace = True)\n                                                      ).reset_index(drop = True)\nprint('New Data Size:', train_df.shape[0], 'Old Size:', raw_train_df.shape[0])\ntrain_df[['diagnosis']].hist(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9954bfda-29bd-4c4d-b526-0a972b3e43e2","_uuid":"9529ab766763a9f122786464c24ab1ebe22c6006","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom keras import backend as K\nfrom keras.applications.inception_v3 import preprocess_input\nimport numpy as np\nIMG_SIZE = (512, 512) # slightly smaller than vgg16 normally expects\ndef tf_image_loader(out_size, \n                      horizontal_flip = True, \n                      vertical_flip = False, \n                     random_brightness = True,\n                     random_contrast = True,\n                    random_saturation = True,\n                    random_hue = True,\n                      color_mode = 'rgb',\n                       preproc_func = preprocess_input,\n                       on_batch = False):\n    def _func(X):\n        with tf.name_scope('image_augmentation'):\n            with tf.name_scope('input'):\n                X = tf.image.decode_png(tf.read_file(X), channels = 3 if color_mode == 'rgb' else 0)\n                X = tf.image.resize_images(X, out_size)\n            with tf.name_scope('augmentation'):\n                if horizontal_flip:\n                    X = tf.image.random_flip_left_right(X)\n                if vertical_flip:\n                    X = tf.image.random_flip_up_down(X)\n                if random_brightness:\n                    X = tf.image.random_brightness(X, max_delta = 0.1)\n                if random_saturation:\n                    X = tf.image.random_saturation(X, lower = 0.75, upper = 1.5)\n                if random_hue:\n                    X = tf.image.random_hue(X, max_delta = 0.15)\n                if random_contrast:\n                    X = tf.image.random_contrast(X, lower = 0.75, upper = 1.5)\n                return preproc_func(X)\n    if on_batch: \n        # we are meant to use it on a batch\n        def _batch_func(X, y):\n            return tf.map_fn(_func, X), y\n        return _batch_func\n    else:\n        # we apply it to everything\n        def _all_func(X, y):\n            return _func(X), y         \n        return _all_func\n    \ndef tf_augmentor(out_size,\n                intermediate_size = (640, 640),\n                 intermediate_trans = 'crop',\n                 batch_size = 16,\n                   horizontal_flip = True, \n                  vertical_flip = False, \n                 random_brightness = True,\n                 random_contrast = True,\n                 random_saturation = True,\n                    random_hue = True,\n                  color_mode = 'rgb',\n                   preproc_func = preprocess_input,\n                   min_crop_percent = 0.001,\n                   max_crop_percent = 0.005,\n                   crop_probability = 0.5,\n                   rotation_range = 10):\n    \n    load_ops = tf_image_loader(out_size = intermediate_size, \n                               horizontal_flip=horizontal_flip, \n                               vertical_flip=vertical_flip, \n                               random_brightness = random_brightness,\n                               random_contrast = random_contrast,\n                               random_saturation = random_saturation,\n                               random_hue = random_hue,\n                               color_mode = color_mode,\n                               preproc_func = preproc_func,\n                               on_batch=False)\n    def batch_ops(X, y):\n        batch_size = tf.shape(X)[0]\n        with tf.name_scope('transformation'):\n            # code borrowed from https://becominghuman.ai/data-augmentation-on-gpu-in-tensorflow-13d14ecf2b19\n            # The list of affine transformations that our image will go under.\n            # Every element is Nx8 tensor, where N is a batch size.\n            transforms = []\n            identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n            if rotation_range > 0:\n                angle_rad = rotation_range / 180 * np.pi\n                angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n                transforms += [tf.contrib.image.angles_to_projective_transforms(angles, intermediate_size[0], intermediate_size[1])]\n\n            if crop_probability > 0:\n                crop_pct = tf.random_uniform([batch_size], min_crop_percent, max_crop_percent)\n                left = tf.random_uniform([batch_size], 0, intermediate_size[0] * (1.0 - crop_pct))\n                top = tf.random_uniform([batch_size], 0, intermediate_size[1] * (1.0 - crop_pct))\n                crop_transform = tf.stack([\n                      crop_pct,\n                      tf.zeros([batch_size]), top,\n                      tf.zeros([batch_size]), crop_pct, left,\n                      tf.zeros([batch_size]),\n                      tf.zeros([batch_size])\n                  ], 1)\n                coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n                transforms += [tf.where(coin, crop_transform, tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]))]\n            if len(transforms)>0:\n                X = tf.contrib.image.transform(X,\n                      tf.contrib.image.compose_transforms(*transforms),\n                      interpolation='BILINEAR') # or 'NEAREST'\n            if intermediate_trans=='scale':\n                X = tf.image.resize_images(X, out_size)\n            elif intermediate_trans=='crop':\n                X = tf.image.resize_image_with_crop_or_pad(X, out_size[0], out_size[1])\n            else:\n                raise ValueError('Invalid Operation {}'.format(intermediate_trans))\n            return X, y\n    def _create_pipeline(in_ds):\n        batch_ds = in_ds.map(load_ops, num_parallel_calls=4).batch(batch_size)\n        return batch_ds.map(batch_ops)\n    return _create_pipeline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b5767f42-da63-4737-8f50-749c1a25aa84","_uuid":"07851e798db3d89ba13db7d4b56ab2b759221464","trusted":true},"cell_type":"code","source":"def flow_from_dataframe(idg, \n                        in_df, \n                        path_col,\n                        y_col, \n                        shuffle = True, \n                        color_mode = 'rgb'):\n    files_ds = tf.data.Dataset.from_tensor_slices((in_df[path_col].values, \n                                                   np.stack(in_df[y_col].values,0)))\n    in_len = in_df[path_col].values.shape[0]\n    while True:\n        if shuffle:\n            files_ds = files_ds.shuffle(in_len) # shuffle the whole dataset\n        \n        next_batch = idg(files_ds).repeat().make_one_shot_iterator().get_next()\n        for i in range(max(in_len//32,1)):\n            # NOTE: if we loop here it is 'thread-safe-ish' if we loop on the outside it is completely unsafe\n            yield K.get_session().run(next_batch)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"810bd229-fec9-43c4-b3bd-afd62e3e9552","_uuid":"1848f5048a9e00668c3778a85deea97f980e4f1c","trusted":true},"cell_type":"code","source":"batch_size = 48\ncore_idg = tf_augmentor(out_size = IMG_SIZE, \n                        color_mode = 'rgb', \n                        vertical_flip = True,\n                        crop_probability=0.0, # crop doesn't work yet\n                        batch_size = batch_size) \nvalid_idg = tf_augmentor(out_size = IMG_SIZE, color_mode = 'rgb', \n                         crop_probability=0.0, \n                         horizontal_flip = False, \n                         vertical_flip = False, \n                         random_brightness = False,\n                         random_contrast = False,\n                         random_saturation = False,\n                         random_hue = False,\n                         rotation_range = 0,\n                        batch_size = batch_size)\n\ntrain_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'path',\n                            y_col = 'level_cat')\n\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'level_cat') # we can use much larger batches for evaluation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ad184b936d5cebae91a265a247d8e0e25920566"},"cell_type":"markdown","source":"# Validation Set\nWe do not perform augmentation at all on these images"},{"metadata":{"_uuid":"6810407e25b887dd8b352f1e46fb3faceaa58ab7","trusted":true},"cell_type":"code","source":"t_x, t_y = next(valid_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(np.argmax(c_y, -1)))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34ce892a19c9734511e2da1d0f2552b361dc826d"},"cell_type":"markdown","source":"# Training Set\nThese are augmented and a real mess"},{"metadata":{"_cell_guid":"2d62234f-aeb0-4eba-8a38-d713d819abf6","_uuid":"8190b4ad60d49fa65af074dd138a19cb8787e983","scrolled":true,"trusted":true},"cell_type":"code","source":"t_x, t_y = next(train_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(np.argmax(c_y, -1)))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"da22790a-672c-474e-b118-9eef15b53160","_uuid":"55d665e1e8a8d83b9db005a66a965f8a90c62da1"},"cell_type":"markdown","source":"# Attention Model\nThe basic idea is that a Global Average Pooling is too simplistic since some of the regions are more relevant than others. So we build an attention mechanism to turn pixels in the GAP on an off before the pooling and then rescale (Lambda layer) the results based on the number of pixels. The model could be seen as a sort of 'global weighted average' pooling. There is probably something published about it and it is very similar to the kind of attention models used in NLP.\nIt is largely based on the insight that the winning solution annotated and trained a UNET model to segmenting the hand and transforming it. This seems very tedious if we could just learn attention."},{"metadata":{"_cell_guid":"eeb36110-0cde-4450-a43c-b8f707adb235","_uuid":"1f0dfaccda346d7bc4758e7329d61028d254a8d6","trusted":true},"cell_type":"code","source":"from keras.applications.inception_v3 import InceptionV3 as PTModel\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda\nfrom keras.models import Model\nin_lay = Input(t_x.shape[1:])\nbase_pretrained_model = PTModel(input_shape =  t_x.shape[1:], include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = False\npt_depth = base_pretrained_model.get_output_shape_at(0)[-1]\npt_features = base_pretrained_model(in_lay)\nfrom keras.layers import BatchNormalization\nbn_features = BatchNormalization()(pt_features)\n\n# here we do an attention mechanism to turn pixels in the GAP on an off\n\nattn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\nattn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(1, \n                    kernel_size = (1,1), \n                    padding = 'valid', \n                    activation = 'sigmoid')(attn_layer)\n# fan it out to all of the channels\nup_c2_w = np.ones((1, 1, 1, pt_depth))\nup_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n               activation = 'linear', use_bias = False, weights = [up_c2_w])\nup_c2.trainable = False\nattn_layer = up_c2(attn_layer)\n\nmask_features = multiply([attn_layer, bn_features])\ngap_features = GlobalAveragePooling2D()(mask_features)\ngap_mask = GlobalAveragePooling2D()(attn_layer)\n# to account for missing values from the attention model\ngap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\ngap_dr = Dropout(0.25)(gap)\ndr_steps = Dropout(0.25)(Dense(128, activation = 'relu')(gap_dr))\nout_layer = Dense(t_y.shape[-1], activation = 'softmax')(dr_steps)\nretina_model = Model(inputs = [in_lay], outputs = [out_layer])\nfrom keras.metrics import top_k_categorical_accuracy\ndef top_2_accuracy(in_gt, in_pred):\n    return top_k_categorical_accuracy(in_gt, in_pred, k=2)\n\nretina_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy',\n                           metrics = ['categorical_accuracy', top_2_accuracy])\nretina_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"17803ae1-bed8-41a4-9a2c-e66287a24830","_uuid":"48b9764e16fb5af52aed35c82bae6299e67d5bc7","trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('retina')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=6) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"58a75586-442b-4804-84a6-d63d5a42ea14","_uuid":"b2148479bfe41c5d9fd0faece4c75adea509dabe","trusted":true},"cell_type":"code","source":"retina_model.fit_generator(train_gen, \n                           steps_per_epoch = train_df.shape[0]//batch_size,\n                           validation_data = valid_gen, \n                           validation_steps = valid_df.shape[0]//batch_size,\n                              epochs = 3, \n                              callbacks = callbacks_list,\n                             workers = 0, # tf-generators are not thread-safe\n                             use_multiprocessing=False, \n                             max_queue_size = 0\n                            )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4d0c45b0-bb23-48d2-83eb-bc3990043e26","_uuid":"3a90f05dd206cd76c72d8c6278ebb93da41ee45f","trusted":true},"cell_type":"code","source":"# load the best version of the model\nretina_model.load_weights(weight_path)\nretina_model.save('full_retina_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f37dd4d8-ecd6-487a-90d8-74fe14a9a318","_uuid":"2b74f4ab850c6e82549d732b6f0524724b95b53c","trusted":true},"cell_type":"code","source":"##### create one fixed dataset for evaluating\nfrom tqdm import tqdm_notebook\n# fresh valid gen\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'level_cat') \nvbatch_count = (valid_df.shape[0]//batch_size-1)\nout_size = vbatch_count*batch_size\ntest_X = np.zeros((out_size,)+t_x.shape[1:], dtype = np.float32)\ntest_Y = np.zeros((out_size,)+t_y.shape[1:], dtype = np.float32)\nfor i, (c_x, c_y) in zip(tqdm_notebook(range(vbatch_count)), \n                         valid_gen):\n    j = i*batch_size\n    test_X[j:(j+c_x.shape[0])] = c_x\n    test_Y[j:(j+c_x.shape[0])] = c_y","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"11f33f0a-61eb-488a-b7ea-4bc9d15ba8f9","_uuid":"cca170eb40bc591f89748ede8aa35de4308faaaf"},"cell_type":"markdown","source":"# Show Attention\nDid our attention model learn anything useful?"},{"metadata":{"_cell_guid":"e41a063f-35c9-410f-be63-f66b63ff9683","_uuid":"ad5b085d351e79b950bf0c2ddc476799d5b0692f","trusted":true},"cell_type":"code","source":"# get the attention layer since it is the only one with a single output dim\nfor attn_layer in retina_model.layers:\n    c_shape = attn_layer.get_output_shape_at(0)\n    if len(c_shape)==4:\n        if c_shape[-1]==1:\n            print(attn_layer)\n            break","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"340eef36-f5b2-4b15-a59f-440061a427eb","_uuid":"00850972ae4298f49ed1838b3fc49c2d8fb07547","scrolled":true,"trusted":true},"cell_type":"code","source":"import keras.backend as K\nrand_idx = np.random.choice(range(len(test_X)), size = 6)\nattn_func = K.function(inputs = [retina_model.get_input_at(0), K.learning_phase()],\n           outputs = [attn_layer.get_output_at(0)]\n          )\nfig, m_axs = plt.subplots(len(rand_idx), 2, figsize = (8, 4*len(rand_idx)))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nfor c_idx, (img_ax, attn_ax) in zip(rand_idx, m_axs):\n    cur_img = test_X[c_idx:(c_idx+1)]\n    attn_img = attn_func([cur_img, 0])[0]\n    img_ax.imshow(np.clip(cur_img[0,:,:,:]*127+127, 0, 255).astype(np.uint8))\n    attn_ax.imshow(attn_img[0, :, :, 0]/attn_img[0, :, :, 0].max(), cmap = 'viridis', \n                   vmin = 0, vmax = 1, \n                   interpolation = 'lanczos')\n    real_cat = np.argmax(test_Y[c_idx, :])\n    img_ax.set_title('Eye Image\\nCat:%2d' % (real_cat))\n    pred_cat = retina_model.predict(cur_img)\n    attn_ax.set_title('Attention Map\\nPred:%2.2f%%' % (100*pred_cat[0,real_cat]))\nfig.savefig('attention_map.png', dpi = 300)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"24796de7-b1e9-4b3b-bcc6-d997aa3e6d16","_uuid":"244bac80d1ea2074e47932e367996e32cbab6a3d"},"cell_type":"markdown","source":"# Evaluate the results\nHere we evaluate the results by loading the best version of the model and seeing how the predictions look on the results. We then visualize spec"},{"metadata":{"_cell_guid":"d0edaf00-4b7c-4f65-af0b-e5a03b9b8428","_uuid":"b421b6183b1919a7414482f0b1ac611079e45174","trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, classification_report\npred_Y = retina_model.predict(test_X, batch_size = 32, verbose = True)\npred_Y_cat = np.argmax(pred_Y, -1)\ntest_Y_cat = np.argmax(test_Y, -1)\nprint('Accuracy on Test Data: %2.2f%%' % (accuracy_score(test_Y_cat, pred_Y_cat)))\nprint(classification_report(test_Y_cat, pred_Y_cat))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"15189df2-3fed-495e-9661-97bb2b712dfd","_uuid":"10162e055ca7cd52878a289bab377231787ab732","trusted":true},"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nsns.heatmap(confusion_matrix(test_Y_cat, pred_Y_cat), \n            annot=True, fmt=\"d\", cbar = False, cmap = plt.cm.Blues, vmax = test_X.shape[0]//16)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"70827da6-bf91-4b65-80e9-bf1e6b885db3","_uuid":"12dfe39ea80194062068589699953c6645e285d6"},"cell_type":"markdown","source":"# ROC Curve for healthy vs sick\nHere we make an ROC curve for healthy (```severity == 0```) and sick (```severity>0```) to see how well the model works at just identifying the disease"},{"metadata":{"_cell_guid":"829475ab-7db2-4421-b9ad-1a51971fd459","_uuid":"2b2aaee6c83043f721b0c9ed5bc4229eb7165200","trusted":true},"cell_type":"code","source":"\nfrom sklearn.metrics import roc_curve, roc_auc_score\nsick_vec = test_Y_cat>0\nsick_score = np.sum(pred_Y[:,1:],1)\nfpr, tpr, _ = roc_curve(sick_vec, sick_score)\nfig, ax1 = plt.subplots(1,1, figsize = (6, 6), dpi = 150)\nax1.plot(fpr, tpr, 'b.-', label = 'Model Prediction (AUC: %2.2f)' % roc_auc_score(sick_vec, sick_score))\nax1.plot(fpr, fpr, 'g-', label = 'Random Guessing')\nax1.legend()\nax1.set_xlabel('False Positive Rate')\nax1.set_ylabel('True Positive Rate');","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c34f049f-b032-45bf-9d5e-a756ecc46a82","_uuid":"ba87d0e7c3a77181487b99ca64d13de2aa8a21ee","scrolled":false,"trusted":true},"cell_type":"code","source":"fig, m_axs = plt.subplots(2, 4, figsize = (32, 20))\nfor (idx, c_ax) in enumerate(m_axs.flatten()):\n    c_ax.imshow(np.clip(test_X[idx]*127+127,0 , 255).astype(np.uint8), cmap = 'bone')\n    c_ax.set_title('Actual Severity: {}\\n{}'.format(test_Y_cat[idx], \n                                                           '\\n'.join(['Predicted %02d (%04.1f%%): %s' % (k, 100*v, '*'*int(10*v)) for k, v in sorted(enumerate(pred_Y[idx]), key = lambda x: -1*x[1])])), loc='left')\n    c_ax.axis('off')\nfig.savefig('trained_img_predictions.png', dpi = 300)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f2e189dc-f80a-4b16-bb1d-5c05a155a80b","_uuid":"eb6752295030ba512263433f8383711e4ca1c14c"},"cell_type":"markdown","source":"# Kaggle eval"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsubmission.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_image_dir = os.path.join('..',\"input\",'aptos2019-blindness-detection', 'test_images')\ntest_df = pd.read_csv(os.path.join('..',\"input\",'aptos2019-blindness-detection','test.csv'))\ntest_df['path'] = test_df['id_code'].map(lambda x: os.path.join(base_image_dir,\n                                                         '{}.png'.format(x)))\ntest_df['diagnosis'] = 0\nfrom keras.utils.np_utils import to_categorical\ntest_df['level_cat'] = test_df['diagnosis'].map(lambda x: to_categorical(x,5))\n\ntest_df.dropna(inplace = True)\ntest_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lst = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom matplotlib import pyplot\nfrom numpy import asarray\nfor i in range(test_df.shape[0]):\n    image = Image.open(test_df['path'][i])\n    img_resized = image.resize(IMG_SIZE)\n    data = asarray(img_resized).reshape([1,512,512,3])\n    res = np.argmax(retina_model.predict(data))\n    lst.append(res)\n    if(i%300==0):\n        print(i)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['diagnosis'] = lst","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(\".\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}