{"cells":[{"metadata":{"_uuid":"34b6997bb115a11f47a7f54ce9d9052791b2e707","_cell_guid":"c67e806c-e0e6-415b-8cf0-ed92eba5ed37"},"cell_type":"markdown","source":"# Overview\nThe goal is to make a nice retinopathy model by using a pretrained inception v3 as a base and retraining some modified final layers with attention. The notebook was originally written for the original Data Science Bowl for diabetic retinopathy and adapted to the new competition (and better Kaggle features!)\n\nThis can be massively improved with \n* high-resolution images\n* better data sampling\n* ensuring there is no leaking between training and validation sets, ```sample(replace = True)``` is real dangerous\n* better target variable (age) normalization\n* pretrained models\n* attention/related techniques to focus on areas"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# copy the weights and configurations for the pre-trained models (so the kernel can be offline)\n!mkdir ~/.keras\n!mkdir ~/.keras/models\n!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Parameters\nHere we keep track of all the parameters so we can easily change them and track performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\nIMG_SIZE = (256, 256) # slightly smaller than vgg16 normally expects\nINTERMEDIATE_SIZE = (384, 384)\nBRIGHTNESS_DELTA = 0.1\nSATURATION_DELTA = 0.2\nHUE_DELTA = 0.05\nCONTRAST_DELTA = 0.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"725d378daf5f836d4885d67240fc7955f113309d","_cell_guid":"c3cc4285-bfa4-4612-ac5f-13d10678c09a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # showing and rendering figures\n# io related\nfrom skimage.io import imread\nimport os\nfrom glob import glob\nimport tensorflow as tf\nimport random\n\n# not needed in Kaggle, but required in Jupyter\n%matplotlib inline ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_random_seed(seed)\n    \nSEED = 2019\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"346da81db6ee7a34af8da8af245b42e681f2ba48","_cell_guid":"c4b38df6-ffa1-4847-b605-511e72b68231","trusted":true},"cell_type":"code","source":"base_image_dir = os.path.join('..', 'input', 'aptos2019-blindness-detection')\nretina_df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))\nretina_df['path'] = retina_df['id_code'].map(lambda x: os.path.join(base_image_dir, 'train_images',\n                                                         '{}.png'.format(x)))\nretina_df['exists'] = retina_df['path'].map(os.path.exists)\nprint(retina_df['exists'].sum(), 'images found of', retina_df.shape[0], 'total')\nretina_df.dropna(inplace = True)\nretina_df = retina_df[retina_df['exists']]\nretina_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"688e4340238e013b8459b6f6470993c7de492d83","_cell_guid":"818da6ca-bbff-4ca0-ad57-ef3a145ae863"},"cell_type":"markdown","source":"# Examine the distribution of eye and severity"},{"metadata":{"_uuid":"60a8111c4093ca6f69d27a4499442ba7dd750839","_cell_guid":"5c8bd288-8261-4cbe-a954-e62ac795cc3e","trusted":true},"cell_type":"code","source":"retina_df[['diagnosis']].hist(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4df45776bae0b8a1bf9d3eb4eaaebce6e24d726d","_cell_guid":"0ba697ed-85bb-4e9a-9765-4c367db078d1"},"cell_type":"markdown","source":"# Split Data into Training and Validation"},{"metadata":{"_uuid":"a48b300ca4d37a6e8b39f82e3c172739635e4baa","_cell_guid":"1192c6b3-a940-4fa0-a498-d7e0d400a796","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nrr_df = retina_df[['id_code', 'diagnosis']].drop_duplicates()\ntrain_ids, valid_ids = train_test_split(rr_df['id_code'], \n                                   test_size = 0.25, \n                                   random_state = 2018,\n                                   stratify = rr_df['diagnosis'])\nraw_train_df = retina_df[retina_df['id_code'].isin(train_ids)]\nvalid_df = retina_df[retina_df['id_code'].isin(valid_ids)]\nprint('train', raw_train_df.shape[0], 'validation', valid_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26e566d6cec5bd41f9afe392f456ddf7ceb306ea","_cell_guid":"f8060459-da1e-4293-8f61-c7f99de1de9f"},"cell_type":"markdown","source":"# Balance the distribution in the training set"},{"metadata":{"_uuid":"ba7befa238b8c11f9672e3539ac58f3da6955bd9","_cell_guid":"7a130199-fbf6-4c60-95f5-0797b2f3eaf1","trusted":true},"cell_type":"code","source":"train_df = raw_train_df.groupby(['diagnosis']).apply(lambda x: x.sample(500, replace = True)\n                                                      ).reset_index(drop = True)\nprint('New Data Size:', train_df.shape[0], 'Old Size:', raw_train_df.shape[0])\ntrain_df[['diagnosis']].hist(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9529ab766763a9f122786464c24ab1ebe22c6006","_cell_guid":"9954bfda-29bd-4c4d-b526-0a972b3e43e2","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom keras import backend as K\nfrom keras.applications.inception_v3 import preprocess_input\nimport numpy as np\n\ndef tf_image_loader(out_size, \n                      horizontal_flip = True, \n                      vertical_flip = False, \n                     random_brightness = True,\n                     random_contrast = True,\n                    random_saturation = True,\n                    random_hue = True,\n                      color_mode = 'rgb',\n                       preproc_func = preprocess_input,\n                       on_batch = False):\n    def _func(X):\n        with tf.name_scope('image_augmentation'):\n            with tf.name_scope('input'):\n                X = tf.image.decode_png(tf.read_file(X), channels = 3 if color_mode == 'rgb' else 0)\n                X = tf.image.resize_images(X, out_size)\n            with tf.name_scope('augmentation'):\n                if horizontal_flip:\n                    X = tf.image.random_flip_left_right(X)\n                if vertical_flip:\n                    X = tf.image.random_flip_up_down(X)\n                if random_brightness:\n                    X = tf.image.random_brightness(X, max_delta = BRIGHTNESS_DELTA)\n                if random_saturation:\n                    X = tf.image.random_saturation(X, lower = 1-SATURATION_DELTA, upper = 1+SATURATION_DELTA)\n                if random_hue:\n                    X = tf.image.random_hue(X, max_delta = HUE_DELTA)\n                if random_contrast:\n                    X = tf.image.random_contrast(X, lower = 1-CONTRAST_DELTA, upper = 1+CONTRAST_DELTA)\n                return preproc_func(X)\n    if on_batch: \n        # we are meant to use it on a batch\n        def _batch_func(X, y):\n            return tf.map_fn(_func, X), y\n        return _batch_func\n    else:\n        # we apply it to everything\n        def _all_func(X, y):\n            return _func(X), y         \n        return _all_func\n    \ndef tf_augmentor(out_size,\n                intermediate_size = INTERMEDIATE_SIZE,\n                 intermediate_trans = 'crop',\n                 batch_size = 16,\n                   horizontal_flip = True, \n                  vertical_flip = False, \n                 random_brightness = True,\n                 random_contrast = True,\n                 random_saturation = True,\n                    random_hue = True,\n                  color_mode = 'rgb',\n                   preproc_func = preprocess_input,\n                   min_crop_percent = 0.001,\n                   max_crop_percent = 0.005,\n                   crop_probability = 0.5,\n                   rotation_range = 10):\n    \n    load_ops = tf_image_loader(out_size = intermediate_size, \n                               horizontal_flip=horizontal_flip, \n                               vertical_flip=vertical_flip, \n                               random_brightness = random_brightness,\n                               random_contrast = random_contrast,\n                               random_saturation = random_saturation,\n                               random_hue = random_hue,\n                               color_mode = color_mode,\n                               preproc_func = preproc_func,\n                               on_batch=False)\n    def batch_ops(X, y):\n        batch_size = tf.shape(X)[0]\n        with tf.name_scope('transformation'):\n            # code borrowed from https://becominghuman.ai/data-augmentation-on-gpu-in-tensorflow-13d14ecf2b19\n            # The list of affine transformations that our image will go under.\n            # Every element is Nx8 tensor, where N is a batch size.\n            transforms = []\n            identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n            if rotation_range > 0:\n                angle_rad = rotation_range / 180 * np.pi\n                angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n                transforms += [tf.contrib.image.angles_to_projective_transforms(angles, intermediate_size[0], intermediate_size[1])]\n\n            if crop_probability > 0:\n                crop_pct = tf.random_uniform([batch_size], min_crop_percent, max_crop_percent)\n                left = tf.random_uniform([batch_size], 0, intermediate_size[0] * (1.0 - crop_pct))\n                top = tf.random_uniform([batch_size], 0, intermediate_size[1] * (1.0 - crop_pct))\n                crop_transform = tf.stack([\n                      crop_pct,\n                      tf.zeros([batch_size]), top,\n                      tf.zeros([batch_size]), crop_pct, left,\n                      tf.zeros([batch_size]),\n                      tf.zeros([batch_size])\n                  ], 1)\n                coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n                transforms += [tf.where(coin, crop_transform, tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]))]\n            if len(transforms)>0:\n                X = tf.contrib.image.transform(X,\n                      tf.contrib.image.compose_transforms(*transforms),\n                      interpolation='BILINEAR') # or 'NEAREST'\n            if intermediate_trans=='scale':\n                X = tf.image.resize_images(X, out_size)\n            elif intermediate_trans=='crop':\n                X = tf.image.resize_image_with_crop_or_pad(X, out_size[0], out_size[1])\n            else:\n                raise ValueError('Invalid Operation {}'.format(intermediate_trans))\n            return X, y\n    def _create_pipeline(in_ds):\n        batch_ds = in_ds.map(load_ops, num_parallel_calls=4).batch(batch_size)\n        return batch_ds.map(batch_ops)\n    return _create_pipeline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07851e798db3d89ba13db7d4b56ab2b759221464","_cell_guid":"b5767f42-da63-4737-8f50-749c1a25aa84","trusted":true},"cell_type":"code","source":"def flow_from_dataframe(idg, \n                        in_df, \n                        path_col,\n                        y_col, \n                        shuffle = True, \n                        color_mode = 'rgb'):\n    files_ds = tf.data.Dataset.from_tensor_slices((in_df[path_col].values, \n                                                   np.stack(in_df[y_col].values,0)))\n    in_len = in_df[path_col].values.shape[0]\n    while True:\n        if shuffle:\n            files_ds = files_ds.shuffle(in_len) # shuffle the whole dataset\n        \n        next_batch = idg(files_ds).repeat().make_one_shot_iterator().get_next()\n        for i in range(max(in_len//32,1)):\n            # NOTE: if we loop here it is 'thread-safe-ish' if we loop on the outside it is completely unsafe\n            yield K.get_session().run(next_batch)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1848f5048a9e00668c3778a85deea97f980e4f1c","_cell_guid":"810bd229-fec9-43c4-b3bd-afd62e3e9552","trusted":true},"cell_type":"code","source":"\ncore_idg = tf_augmentor(out_size = IMG_SIZE, \n                        color_mode = 'rgb', \n                        vertical_flip = True,\n                        horizontal_flip = True,\n                        crop_probability=0.0, # crop doesn't work yet\n                        batch_size = BATCH_SIZE) \nvalid_idg = tf_augmentor(out_size = IMG_SIZE, color_mode = 'rgb', \n                         crop_probability=0.0, \n                         horizontal_flip = False, \n                         vertical_flip = False, \n                         random_brightness = False,\n                         random_contrast = False,\n                         random_saturation = False,\n                         random_hue = False,\n                         rotation_range = 0,\n                        batch_size = BATCH_SIZE)\n\ntrain_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'path',\n                            y_col = 'diagnosis')\n\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'diagnosis') # we can use much larger batches for evaluation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ad184b936d5cebae91a265a247d8e0e25920566"},"cell_type":"markdown","source":"# Validation Set\nWe do not perform augmentation at all on these images"},{"metadata":{"trusted":true,"_uuid":"6810407e25b887dd8b352f1e46fb3faceaa58ab7"},"cell_type":"code","source":"t_x, t_y = next(valid_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(c_y))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34ce892a19c9734511e2da1d0f2552b361dc826d"},"cell_type":"markdown","source":"# Training Set\nThese are augmented and a real mess"},{"metadata":{"_uuid":"8190b4ad60d49fa65af074dd138a19cb8787e983","scrolled":true,"_cell_guid":"2d62234f-aeb0-4eba-8a38-d713d819abf6","trusted":true},"cell_type":"code","source":"t_x, t_y = next(train_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(c_y))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"55d665e1e8a8d83b9db005a66a965f8a90c62da1","_cell_guid":"da22790a-672c-474e-b118-9eef15b53160"},"cell_type":"markdown","source":"# Attention Model\nThe basic idea is that a Global Average Pooling is too simplistic since some of the regions are more relevant than others. So we build an attention mechanism to turn pixels in the GAP on an off before the pooling and then rescale (Lambda layer) the results based on the number of pixels. The model could be seen as a sort of 'global weighted average' pooling. There is probably something published about it and it is very similar to the kind of attention models used in NLP.\nIt is largely based on the insight that the winning solution annotated and trained a UNET model to segmenting the hand and transforming it. This seems very tedious if we could just learn attention."},{"metadata":{"_uuid":"1f0dfaccda346d7bc4758e7329d61028d254a8d6","_cell_guid":"eeb36110-0cde-4450-a43c-b8f707adb235","trusted":true},"cell_type":"code","source":"from keras.applications.vgg16 import VGG16 as PTModel\nfrom keras.applications.inception_resnet_v2 import InceptionResNetV2 as PTModel\nfrom keras.applications.inception_v3 import InceptionV3 as PTModel\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda\nfrom keras.models import Model\nfrom keras.layers import BatchNormalization\nin_lay = Input(t_x.shape[1:])\nbase_pretrained_model = PTModel(input_shape =  t_x.shape[1:], include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = False\npt_depth = base_pretrained_model.get_output_shape_at(0)[-1]\npt_features = base_pretrained_model(in_lay)\n\nbn_features = BatchNormalization()(pt_features)\n\n# here we do an attention mechanism to turn pixels in the GAP on an off\n\nattn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\nattn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(1, \n                    kernel_size = (1,1), \n                    padding = 'valid', \n                    activation = 'sigmoid')(attn_layer)\n# fan it out to all of the channels\nup_c2_w = np.ones((1, 1, 1, pt_depth))\nup_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n               activation = 'linear', use_bias = False, weights = [up_c2_w])\nup_c2.trainable = False\nattn_layer = up_c2(attn_layer)\n\nmask_features = multiply([attn_layer, bn_features])\ngap_features = GlobalAveragePooling2D()(mask_features)\ngap_mask = GlobalAveragePooling2D()(attn_layer)\n# to account for missing values from the attention model\ngap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\ngap_dr = Dropout(0.25)(gap)\ndr_steps = Dropout(0.25)(Dense(128, activation = 'relu')(gap_dr))\nout_layer = Dense(train_df['diagnosis'].max()+1, activation = 'softmax')(dr_steps)\nretina_model = Model(inputs = [in_lay], outputs = [out_layer])\nretina_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.metrics import sparse_top_k_categorical_accuracy\ndef top_2_accuracy(in_gt, in_pred):\n    return sparse_top_k_categorical_accuracy(in_gt, in_pred, k=2)\n\nretina_model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy',\n                           metrics = ['categorical_accuracy', top_2_accuracy])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48b9764e16fb5af52aed35c82bae6299e67d5bc7","_cell_guid":"17803ae1-bed8-41a4-9a2c-e66287a24830","trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('retina')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=6) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"!rm -rf ~/.keras # clean up before starting training\nimport gc\ngc.enable()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2148479bfe41c5d9fd0faece4c75adea509dabe","_cell_guid":"58a75586-442b-4804-84a6-d63d5a42ea14","trusted":true},"cell_type":"code","source":"retina_model.fit_generator(train_gen, \n                           steps_per_epoch = train_df.shape[0]//BATCH_SIZE,\n                           validation_data = valid_gen, \n                           validation_steps = valid_df.shape[0]//BATCH_SIZE,\n                              epochs = 5, \n                              callbacks = callbacks_list,\n                             workers = 0, # tf-generators are not thread-safe\n                             use_multiprocessing=False, \n                             max_queue_size = 0\n                            )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a90f05dd206cd76c72d8c6278ebb93da41ee45f","_cell_guid":"4d0c45b0-bb23-48d2-83eb-bc3990043e26","trusted":true},"cell_type":"code","source":"# load the best version of the model\nretina_model.load_weights(weight_path)\nretina_model.save('full_retina_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b74f4ab850c6e82549d732b6f0524724b95b53c","_cell_guid":"f37dd4d8-ecd6-487a-90d8-74fe14a9a318","trusted":true},"cell_type":"code","source":"##### create one fixed dataset for evaluating\nfrom tqdm import tqdm_notebook\n# fresh valid gen\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'diagnosis') \nvbatch_count = (valid_df.shape[0]//BATCH_SIZE-1)\nout_size = vbatch_count*BATCH_SIZE\ntest_X = np.zeros((out_size,)+t_x.shape[1:], dtype = np.float32)\ntest_Y = np.zeros((out_size,)+t_y.shape[1:], dtype = np.float32)\nfor i, (c_x, c_y) in zip(tqdm_notebook(range(vbatch_count)), \n                         valid_gen):\n    j = i*BATCH_SIZE\n    test_X[j:(j+c_x.shape[0])] = c_x\n    test_Y[j:(j+c_x.shape[0])] = c_y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cca170eb40bc591f89748ede8aa35de4308faaaf","_cell_guid":"11f33f0a-61eb-488a-b7ea-4bc9d15ba8f9"},"cell_type":"markdown","source":"# Show Attention\nDid our attention model learn anything useful?"},{"metadata":{"_uuid":"ad5b085d351e79b950bf0c2ddc476799d5b0692f","_cell_guid":"e41a063f-35c9-410f-be63-f66b63ff9683","trusted":true},"cell_type":"code","source":"# get the attention layer since it is the only one with a single output dim\nfor attn_layer in retina_model.layers:\n    c_shape = attn_layer.get_output_shape_at(0)\n    if len(c_shape)==4:\n        if c_shape[-1]==1:\n            print(attn_layer)\n            break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00850972ae4298f49ed1838b3fc49c2d8fb07547","_cell_guid":"340eef36-f5b2-4b15-a59f-440061a427eb","trusted":true},"cell_type":"code","source":"import keras.backend as K\nrand_idx = np.random.choice(range(len(test_X)), size = 6)\nattn_func = K.function(inputs = [retina_model.get_input_at(0), K.learning_phase()],\n           outputs = [attn_layer.get_output_at(0)]\n          )\nfig, m_axs = plt.subplots(len(rand_idx), 2, figsize = (8, 4*len(rand_idx)))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nfor c_idx, (img_ax, attn_ax) in zip(rand_idx, m_axs):\n    cur_img = test_X[c_idx:(c_idx+1)]\n    attn_img = attn_func([cur_img, 0])[0]\n    img_ax.imshow(np.clip(cur_img[0,:,:,:]*127+127, 0, 255).astype(np.uint8))\n    attn_ax.imshow(attn_img[0, :, :, 0]/attn_img[0, :, :, 0].max(), cmap = 'viridis', \n                   vmin = 0, vmax = 1, \n                   interpolation = 'lanczos')\n    real_cat = test_Y[c_idx]\n    img_ax.set_title('Eye Image\\nCat:%2d' % (real_cat))\n    pred_cat = retina_model.predict(cur_img)\n    attn_ax.set_title('Attention Map\\nPred:%2.2f%%' % (100*pred_cat[0,int(real_cat)]))\nfig.savefig('attention_map.png', dpi = 300)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"244bac80d1ea2074e47932e367996e32cbab6a3d","_cell_guid":"24796de7-b1e9-4b3b-bcc6-d997aa3e6d16"},"cell_type":"markdown","source":"# Evaluate the results\nHere we evaluate the results by loading the best version of the model and seeing how the predictions look on the results. We then visualize spec"},{"metadata":{"_uuid":"b421b6183b1919a7414482f0b1ac611079e45174","_cell_guid":"d0edaf00-4b7c-4f65-af0b-e5a03b9b8428","trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, classification_report\npred_Y = retina_model.predict(test_X, batch_size = 32, verbose = True)\npred_Y_cat = np.argmax(pred_Y, -1)\ntest_Y_cat = test_Y\nprint('Accuracy on Test Data: %2.2f%%' % (100*accuracy_score(test_Y_cat, pred_Y_cat)))\nprint(classification_report(test_Y_cat, pred_Y_cat))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10162e055ca7cd52878a289bab377231787ab732","_cell_guid":"15189df2-3fed-495e-9661-97bb2b712dfd","trusted":true},"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nsns.heatmap(confusion_matrix(test_Y_cat, pred_Y_cat), \n            annot=True, fmt=\"d\", cbar = False, cmap = plt.cm.Blues, vmax = test_X.shape[0]//16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12dfe39ea80194062068589699953c6645e285d6","_cell_guid":"70827da6-bf91-4b65-80e9-bf1e6b885db3"},"cell_type":"markdown","source":"# ROC Curve for healthy vs sick\nHere we make an ROC curve for healthy (```severity == 0```) and sick (```severity>0```) to see how well the model works at just identifying the disease"},{"metadata":{"_uuid":"2b2aaee6c83043f721b0c9ed5bc4229eb7165200","_cell_guid":"829475ab-7db2-4421-b9ad-1a51971fd459","trusted":true},"cell_type":"code","source":"\nfrom sklearn.metrics import roc_curve, roc_auc_score\nsick_vec = test_Y_cat>0\nsick_score = np.sum(pred_Y[:,1:],1)\nfpr, tpr, _ = roc_curve(sick_vec, sick_score)\nfig, ax1 = plt.subplots(1,1, figsize = (6, 6), dpi = 150)\nax1.plot(fpr, tpr, 'b.-', label = 'Model Prediction (AUC: %2.2f)' % roc_auc_score(sick_vec, sick_score))\nax1.plot(fpr, fpr, 'g-', label = 'Random Guessing')\nax1.legend()\nax1.set_xlabel('False Positive Rate')\nax1.set_ylabel('True Positive Rate');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba87d0e7c3a77181487b99ca64d13de2aa8a21ee","scrolled":false,"_cell_guid":"c34f049f-b032-45bf-9d5e-a756ecc46a82","trusted":true},"cell_type":"code","source":"fig, m_axs = plt.subplots(2, 4, figsize = (32, 20))\nfor (idx, c_ax) in enumerate(m_axs.flatten()):\n    c_ax.imshow(np.clip(test_X[idx]*127+127,0 , 255).astype(np.uint8), cmap = 'bone')\n    c_ax.set_title('Actual Severity: {}\\n{}'.format(test_Y_cat[idx], \n                                                           '\\n'.join(['Predicted %02d (%04.1f%%): %s' % (k, 100*v, '*'*int(10*v)) for k, v in sorted(enumerate(pred_Y[idx]), key = lambda x: -1*x[1])])), loc='left')\n    c_ax.axis('off')\nfig.savefig('trained_img_predictions.png', dpi = 300)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction on Test Set"},{"metadata":{"_uuid":"eb6752295030ba512263433f8383711e4ca1c14c","_cell_guid":"f2e189dc-f80a-4b16-bb1d-5c05a155a80b","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(base_image_dir, 'test.csv'))\ntest_df['path'] = test_df['id_code'].map(lambda x: os.path.join(base_image_dir, 'test_images',\n                                                         '{}.png'.format(x)))\ntest_df['exists'] = test_df['path'].map(os.path.exists)\nprint(test_df['exists'].sum(), 'images found of', test_df.shape[0], 'total')\ntest_df.dropna(inplace = True)\ntest_df = test_df[test_df['exists']]\ntest_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### create one fixed dataset for evaluating\nfrom tqdm import tqdm_notebook\n# fresh valid gen\ntest_gen = flow_from_dataframe(valid_idg, test_df, \n                             path_col = 'path',\n                            y_col = 'path' # we want the path so we can match it back up to the input\n                              ) \nvbatch_count = (test_df.shape[0]//BATCH_SIZE+1)\nout_size = vbatch_count*BATCH_SIZE\npred_Y = []\npath_Y = []\nfor i, (c_x, c_y) in zip(tqdm_notebook(range(vbatch_count)), \n                         test_gen):\n    j = i*BATCH_SIZE\n    pred_Y[j:(j+c_x.shape[0])] = np.argmax(retina_model.predict(c_x), -1)\n    path_Y[j:(j+c_x.shape[0])] = c_y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make it into a dictionary so we can read out each row (missing ones we assume are zero like the `submissions_sample`"},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_dict = dict(zip(path_Y, pred_Y))\ntest_df['diagnosis'] = test_df['path'].map(lambda x: prediction_dict.get(x.encode(), 0))\ntest_df[['id_code', 'diagnosis']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['diagnosis'].hist(figsize=(10, 5))\ntest_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}