{"cells":[{"metadata":{"_cell_guid":"c67e806c-e0e6-415b-8cf0-ed92eba5ed37","_uuid":"34b6997bb115a11f47a7f54ce9d9052791b2e707"},"cell_type":"markdown","source":"# Overview\nThe goal is to make a nice retinopathy model by using a pretrained inception v3 as a base and retraining some modified final layers with attention\n\nThis can be massively improved with \n* high-resolution images\n* better data sampling\n* ensuring there is no leaking between training and validation sets, ```sample(replace = True)``` is real dangerous\n* better target variable (age) normalization\n* pretrained models\n* attention/related techniques to focus on areas"},{"metadata":{"_cell_guid":"c3cc4285-bfa4-4612-ac5f-13d10678c09a","_uuid":"725d378daf5f836d4885d67240fc7955f113309d","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # showing and rendering figures\n# io related\nfrom skimage.io import imread\nimport os\nfrom glob import glob\n# not needed in Kaggle, but required in Jupyter\n%matplotlib inline ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9a7a25ec995ad5289ea7c268c20642f676cf29d"},"cell_type":"code","source":"batch_size = 64\nLEARNING_RATE = 3e-4\nIMG_SIZE = (224, 224) # slightly smaller than vgg16 normally expects\nEPOCHS = 15","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c4b38df6-ffa1-4847-b605-511e72b68231","_uuid":"346da81db6ee7a34af8da8af245b42e681f2ba48","trusted":true},"cell_type":"code","source":"base_image_dir = os.path.join('..', 'input')\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\nall_images = glob(os.path.join(base_image_dir, 'images', '*', '*'))\nprint(len(all_images), 'images found')\nfull_food_df = pd.DataFrame(dict(path = all_images))\nfood_cat = LabelEncoder()\nfull_food_df['category'] = full_food_df['path'].map(lambda x: x.split('/')[-2].replace('_', ' '))\nfood_cat.fit(full_food_df['category'].values)\nfull_food_df['cat_vec'] = full_food_df['category'].map(lambda x: to_categorical(food_cat.transform([x]), num_classes=len(food_cat.classes_))[0])\nfull_food_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"818da6ca-bbff-4ca0-ad57-ef3a145ae863","_uuid":"688e4340238e013b8459b6f6470993c7de492d83"},"cell_type":"markdown","source":"# Examine the distribution of classes"},{"metadata":{"_cell_guid":"5c8bd288-8261-4cbe-a954-e62ac795cc3e","_uuid":"60a8111c4093ca6f69d27a4499442ba7dd750839","trusted":true},"cell_type":"code","source":"full_food_df.groupby(['category']).size().plot.bar(figsize = (10, 5))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0ba697ed-85bb-4e9a-9765-4c367db078d1","_uuid":"4df45776bae0b8a1bf9d3eb4eaaebce6e24d726d"},"cell_type":"markdown","source":"# Split Data into Training and Validation"},{"metadata":{"_cell_guid":"1192c6b3-a940-4fa0-a498-d7e0d400a796","_uuid":"a48b300ca4d37a6e8b39f82e3c172739635e4baa","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, valid_df = train_test_split(full_food_df, \n                                   test_size = 0.25, \n                                   random_state = 2018,\n                                   stratify = full_food_df['category'])\nprint('train', train_df.shape[0], 'validation', valid_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9954bfda-29bd-4c4d-b526-0a972b3e43e2","_uuid":"9529ab766763a9f122786464c24ab1ebe22c6006","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom keras import backend as K\nfrom keras.applications.vgg16 import preprocess_input\nimport numpy as np\ndef tf_image_loader(out_size, \n                      horizontal_flip = True, \n                      vertical_flip = False, \n                     random_brightness = True,\n                     random_contrast = True,\n                    random_saturation = True,\n                    random_hue = True,\n                      color_mode = 'rgb',\n                       preproc_func = preprocess_input,\n                       on_batch = False):\n    def _func(X):\n        with tf.name_scope('image_augmentation'):\n            with tf.name_scope('input'):\n                X = tf.image.decode_png(tf.read_file(X), channels = 3 if color_mode == 'rgb' else 0)\n                X = tf.image.resize_images(X[:,:,::-1], out_size)\n            with tf.name_scope('augmentation'):\n                if horizontal_flip:\n                    X = tf.image.random_flip_left_right(X)\n                if vertical_flip:\n                    X = tf.image.random_flip_up_down(X)\n                if random_brightness:\n                    X = tf.image.random_brightness(X, max_delta = 0.15)\n                if random_saturation:\n                    X = tf.image.random_saturation(X, lower = 0.5, upper = 2)\n                if random_hue:\n                    X = tf.image.random_hue(X, max_delta = 0.15)\n                if random_contrast:\n                    X = tf.image.random_contrast(X, lower = 0.75, upper = 1.5)\n                return preproc_func(X)\n    if on_batch: \n        # we are meant to use it on a batch\n        def _batch_func(X, y):\n            return tf.map_fn(_func, X), y\n        return _batch_func\n    else:\n        # we apply it to everything\n        def _all_func(X, y):\n            return _func(X), y         \n        return _all_func\n    \ndef tf_augmentor(out_size,\n                intermediate_size = (480, 480),\n                 intermediate_trans = 'crop',\n                 batch_size = 16,\n                   horizontal_flip = True, \n                  vertical_flip = False, \n                 random_brightness = True,\n                 random_contrast = True,\n                 random_saturation = True,\n                    random_hue = True,\n                  color_mode = 'rgb',\n                   preproc_func = preprocess_input,\n                   min_crop_percent = 0.001,\n                   max_crop_percent = 0.005,\n                   crop_probability = 0.5,\n                   rotation_range = 10):\n    \n    load_ops = tf_image_loader(out_size = intermediate_size, \n                               horizontal_flip=horizontal_flip, \n                               vertical_flip=vertical_flip, \n                               random_brightness = random_brightness,\n                               random_contrast = random_contrast,\n                               random_saturation = random_saturation,\n                               random_hue = random_hue,\n                               color_mode = color_mode,\n                               preproc_func = preproc_func,\n                               on_batch=False)\n    def batch_ops(X, y):\n        batch_size = tf.shape(X)[0]\n        with tf.name_scope('transformation'):\n            # code borrowed from https://becominghuman.ai/data-augmentation-on-gpu-in-tensorflow-13d14ecf2b19\n            # The list of affine transformations that our image will go under.\n            # Every element is Nx8 tensor, where N is a batch size.\n            transforms = []\n            identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n            if rotation_range > 0:\n                angle_rad = rotation_range / 180 * np.pi\n                angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n                transforms += [tf.contrib.image.angles_to_projective_transforms(angles, intermediate_size[0], intermediate_size[1])]\n\n            if crop_probability > 0:\n                crop_pct = tf.random_uniform([batch_size], min_crop_percent, max_crop_percent)\n                left = tf.random_uniform([batch_size], 0, intermediate_size[0] * (1.0 - crop_pct))\n                top = tf.random_uniform([batch_size], 0, intermediate_size[1] * (1.0 - crop_pct))\n                crop_transform = tf.stack([\n                      crop_pct,\n                      tf.zeros([batch_size]), top,\n                      tf.zeros([batch_size]), crop_pct, left,\n                      tf.zeros([batch_size]),\n                      tf.zeros([batch_size])\n                  ], 1)\n                coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n                transforms += [tf.where(coin, crop_transform, tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]))]\n            if len(transforms)>0:\n                X = tf.contrib.image.transform(X,\n                      tf.contrib.image.compose_transforms(*transforms),\n                      interpolation='BILINEAR') # or 'NEAREST'\n            if intermediate_trans=='scale':\n                X = tf.image.resize_images(X, out_size)\n            elif intermediate_trans=='crop':\n                X = tf.image.resize_image_with_crop_or_pad(X, out_size[0], out_size[1])\n            else:\n                raise ValueError('Invalid Operation {}'.format(intermediate_trans))\n            return X, y\n    def _create_pipeline(in_ds):\n        batch_ds = in_ds.map(load_ops, num_parallel_calls=4).batch(batch_size)\n        return batch_ds.map(batch_ops)\n    return _create_pipeline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b5767f42-da63-4737-8f50-749c1a25aa84","_uuid":"07851e798db3d89ba13db7d4b56ab2b759221464","trusted":true},"cell_type":"code","source":"def flow_from_dataframe(idg, \n                        in_df, \n                        path_col,\n                        y_col, \n                        shuffle = True, \n                        color_mode = 'rgb'):\n    files_ds = tf.data.Dataset.from_tensor_slices((in_df[path_col].values, \n                                                   np.stack(in_df[y_col].values,0)))\n    in_len = in_df[path_col].values.shape[0]\n    while True:\n        if shuffle:\n            files_ds = files_ds.shuffle(in_len) # shuffle the whole dataset\n        \n        next_batch = idg(files_ds).repeat().make_one_shot_iterator().get_next()\n        for i in range(max(in_len//32,1)):\n            # NOTE: if we loop here it is 'thread-safe-ish' if we loop on the outside it is completely unsafe\n            yield K.get_session().run(next_batch)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"810bd229-fec9-43c4-b3bd-afd62e3e9552","_uuid":"1848f5048a9e00668c3778a85deea97f980e4f1c","trusted":true},"cell_type":"code","source":"core_idg = tf_augmentor(out_size = IMG_SIZE, \n                        color_mode = 'rgb', \n                        vertical_flip = True,\n                        crop_probability=0.0, # crop doesn't work yet\n                        rotation_range = 0,\n                        batch_size = batch_size) \nvalid_idg = tf_augmentor(out_size = IMG_SIZE, color_mode = 'rgb', \n                         crop_probability=0.0, \n                         horizontal_flip = False, \n                         vertical_flip = False, \n                         random_brightness = False,\n                         random_contrast = False,\n                         random_saturation = False,\n                         random_hue = False,\n                         rotation_range = 0,\n                        batch_size = batch_size)\n\ntrain_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'path',\n                            y_col = 'cat_vec')\n\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'cat_vec') # we can use much larger batches for evaluation","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"583a42e3-6c64-4637-b5dc-61f205e63b2c","_uuid":"2ad184b936d5cebae91a265a247d8e0e25920566"},"cell_type":"markdown","source":"# Validation Set\nWe do not perform augmentation at all on these images"},{"metadata":{"_cell_guid":"14949161-cae1-41b1-a744-95cf4e491bcd","_uuid":"6810407e25b887dd8b352f1e46fb3faceaa58ab7","trusted":true},"cell_type":"code","source":"t_x, t_y = next(valid_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('{}'.format(food_cat.classes_[np.argmax(c_y, -1)]))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7a472209-07fa-4ff9-af4b-72d6ccacb21e","_uuid":"34ce892a19c9734511e2da1d0f2552b361dc826d"},"cell_type":"markdown","source":"# Training Set\nThese are augmented and a real mess"},{"metadata":{"_cell_guid":"2d62234f-aeb0-4eba-8a38-d713d819abf6","_uuid":"8190b4ad60d49fa65af074dd138a19cb8787e983","scrolled":true,"trusted":true},"cell_type":"code","source":"t_x, t_y = next(train_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('{}'.format(food_cat.classes_[np.argmax(c_y, -1)]))\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"da22790a-672c-474e-b118-9eef15b53160","_uuid":"55d665e1e8a8d83b9db005a66a965f8a90c62da1"},"cell_type":"markdown","source":"# Basic VGG Model"},{"metadata":{"_cell_guid":"eeb36110-0cde-4450-a43c-b8f707adb235","_uuid":"1f0dfaccda346d7bc4758e7329d61028d254a8d6","trusted":true},"cell_type":"code","source":"from keras.applications.vgg16 import VGG16 as PTModel\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda\nfrom keras.models import Model\nin_lay = Input(t_x.shape[1:])\nbase_pretrained_model = PTModel(input_shape = t_x.shape[1:], \n                                include_top = True, \n                                weights = None, # 'imagenet')\n                                classes=t_y.shape[-1]\n                               )\n\nfood_model = base_pretrained_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"304488eab5ede23e77849723aef12a1f4602d7c7"},"cell_type":"code","source":"from keras.metrics import top_k_categorical_accuracy\nfrom keras.losses import categorical_crossentropy, binary_crossentropy\nfrom keras.optimizers import Adam\ndef top_5_accuracy(in_gt, in_pred):\n    return top_k_categorical_accuracy(in_gt, in_pred, k=5)\n\nfood_model.compile(optimizer = Adam(lr=LEARNING_RATE), \n                   loss = 'categorical_crossentropy',\n                   metrics = ['categorical_accuracy', top_5_accuracy])\nfood_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"17803ae1-bed8-41a4-9a2c-e66287a24830","_uuid":"48b9764e16fb5af52aed35c82bae6299e67d5bc7","trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('food')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=6) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"58a75586-442b-4804-84a6-d63d5a42ea14","_uuid":"b2148479bfe41c5d9fd0faece4c75adea509dabe","trusted":true},"cell_type":"code","source":"food_model.fit_generator(train_gen, \n                           steps_per_epoch = 100, #train_df.shape[0]//batch_size,\n                           validation_data = valid_gen, \n                           validation_steps = 10, #valid_df.shape[0]//batch_size,\n                              epochs = EPOCHS, \n                              callbacks = callbacks_list,\n                             workers = 0, # tf-generators are not thread-safe\n                             use_multiprocessing=False, \n                             max_queue_size = 0\n                            )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4d0c45b0-bb23-48d2-83eb-bc3990043e26","_uuid":"3a90f05dd206cd76c72d8c6278ebb93da41ee45f","trusted":true},"cell_type":"code","source":"# load the best version of the model\nfood_model.load_weights(weight_path)\nfood_model.save('full_food_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd8199becdc7d9ae403deb2e3a21e7826c294c9c"},"cell_type":"code","source":"import gc\ngc.enable()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f37dd4d8-ecd6-487a-90d8-74fe14a9a318","_uuid":"2b74f4ab850c6e82549d732b6f0524724b95b53c","trusted":true},"cell_type":"code","source":"##### create one fixed dataset for evaluating\nfrom tqdm import tqdm_notebook\n# fresh valid gen\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'cat_vec') \nvbatch_count = min(5, (valid_df.shape[0]//batch_size-1))\nout_size = vbatch_count*batch_size\ntest_X = np.zeros((out_size,)+t_x.shape[1:], dtype = np.float32)\ntest_Y = np.zeros((out_size,)+t_y.shape[1:], dtype = np.float32)\nfor i, (c_x, c_y) in zip(tqdm_notebook(range(vbatch_count)), \n                         valid_gen):\n    j = i*batch_size\n    test_X[j:(j+c_x.shape[0])] = c_x\n    test_Y[j:(j+c_x.shape[0])] = c_y","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"24796de7-b1e9-4b3b-bcc6-d997aa3e6d16","_uuid":"244bac80d1ea2074e47932e367996e32cbab6a3d"},"cell_type":"markdown","source":"# Evaluate the results\nHere we evaluate the results by loading the best version of the model and seeing how the predictions look on the results. We then visualize spec"},{"metadata":{"_cell_guid":"d0edaf00-4b7c-4f65-af0b-e5a03b9b8428","_uuid":"b421b6183b1919a7414482f0b1ac611079e45174","scrolled":false,"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, classification_report\npred_Y = food_model.predict(test_X, batch_size = 32, verbose = True)\npred_Y_cat = np.argmax(pred_Y, -1)\ntest_Y_cat = np.argmax(test_Y, -1)\nprint('Accuracy on Test Data: %2.2f%%' % (100*accuracy_score(test_Y_cat, pred_Y_cat)))\nprint(classification_report(test_Y_cat, pred_Y_cat, target_names = food_cat.classes_))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"15189df2-3fed-495e-9661-97bb2b712dfd","_uuid":"10162e055ca7cd52878a289bab377231787ab732","trusted":true},"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfig, ax1 = plt.subplots(1,1, figsize = (20, 20))\nsns.heatmap(confusion_matrix(test_Y_cat, pred_Y_cat), \n            annot=False, fmt=\"d\", cbar = False, cmap = plt.cm.Blues, vmax = test_X.shape[0]//16, ax = ax1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f4b605523534e3e7fcf5777f4e99b8f79668fce"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}