{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nsns.set(style=\"darkgrid\")\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import applications \nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Sequential, Model \nfrom tensorflow.keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, MaxPooling2D, BatchNormalization\nfrom tensorflow.keras import backend as k \nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 224\nNUM_CLASSES = 5\nBATCH_SIZE = 32","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Enhance Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\n\ndef grayCLAHE(img):\n    img = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY).astype(np.uint8)\n    \n    # CLAHE (Contrast Limited Adaptive Histogram Equalization)\n    # https://docs.opencv.org/3.1.0/d5/daf/tutorial_py_histogram_equalization.html\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    img2 = clahe.apply(img)\n    \n    claheImg = cv2.cvtColor(img2,cv2.COLOR_GRAY2RGB)\n    \n    return claheImg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessImage(img):\n    #img = grayCLAHE(img)    \n    return applications.mobilenet_v2.preprocess_input(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv2.imread('../input/aptos2019-blindness-detection/train_images/ae49cc60f251.png',-1)\nprint(img.shape)\n_, axes = plt.subplots(nrows=1, ncols=2, figsize=(10, 10))\naxes[0].imshow(img);\naxes[1].imshow(preprocessImage(img)); ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Data Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntrain_df['FILE'] = [f\"{x.id_code}.png\" for _,x in train_df.iterrows()]\ntrain_df.diagnosis = train_df.diagnosis.astype('str')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_datagen = ImageDataGenerator(validation_split=0.1, \n                                   #rescale=1./255.,\n                                   #featurewise_center=True,\n                                   #samplewise_center=False,\n                                   #featurewise_std_normalization=True,\n                                   #samplewise_std_normalization=False,\n                                   rotation_range=360,\n                                   #shear_range=5,\n                                   width_shift_range=0.2,\n                                   height_shift_range=0.2,\n                                   zoom_range=0.2,\n                                   horizontal_flip=True, \n                                   vertical_flip=True,\n                                   preprocessing_function=preprocessImage #applications.mobilenet_v2.preprocess_input\n                                  )\n\nTRAIN_IMAGE_DIRECTORY = '../input/aptos2019-blindness-detection/train_images/'\n\ntrain_generator = image_datagen.flow_from_dataframe(train_df, \n                                                    directory=TRAIN_IMAGE_DIRECTORY, \n                                                    subset='training', \n                                                    shuffle=True,\n                                                    drop_duplicates=False, \n                                                    color_mode='rgb',\n                                                    x_col='FILE', \n                                                    #y_col=['D_0','D_1','D_2','D_3','D_4'],\n                                                    #class_mode='other', \n                                                    y_col='diagnosis',\n                                                    class_mode='categorical',\n                                                    batch_size=BATCH_SIZE, \n                                                    target_size=(SIZE,SIZE), \n                                                    #seed=42\n                                                   )\n\nvalid_generator = image_datagen.flow_from_dataframe(train_df, \n                                                    directory=TRAIN_IMAGE_DIRECTORY, \n                                                    subset='validation', \n                                                    shuffle=False,\n                                                    drop_duplicates=False, \n                                                    color_mode='rgb',\n                                                    x_col='FILE', \n                                                    #y_col=['D_0','D_1','D_2','D_3','D_4'],\n                                                    #class_mode='other', \n                                                    y_col='diagnosis',\n                                                    class_mode='categorical',\n                                                    batch_size=BATCH_SIZE, \n                                                    target_size=(SIZE,SIZE),\n                                                    #seed=42\n                                                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAIN_IMAGES = len(train_generator.filenames)\nNUM_VALID_IMAGES = len(valid_generator.filenames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Data Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\ntest_df['FILE'] = [f\"{x.id_code}.png\" for _,x in test_df.iterrows()]\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_datagen = ImageDataGenerator(#rescale=1./255.,\n                                  #featurewise_center=True,\n                                  #samplewise_center=False,\n                                  #featurewise_std_normalization=True,\n                                  #samplewise_std_normalization=False,\n                                  preprocessing_function=preprocessImage #applications.mobilenet_v2.preprocess_input\n                                 )\n\ntest_generator = test_datagen.flow_from_dataframe(test_df, \n                                                  directory='../input/aptos2019-blindness-detection/test_images/', \n                                                  shuffle=False,\n                                                  drop_duplicates=False, \n                                                  color_mode='rgb',\n                                                  x_col='FILE', y_col=None,\n                                                  batch_size=2, \n                                                  target_size=(SIZE,SIZE), \n                                                  class_mode=None\n                                                 )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MobileNetV2 \n## Download Trained Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"mnet2 = applications.mobilenet_v2.MobileNetV2(weights = None, include_top=True, input_shape = (SIZE, SIZE, 3) )\nmnet2.load_weights('../input/mobilenetv2-full-weights/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224.h5')\n\nmnet2.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Customize MobileNetV2 Head"},{"metadata":{"trusted":true},"cell_type":"code","source":"x = mnet2.layers[-2].output\nx = Flatten()(x)\nx = Dropout(0.2)(x)\nx = Dense(512)(x)\nx = Dropout(0.3)(x)\npredictions = Dense(NUM_CLASSES, activation='softmax', name='predictions')(x)\n\ncustom_model = Model(inputs=mnet2.input, outputs=predictions)\n\nfor layer in custom_model.layers[:-10]:\n    layer.trainable = False\n\nfor i, layer in enumerate(custom_model.layers):\n    print(f'layer {i}: `{layer.name}` {layer.trainable}')\n#custom_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from tensorflow.keras.optimizers import *\n\ncustom_model.compile(loss='categorical_crossentropy',\n                     #optimizer=Adam(0.0001),\n                     optimizer=SGD(lr=1.62e-2, decay=1e-6, momentum=0.9, nesterov=True),\n                     #optimizer=Adagrad(0.0001),\n                     metrics=['accuracy'])\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class expandTrainableLayers(tf.keras.callbacks.Callback):\n    def reportLayers(self):\n        for i, layer in enumerate(custom_model.layers):\n                print(f'layer {i}: `{layer.name}` {layer.trainable}')\n    \n    def on_epoch_end(self, epoch, logs=None):\n        if epoch+1==1:\n            for layer in custom_model.layers:\n                layer.trainable = True\n                \n            for layer in custom_model.layers[:-50]:\n                layer.trainable = False\n            \n            self.reportLayers()\n            \n        if epoch+1==2:\n            for layer in custom_model.layers:\n                layer.trainable = True\n                \n            for layer in custom_model.layers[:-100]:\n                layer.trainable = False\n            \n            self.reportLayers()\n\n        if epoch+1==3:\n            for layer in custom_model.layers:\n                layer.trainable = True\n\n            self.reportLayers()\n            \n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n\nlr_reduce = ReduceLROnPlateau(monitor='val_acc', factor=.5, patience=3, verbose=1, \n                              mode='max', min_delta=0.001, cooldown=0, min_lr=0)\n\ncheck_pt = ModelCheckpoint('../checkpoint', monitor='val_acc', verbose=1, save_best_only=True, \n                           save_weights_only=False, mode='max', save_freq='epoch')\n\nearly_stop = EarlyStopping(monitor='val_acc', patience=10, verbose=1, \n                           mode='max', baseline=None, restore_best_weights=True)\n\nmoreTrainableLayers = expandTrainableLayers()\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory = custom_model.fit_generator(train_generator,\n                                     steps_per_epoch=2*NUM_TRAIN_IMAGES // BATCH_SIZE,\n                                     validation_data=valid_generator,\n                                     validation_steps= NUM_VALID_IMAGES // BATCH_SIZE,\n                                     callbacks=[lr_reduce, check_pt, early_stop, moreTrainableLayers],\n                                     epochs=70)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for metric in custom_model.metrics_names:\n    # Plot training & validation loss and metrics\n    ROLLING_WINDOW = 5\n    \n    plt.figure(figsize=(16,7))\n    plt.plot(history.history[metric])\n    plt.plot(pd.DataFrame(data=history.history[metric]).\\\n             rename(columns={0:f'{metric} {ROLLING_WINDOW} Rolling Average'}).\\\n             rolling(ROLLING_WINDOW).mean())\n    plt.plot(history.history[f'val_{metric}'])\n    plt.plot(pd.DataFrame(data=history.history[f'val_{metric}']).\\\n             rename(columns={0:f'val_{metric} {ROLLING_WINDOW} Rolling Average'}).\\\n             rolling(ROLLING_WINDOW).mean())\n    \n    plt.title(f'Model {metric}')\n    plt.ylabel(metric)\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper right');\n    \n    plt.show()\n    \n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nreport = classification_report(y_true=valid_generator.classes, \n                               y_pred=np.argmax(custom_model.predict_generator(valid_generator),axis=1),\n                               target_names=list(valid_generator.class_indices.keys())\n                              )\n\nprint(report)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npred = custom_model.predict_generator(test_generator, steps=test_df.shape[0]//2)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDICTION = np.argmax(pred, axis=1) \nPREDICTION.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['diagnosis'] = PREDICTION\ntest_df[['id_code','diagnosis']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[['id_code','diagnosis']].to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}