{"cells":[{"metadata":{},"cell_type":"markdown","source":"# APTOS - Classification #"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action='ignore', category=UserWarning)\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport time\nimport cv2\nimport gc\n\nfrom keras import backend as K\nfrom keras import losses\nfrom keras.applications.resnet50 import ResNet50, preprocess_input\n#from keras.applications.densenet import DenseNet121, preprocess_input\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import Dense, GlobalAveragePooling2D, GlobalMaxPooling2D, MaxPooling2D, Dropout, Conv2D, BatchNormalization, Input, Flatten, LeakyReLU\nfrom keras.models import load_model, Model, Sequential\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import to_categorical\nfrom keras.preprocessing import image\n\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Constants ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 575\nIMAGE_SIZE = 256\nNUM_CLASSES = 5\nBATCH_SIZE = 16\nINITIAL_EPOCHS = 5\nMIDDLE_EPOCHS = 10\nFINAL_EPOCHS = 30\nINITIAL_LR = 5e-3\nMIDDLE_LR = 5e-4\nFINAL_LR = 1e-5\nNUM_SAMPLES_2015 = 1100\nNUM_SAMPLES_2019 = 1600\nTEST_SIZE = 0.3\nQUEUE_SIZE = 300\nWORKERS = 3\n\ntrain_directory_2015 = '../input/aptos-converted-2015/train_images/'\ntrain_labels_2015 = '../input/aptos-converted-2015/train.csv'\n\ntrain_directory_2019 = '../input/aptos-converted/training_aptos/'\ntrain_labels_2019 = '../input/aptos-converted/train.csv'\n\ntest_image_directory = '../input/aptos2019-blindness-detection/test_images/'\ntest_data_file = '../input/aptos2019-blindness-detection/test.csv'\n\nweights_file = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'\n#weights_file = '../input/densenet-keras/DenseNet-BC-121-32-no-top.h5'\nbest_weight_file = 'aptos_best_weights.h5'\nsubmission_file = 'submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#os.listdir('../input/aptos-converted/training_aptos')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data files ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2015 = pd.read_csv(train_labels_2015).sample(frac=1)\ndf_train_2019 = pd.read_csv(train_labels_2019).sample(frac=1)\n\ndf_train_2015['image'] = df_train_2015['image'].apply(lambda i : \"{}.jpeg\".format(i))\ndf_train_2019['id_code'] = df_train_2019['id_code'].apply(lambda i : \"{}.png\".format(i))\n\ndf_train_2015.columns = ['image', 'diagnosis']\ndf_train_2019.columns = ['image', 'diagnosis']\n\ndf_train_2015['diagnosis'] = df_train_2015['diagnosis'].astype('str')\ndf_train_2019['diagnosis'] = df_train_2019['diagnosis'].astype('str')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Class balancing ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2015_0 = df_train_2015[df_train_2015['diagnosis'] == '0'].sample(NUM_SAMPLES_2015, replace=False, random_state=SEED)\ndf_train_2015_1 = df_train_2015[df_train_2015['diagnosis'] == '1'].sample(NUM_SAMPLES_2015, replace=False, random_state=SEED)\ndf_train_2015_2 = df_train_2015[df_train_2015['diagnosis'] == '2'].sample(NUM_SAMPLES_2015, replace=False, random_state=SEED)\ndf_train_2015_3 = df_train_2015[df_train_2015['diagnosis'] == '3'].sample(NUM_SAMPLES_2015, replace=True, random_state=SEED)\ndf_train_2015_4 = df_train_2015[df_train_2015['diagnosis'] == '4'].sample(NUM_SAMPLES_2015, replace=True, random_state=SEED)\ndf_train_2015 = pd.concat([df_train_2015_0, df_train_2015_1, df_train_2015_2, df_train_2015_3, df_train_2015_4])\ndf_train_2015 = df_train_2015.sample(frac=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2019_0 = df_train_2019[df_train_2019['diagnosis'] == '0'].sample(NUM_SAMPLES_2019, replace=False, random_state=SEED)\ndf_train_2019_1 = df_train_2019[df_train_2019['diagnosis'] == '1'].sample(NUM_SAMPLES_2019, replace=True, random_state=SEED)\ndf_train_2019_2 = df_train_2019[df_train_2019['diagnosis'] == '2'].sample(NUM_SAMPLES_2019, replace=True, random_state=SEED)\ndf_train_2019_3 = df_train_2019[df_train_2019['diagnosis'] == '3'].sample(NUM_SAMPLES_2019, replace=True, random_state=SEED)\ndf_train_2019_4 = df_train_2019[df_train_2019['diagnosis'] == '4'].sample(NUM_SAMPLES_2019, replace=True, random_state=SEED)\ndf_train_2019 = pd.concat([df_train_2019_0, df_train_2019_1, df_train_2019_2, df_train_2019_3, df_train_2019_4])\ndf_train_2019 = df_train_2019.sample(frac=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LR Scheduler ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_scheduler(epoch):\n    step = (FINAL_LR - 1e-6) / FINAL_EPOCHS\n    lr = FINAL_LR - (epoch * step)\n    print(\"Reducing learning rate to {}\".format(lr))\n    return lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Kappa callback ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Kappa(Callback):\n    def __init__(self, val_data, val_stop_patience=5):\n        super(Callback, self).__init__()\n        self.validation_data = val_data\n        self.val_kappas = []\n        self.val_stop_patience = val_stop_patience\n\n    def on_epoch_end(self, epoch, logs={}):\n        print('Calculating Kappa score...')\n        y_pred = [] #np.empty(self.validation_data.n, dtype=np.uint8)\n        y_val = [] #np.empty(self.validation_data.n, dtype=np.uint8)\n        w_size = 0\n        for idx in range(self.validation_data.n // self.validation_data.batch_size):\n            x, y = self.validation_data.next()\n            y_res = np.argmax(y, axis=1)\n            y_pres = np.argmax(model.predict(x), axis=1)\n            for res in y_pres:\n                y_pred.append(res)\n            for res in y_res:\n                y_val.append(res)\n\n        print(\"y_val(50)=\", y_val[:50])\n        print(\"y_pred(50)=\", y_pred[:50])\n        \n        _val_kappa = cohen_kappa_score(\n            np.array(y_val),\n            np.array(y_pred), \n            weights='quadratic')\n\n        self.val_kappas.append(_val_kappa)\n        print(\"val_kappa:\", np.round(_val_kappa, 3))\n\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save_weights(best_weight_file)\n            self.val_stop_p_count = 0\n        else:\n            # stop the training\n            self.val_stop_p_count = self.val_stop_p_count + 1;\n            if(self.val_stop_p_count > self.val_stop_patience):\n                print(\"Epoch %05d: early stopping \" % epoch)\n                self.model.stop_training = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ordinal loss function ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ordinal_loss(y_true, y_pred):\n    weights = K.cast(K.abs(K.argmax(y_true, axis=1) - K.argmax(y_pred, axis=1))/(K.int_shape(y_pred)[1] - 1), dtype='float32')\n    return (1.0 + weights) * losses.categorical_crossentropy(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image processing function ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_image(img):\n    img = preprocess_input(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model definition ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"def design_model():\n    model = Sequential()\n    model.add(Conv2D(filters=16, kernel_size=(2, 2), input_shape=[IMAGE_SIZE, IMAGE_SIZE, 3], activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(filters=32, kernel_size=(2, 2), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(filters=64, kernel_size=(2, 2), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(BatchNormalization())\n    model.add(GlobalAveragePooling2D())\n    model.add(Dense(units=1000, activation='relu'))\n    model.add(Dropout(rate=0.3))\n    model.add(Dense(units=1000, activation='relu'))\n    model.add(Dropout(rate=0.3))\n    model.add(Dense(NUM_CLASSES, activation='softmax'))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    input_tensor = Input(shape=(IMAGE_SIZE, IMAGE_SIZE, 3))\n    base_model = ResNet50(include_top=False, weights=None, input_tensor=input_tensor)\n    #base_model = DenseNet121(include_top=False, weights=None, input_tensor=input_tensor)\n    base_model.load_weights(weights_file)\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    output_tensor = Dense(NUM_CLASSES, activation='softmax')(x)\n    model = Model(inputs=input_tensor, outputs=output_tensor)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run the training ##"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = design_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### Initial training on top classifier only, 2015 data ####"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = ImageDataGenerator(preprocessing_function=process_image, horizontal_flip=True, vertical_flip=True, zoom_range=0.1, rotation_range=10)\n#valid_generator = ImageDataGenerator(preprocessing_function=process_image)\ntrain_flow = train_generator.flow_from_dataframe(directory=train_directory_2015, dataframe=df_train_2015, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=False, batch_size=BATCH_SIZE, seed=SEED)\n#valid_flow = valid_generator.flow_from_dataframe(directory=train_directory_2019, dataframe=df_train_2019, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=True, target_size=(IMAGE_SIZE, IMAGE_SIZE), batch_size=BATCH_SIZE, seed=SEED)\n\n#for layer in model.layers[:-3]:\n#    layer.trainable = False\n#for layer in model.layers[-3:]:\n#    layer.trainable = True\n\nmodel.compile(optimizer=Adam(INITIAL_LR), loss=ordinal_loss, metrics=['accuracy'])\n\nmodel.fit_generator(\n    generator=train_flow,\n    epochs=INITIAL_EPOCHS,\n    steps_per_epoch = train_flow.n // train_flow.batch_size,\n    verbose=1,\n#    validation_data=valid_flow,\n#    validation_steps = valid_flow.n // valid_flow.batch_size,\n    max_queue_size = QUEUE_SIZE,\n    workers = WORKERS\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### Training on full model, 2015 data ####"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = ImageDataGenerator(preprocessing_function=process_image, horizontal_flip=True, vertical_flip=True, zoom_range=0.1, rotation_range=10)\n#valid_generator = ImageDataGenerator(preprocessing_function=process_image)\ntrain_flow = train_generator.flow_from_dataframe(directory=train_directory_2015, dataframe=df_train_2015, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=False, batch_size=BATCH_SIZE, seed=SEED)\n#valid_flow = valid_generator.flow_from_dataframe(directory=train_directory_2019, dataframe=df_train_2019, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=True, target_size=(IMAGE_SIZE, IMAGE_SIZE), batch_size=BATCH_SIZE, seed=SEED)\n\n#lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, min_lr=1e-6, verbose=1)\n#callbacks = []\n\nfor layer in model.layers:\n    layer.trainable = True\n\nmodel.compile(optimizer=Adam(MIDDLE_LR), loss=ordinal_loss, metrics=['accuracy'])\n\nmodel.fit_generator(\n    generator=train_flow,\n    epochs=MIDDLE_EPOCHS,\n    steps_per_epoch = train_flow.n // train_flow.batch_size,\n    verbose=1,\n#    validation_data=valid_flow,\n#    validation_steps = valid_flow.n // valid_flow.batch_size,\n    max_queue_size = QUEUE_SIZE,\n    workers = WORKERS\n#    callbacks = callbacks\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### Training on full model, 2019 data ####"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = ImageDataGenerator(preprocessing_function=process_image, horizontal_flip=True, vertical_flip=True, zoom_range=0.1, rotation_range=10, validation_split=TEST_SIZE)\ntrain_flow = train_generator.flow_from_dataframe(directory=train_directory_2019, dataframe=df_train_2019, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=False, batch_size=BATCH_SIZE, seed=SEED, subset=\"training\")\nvalid_flow = train_generator.flow_from_dataframe(directory=train_directory_2019, dataframe=df_train_2019, x_col=\"image\", y_col=\"diagnosis\", class_mode=\"categorical\", drop_duplicates=True, batch_size=BATCH_SIZE, seed=SEED, subset=\"validation\")\n\nlr = LearningRateScheduler(lr_scheduler)\ncallbacks = [lr]\n\nmodel.fit_generator(\n    generator=train_flow,\n    epochs=FINAL_EPOCHS,\n    steps_per_epoch = train_flow.n // train_flow.batch_size,\n    verbose=1,\n#    validation_data=valid_flow,\n#    validation_steps = valid_flow.n // valid_flow.batch_size,\n    max_queue_size = QUEUE_SIZE,\n    workers = WORKERS,\n    callbacks = callbacks\n    )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Final training on 2019 data with kappa score ####"},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa = Kappa(valid_flow)\ncallbacks = [kappa]\n\nmodel.fit_generator(\n    generator=train_flow,\n    epochs=FINAL_EPOCHS,\n    steps_per_epoch = train_flow.n // train_flow.batch_size,\n    verbose=1,\n#    validation_data=valid_flow,\n#    validation_steps = valid_flow.n // valid_flow.batch_size,\n    max_queue_size = QUEUE_SIZE,\n    workers = WORKERS,\n    callbacks = callbacks\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa = Kappa(valid_flow)\ncallbacks = [kappa]\n\nmodel.fit_generator(\n    generator=train_flow,\n    epochs=FINAL_EPOCHS * 40,\n    initial_epoch = FINAL_EPOCHS + 1,\n    steps_per_epoch = train_flow.n // train_flow.batch_size,\n    verbose=1,\n#    validation_data=valid_flow,\n#    validation_steps = valid_flow.n // valid_flow.batch_size,\n    max_queue_size = QUEUE_SIZE,\n    workers = WORKERS,\n    callbacks = callbacks\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download link #"},{"metadata":{"trusted":true},"cell_type":"code","source":"    from IPython.display import FileLink\n    FileLink(best_weight_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run prediction #"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator = ImageDataGenerator(preprocessing_function=preprocess_input)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(test_data_file)\ndf_test['filename'] = df_test['id_code'].apply(lambda i : \"{}.png\".format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_flow = test_generator.flow_from_dataframe(directory=test_image_directory, dataframe=df_test, x_col='filename', batch_size=BATCH_SIZE, max_queue_size=128, class_mode=None, target_size=(IMAGE_SIZE, IMAGE_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict_generator(\n                    generator = test_flow,\n                    steps = (test_flow.n // test_flow.batch_size) + 1,\n                    verbose=1,\n                    workers=WORKERS,\n                    max_queue_size=QUEUE_SIZE\n                )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['diagnosis'] = np.argmax(y_pred, axis=-1).astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display results ###"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.groupby('diagnosis').count()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}