{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport shutil\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.utils import to_categorical\nfrom keras import optimizers, applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, Callback, LearningRateScheduler\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\n\n# Set seeds to make the experiment more reproducible.\nfrom tensorflow import set_random_seed\ndef seed_everything(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    set_random_seed(0)\nseed = 0\nseed_everything(seed)\n\n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"_kg_hide-output":true}},{"cell_type":"code","source":"hold_out_set = pd.read_csv('../input/aptos-data-split/hold-out.csv')\nX_train = hold_out_set[hold_out_set['set'] == 'train']\nX_val = hold_out_set[hold_out_set['set'] == 'validation']\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint('Number of train samples: ', X_train.shape[0])\nprint('Number of validation samples: ', X_val.shape[0])\nprint('Number of test samples: ', test.shape[0])\n\n# Preprocecss data\nX_train[\"id_code\"] = X_train[\"id_code\"].apply(lambda x: x + \".png\")\nX_val[\"id_code\"] = X_val[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\nX_train['diagnosis'] = X_train['diagnosis']\nX_val['diagnosis'] = X_val['diagnosis']\ndisplay(X_train.head())","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{}},{"cell_type":"code","source":"# Model parameters\nBATCH_SIZE = 32\nEPOCHS = 40\nWARMUP_EPOCHS = 5\nLEARNING_RATE = 1e-4\nWARMUP_LEARNING_RATE = 1e-3\nHEIGHT = 320\nWIDTH = 320\nCHANNELS = 3\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-procecess images","metadata":{}},{"cell_type":"code","source":"train_base_path = '../input/aptos2019-blindness-detection/train_images/'\ntest_base_path = '../input/aptos2019-blindness-detection/test_images/'\ntrain_dest_path = 'base_dir/train_images/'\nvalidation_dest_path = 'base_dir/validation_images/'\ntest_dest_path =  'base_dir/test_images/'\n\n# Making sure directories don't exist\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)\n    \n# Creating train, validation and test directories\nos.makedirs(train_dest_path)\nos.makedirs(validation_dest_path)\nos.makedirs(test_dest_path)\n\ndef crop_image(img, tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n            \n        return img\n\ndef circle_crop(img):\n    img = crop_image(img)\n\n    height, width, depth = img.shape\n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n\n    x = width//2\n    y = height//2\n    r = np.amin((x, y))\n\n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x, y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image(img)\n\n    return img\n    \ndef preprocess_image(base_path, save_path, image_id, HEIGHT, WIDTH, sigmaX=10):\n    image = cv2.imread(base_path + image_id)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = circle_crop(image)\n    image = cv2.resize(image, (HEIGHT, WIDTH))\n    image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), sigmaX), -4 , 128)\n    cv2.imwrite(save_path + image_id, image)\n    \n# Pre-procecss train set\nfor i, image_id in enumerate(X_train['id_code']):\n    preprocess_image(train_base_path, train_dest_path, image_id, HEIGHT, WIDTH)\n    \n# Pre-procecss validation set\nfor i, image_id in enumerate(X_val['id_code']):\n    preprocess_image(train_base_path, validation_dest_path, image_id, HEIGHT, WIDTH)\n    \n# Pre-procecss test set\nfor i, image_id in enumerate(test['id_code']):\n    preprocess_image(test_base_path, test_dest_path, image_id, HEIGHT, WIDTH)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data generator","metadata":{}},{"cell_type":"code","source":"datagen=ImageDataGenerator(rescale=1./255, \n                           rotation_range=360,\n                           horizontal_flip=True,\n                           vertical_flip=True)\n\ntrain_generator=datagen.flow_from_dataframe(\n                          dataframe=X_train,\n                          directory=train_dest_path,\n                          x_col=\"id_code\",\n                          y_col=\"diagnosis\",\n                          class_mode=\"raw\",\n                          batch_size=BATCH_SIZE,\n                          target_size=(HEIGHT, WIDTH),\n                          seed=seed)\n\nvalid_generator=datagen.flow_from_dataframe(\n                        dataframe=X_val,\n                        directory=validation_dest_path,\n                        x_col=\"id_code\",\n                        y_col=\"diagnosis\",\n                        class_mode=\"raw\",\n                        batch_size=BATCH_SIZE,\n                        target_size=(HEIGHT, WIDTH),\n                        seed=seed)\n\ntest_generator=datagen.flow_from_dataframe(  \n                          dataframe=test,\n                          directory=test_dest_path,\n                          x_col=\"id_code\",\n                          batch_size=1,\n                          class_mode=None,\n                          shuffle=False,\n                          target_size=(HEIGHT, WIDTH),\n                          seed=seed)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model(input_shape):\n    input_tensor = Input(shape=input_shape)\n    base_model = applications.InceptionResNetV2(weights=None, \n                                       include_top=False,\n                                       input_tensor=input_tensor)\n    base_model.load_weights('../input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(2048, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(1, activation='linear', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train top layers","metadata":{}},{"cell_type":"code","source":"model = create_model(input_shape=(HEIGHT, WIDTH, CHANNELS))\n\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-5, 0):\n    model.layers[i].trainable = True\n    \nmetric_list = [\"accuracy\"]\noptimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='mean_squared_error', metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size\n\nhistory_warmup = model.fit_generator(generator=train_generator,\n                                     steps_per_epoch=STEP_SIZE_TRAIN,\n                                     validation_data=valid_generator,\n                                     validation_steps=STEP_SIZE_VALID,\n                                     epochs=WARMUP_EPOCHS,\n                                     verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune the complete model (1st step)","metadata":{}},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)\n\ncallback_list = [es, rlrop]\noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='mean_squared_error', metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-input":false,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_finetunning = model.fit_generator(generator=train_generator,\n                                          steps_per_epoch=STEP_SIZE_TRAIN,\n                                          validation_data=valid_generator,\n                                          validation_steps=STEP_SIZE_VALID,\n                                          epochs=int(EPOCHS*0.8),\n                                          callbacks=callback_list,\n                                          verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune the complete model (2nd step)","metadata":{}},{"cell_type":"code","source":"optimizer = optimizers.SGD(lr=LEARNING_RATE, momentum=0.9, nesterov=True)\nmodel.compile(optimizer=optimizer, loss='mean_squared_error', metrics=metric_list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_finetunning_2 = model.fit_generator(generator=train_generator,\n                                          steps_per_epoch=STEP_SIZE_TRAIN,\n                                          validation_data=valid_generator,\n                                          validation_steps=STEP_SIZE_VALID,\n                                          epochs=int(EPOCHS*0.2),\n                                          callbacks=callback_list,\n                                          verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model loss graph ","metadata":{}},{"cell_type":"code","source":"history = {'loss': history_finetunning['loss'] + history_finetunning_2['loss'], \n           'val_loss': history_finetunning['val_loss'] + history_finetunning_2['val_loss'], \n           'acc': history_finetunning['acc'] + history_finetunning_2['acc'], \n           'val_acc': history_finetunning['val_acc'] + history_finetunning_2['val_acc']}\n\nsns.set_style(\"whitegrid\")\nfig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))\n\nax1.plot(history['loss'], label='Train loss')\nax1.plot(history['val_loss'], label='Validation loss')\nax1.legend(loc='best')\nax1.set_title('Loss')\n\nax2.plot(history['acc'], label='Train accuracy')\nax2.plot(history['val_acc'], label='Validation accuracy')\nax2.legend(loc='best')\nax2.set_title('Accuracy')\n\nplt.xlabel('Epochs')\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create empty arays to keep the predictions and labels\ndf_preds = pd.DataFrame(columns=['label', 'pred', 'set'])\ntrain_generator.reset()\nvalid_generator.reset()\n\n# Add train predictions and labels\nfor i in range(STEP_SIZE_TRAIN + 1):\n    im, lbl = next(train_generator)\n    preds = model.predict(im, batch_size=train_generator.batch_size)\n    for index in range(len(preds)):\n        df_preds.loc[len(df_preds)] = [lbl[index], preds[index][0], 'train']\n\n# Add validation predictions and labels\nfor i in range(STEP_SIZE_VALID + 1):\n    im, lbl = next(valid_generator)\n    preds = model.predict(im, batch_size=valid_generator.batch_size)\n    for index in range(len(preds)):\n        df_preds.loc[len(df_preds)] = [lbl[index], preds[index][0], 'validation']\n\ndf_preds['label'] = df_preds['label'].astype('int')","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Threshold optimization","metadata":{}},{"cell_type":"code","source":"def classify(x):\n    if x < 0.5:\n        return 0\n    elif x < 1.5:\n        return 1\n    elif x < 2.5:\n        return 2\n    elif x < 3.5:\n        return 3\n    return 4\n\ndef classify_opt(x):\n    if x <= (0 + best_thr_0):\n        return 0\n    elif x <= (1 + best_thr_1):\n        return 1\n    elif x <= (2 + best_thr_2):\n        return 2\n    elif x <= (3 + best_thr_3):\n        return 3\n    return 4\n\n\ndef find_best_threshold(df, label, label_col='label', pred_col='pred', do_plot=True):\n    score = []\n    thrs = np.arange(0, 1, 0.01)\n    for thr in thrs:\n        preds_thr = [label if ((pred >= label and pred < label+1) and (pred < (label+thr))) else classify(pred) for pred in df[pred_col]]\n        score.append(cohen_kappa_score(df[label_col].astype('int'), preds_thr))\n\n    score = np.array(score)\n    pm = score.argmax()\n    best_thr, best_score = thrs[pm], score[pm].item()\n    print('Label %s: thr=%.2f, Kappa=%.3f' % (label, best_thr, best_score))\n    plt.rcParams[\"figure.figsize\"] = (20, 5)\n    plt.plot(thrs, score)\n    plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n    plt.show()\n    \n    return best_thr\n\n# Best threshold for label 3\nbest_thr_3 = find_best_threshold(df_preds, 3)\n\n# Best threshold for label 2\nbest_thr_2 = find_best_threshold(df_preds, 2)\n\n# Best threshold for label 1\nbest_thr_1 = find_best_threshold(df_preds, 1)\n\n# Best threshold for label 0\nbest_thr_0 = find_best_threshold(df_preds, 0)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classify predictions\ndf_preds['predictions'] = df_preds['pred'].apply(lambda x: classify(x))\n# Apply optimized thresholds to the predictions\ndf_preds['predictions_opt'] = df_preds['pred'].apply(lambda x: classify_opt(x))\n\ntrain_preds = df_preds[df_preds['set'] == 'train']\nvalidation_preds = df_preds[df_preds['set'] == 'validation']","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation","metadata":{}},{"cell_type":"markdown","source":"## Confusion Matrix\n\n### Original thresholds","metadata":{}},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ndef plot_confusion_matrix(train, validation, labels=labels):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 7))\n    train_cnf_matrix = confusion_matrix(train_labels, train_preds)\n    validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)\n\n    train_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\n    validation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\n    train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\n    validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\n    sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\",ax=ax1).set_title('Train')\n    sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2).set_title('Validation')\n    plt.show()\n\nplot_confusion_matrix((train_preds['label'], train_preds['predictions']), (validation_preds['label'], validation_preds['predictions']))","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Quadratic Weighted Kappa","metadata":{}},{"cell_type":"code","source":"def evaluate_model(train, validation):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    print(\"Train        Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds, train_labels, weights='quadratic'))\n    print(\"Validation   Cohen Kappa score: %.3f\" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))\n    print(\"Complete set Cohen Kappa score: %.3f\" % cohen_kappa_score(np.append(train_preds, validation_preds), np.append(train_labels, validation_labels), weights='quadratic'))\n    \nprint(\"   Original thresholds\")\nevaluate_model((train_preds['label'], train_preds['predictions']), (validation_preds['label'], validation_preds['predictions']))\nprint(\"   Optimized thresholds\")\nevaluate_model((train_preds['label'], train_preds['predictions_opt']), (validation_preds['label'], validation_preds['predictions_opt']))","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Apply model to test set and output predictions","metadata":{}},{"cell_type":"code","source":"def apply_tta(model, generator, steps=5):\n    step_size = generator.n//generator.batch_size\n    preds_tta = []\n    for i in range(steps):\n        generator.reset()\n        preds = model.predict_generator(generator, steps=step_size)\n        preds_tta.append(preds)\n\n    return np.mean(preds_tta, axis=0)\n\npreds = apply_tta(model, test_generator)\npredictions = [classify(x) for x in preds]\npredictions_opt = [classify_opt(x) for x in preds]\n\nresults = pd.DataFrame({'id_code':test['id_code'], 'diagnosis':predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])\n\nresults_opt = pd.DataFrame({'id_code':test['id_code'], 'diagnosis':predictions_opt})\nresults_opt['id_code'] = results_opt['id_code'].map(lambda x: str(x)[:-4])","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning created directories\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions class distribution","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 8.7))\nsns.countplot(x=\"diagnosis\", data=results, palette=\"GnBu_d\", ax=ax1).set_title('Test')\nsns.countplot(x=\"diagnosis\", data=results_opt, palette=\"GnBu_d\", ax=ax2).set_title('Test optimized')\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_kappa =  cohen_kappa_score(validation_preds['label'], validation_preds['predictions'], weights='quadratic')\nval_opt_kappa = cohen_kappa_score(validation_preds['label'], validation_preds['predictions_opt'], weights='quadratic')\nresults_name = 'submission.csv'\nresults_opt_name = 'submission_opt.csv'\n\n# if val_kappa > val_opt_kappa:\n#     results_name = 'submission.csv'\n#     results_opt_name = 'submission_opt.csv'\n# else:\n#     results_name = 'submission_norm.csv'\n#     results_opt_name = 'submission.csv'","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv(results_name, index=False)\ndisplay(results.head())\n\nresults_opt.to_csv(results_opt_name, index=False)\ndisplay(results_opt.head())","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]}]}