{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Init\nImport needed libraries and set pseudo-random generators at a static seed.\nAlso define some testing variables, like batch-size, learning-rate and epochs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nfrom PIL import Image\nimport numpy as np\nfrom keras import layers\nfrom keras.applications import ResNet152, InceptionV3\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nimport tensorflow as tf\nfrom tqdm import tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16\nLEARNING_RATE = 5e-05\nEPOCHS = 20","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load and preprocess data\nRead the data into a pandas dataframe. Resize the images to 224 x 224. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_image(image_path, desired_size=224):\n    im = Image.open(image_path)\n    im = im.resize((desired_size, )*2, resample=Image.LANCZOS)\n    \n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = train_df.shape[0]\nx_train = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = pd.get_dummies(train_df['diagnosis']).values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Multilabels and splitting data\nA multilabel problems scores higher on the kappa score, which is used by this competiton. Thanks to https://www.kaggle.com/lextoumbourou/blindness-detection-resnet34-ordinal-targets.\nFurthermore the data is split in a 85/15 train/val split."},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(\n    x_train, y_train_multi, \n    test_size=0.15, \n    random_state=SEED\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing training\nSome functions which need to be defined before training, the datagen builder, the kappa score callback and the model builder, downloading the ResNet and Inception backbones."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_datagen():\n    return ImageDataGenerator(\n        zoom_range=0.15,\n        fill_mode='constant',\n        cval=0.,\n        horizontal_flip=True,\n        vertical_flip=True,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        X_val, y_val = self.validation_data[:2]\n        y_val = y_val.sum(axis=1) - 1\n        \n        y_pred = self.model.predict(X_val) > 0.5\n        y_pred = y_pred.astype(int).sum(axis=1) - 1\n\n        _val_kappa = cohen_kappa_score(\n            y_val,\n            y_pred, \n            weights='quadratic'\n        )\n\n        self.val_kappas.append(_val_kappa)\n\n        print(f\"val_kappa: {_val_kappa:.4f}\")\n        \n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save('model.h5')\n\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet_backbone = ResNet152(\n    weights='imagenet',\n    include_top=False,\n    input_shape=(224,224,3)\n)\n\ninception_backbone = InceptionV3(\n    weights='imagenet',\n    include_top=False,\n    input_shape=(224,224,3)\n)\n\nkappa_metrics = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(backbone=inception_backbone, lr=0.00005):\n    model = Sequential()\n    model.add(backbone)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Dense(5, activation='sigmoid'))\n    \n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=Adam(lr=lr),\n        metrics=['accuracy']\n    )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, epochs):\n    history = model.fit_generator(\n        data_generator,\n        steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n        epochs=epochs,\n        validation_data=(x_val, y_val),\n        callbacks=[kappa_metrics]\n    )\n\n    return history\n\ndef print_output(history):\n    history_df = pd.DataFrame(history.history)\n    history_df[['loss', 'val_loss']].plot()\n    history_df[['accuracy', 'val_accuracy']].plot()\n\n#    plt.plot(kappa_metrics.val_kappas)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training the models\nTrain both the ResNet and Inception model. Print their training process and print the accuracy and loss plots."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('creating datagenerator')\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=SEED)\nprint('datagenerator done, building model')\nresnet_model = build_model(backbone=resnet_backbone, lr=LEARNING_RATE)\nprint('model done, training model')\nprint(f'ResNet152, {BATCH_SIZE}, {LEARNING_RATE}, sigmoid')\nhistory_resnet = train_model(resnet_model, EPOCHS)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_output(history_resnet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('creating datagenerator')\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=SEED)\nprint('datagenerator done, building model')\ninception_model = build_model(backbone=inception_backbone, lr=LEARNING_RATE)\nprint('model done, training model')\nprint(f'InceptionV3, {BATCH_SIZE}, {LEARNING_RATE}, sigmoid')\n\nhistory_inception = train_model(inception_model, EPOCHS)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_output(history_inception)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# McNemar's Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_contingency(c1, c2, x, y):\n    y2 = y.sum(axis=1) - 1\n\n    c1_pred = c1.predict(x) > 0.5\n    c1_pred = c1_pred.astype(int).sum(axis=1) - 1\n\n    c2_pred = c2.predict(x) > 0.5\n    c2_pred = c2_pred.astype(int).sum(axis=1) - 1\n    \n    c1c2_correct = 0\n    c1_correct = 0\n    c2_correct = 0\n    none_correct = 0\n\n    for i, sample in enumerate(x):\n        if ((y2[i] == c1_pred[i]) and (y2[i] == c2_pred[i])):\n            c1c2_correct += 1\n        elif ((y2[i] == c1_pred[i]) and (y2[i] != c2_pred[i])):\n            c1_correct += 1\n        elif ((y2[i] != c1_pred[i]) and (y2[i] == c2_pred[i])):\n            c2_correct += 1\n        elif ((y2[i] != c1_pred[i]) and (y2[i] != c2_pred[i])):\n            none_correct += 1\n    \n    table = [[c1c2_correct, c1_correct],\n\t\t [c2_correct, none_correct]]\n    total = c1c2_correct + c1_correct + c2_correct + none_correct\n    \n    \n    print(f'c1c2_correct: {c1c2_correct}, c1_correct: {c1_correct}, c2_correct: {c2_correct}, none_correct: {none_correct}, total: {total}')\n    return table","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"table = generate_contingency(resnet_model, inception_model, x_val, y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Example of calculating the mcnemar test\nfrom statsmodels.stats.contingency_tables import mcnemar\n# calculate mcnemar test\nresult = mcnemar(table, exact=False, correction=True)\n# summarize the finding\nprint('statistic=%.3f, p-value=%.3f' % (result.statistic, result.pvalue))\n# interpret the p-value\nalpha = 0.05\nif result.pvalue > alpha:\n\tprint('Same proportions of errors (fail to reject H0)')\nelse:\n\tprint('Different proportions of errors (reject H0)')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}