{"cells":[{"metadata":{},"cell_type":"markdown","source":"imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport os\nfrom PIL import Image\nfrom glob import glob\nfrom zipfile import ZipFile\nimport pandas as pd\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\nfrom keras.models import Model, Input, load_model\nfrom keras.layers import Dense, GlobalAveragePooling2D\nfrom keras.applications.mobilenetv2 import MobileNetV2\nfrom keras import backend as K\nfrom matplotlib import pyplot as plt\nfrom matplotlib.image import imread\nimport cv2\nimport numpy as np\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading a dataset into the ImageDataGenerator"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def process_csv(dataframe: pd.DataFrame, image_column_name: str,\n                label_column_name: str,\n                folder_with_images: str) -> pd.DataFrame:\n    \"\"\"This function process Pandas DataFrame, which contains image filenames\n    and their corresponding labels.\n\n    Args:\n        dataframe: Pandas DataFrame object. It should consist of 2 columns\n        image_column_name: The name of the column containing the image\n            filenames\n        label_column_name: The name of the column containing the image\n            labels\n        folder_with_images: Folder with images\n\n    Returns:\n        dataframe: processed DataFrame with full paths to images\n    \"\"\"\n    dataframe[image_column_name] = dataframe[image_column_name].apply(\n        lambda x: f\"{folder_with_images}{x}.png\")\n    dataframe[label_column_name] = dataframe[label_column_name].astype('str')\n    return dataframe","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Creating training and validation generators"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1. / 255,\n                                   rotation_range=15,\n                                   width_shift_range=0.1,\n                                   height_shift_range=0.1,\n                                   shear_range=0.01,\n                                   zoom_range=[0.9, 1.25],\n                                   horizontal_flip=True,\n                                   vertical_flip=True,\n                                   fill_mode='reflect',\n                                   data_format='channels_last',\n                                   brightness_range=[0.5, 1.5],\n                                   validation_split=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv(\"/kaggle/input/aptos2019-blindness-detection/train.csv\")\ntrain_csv = process_csv(\n    dataframe=train_csv,\n    image_column_name=\"id_code\",\n    label_column_name=\"diagnosis\",\n    folder_with_images=\"/kaggle/input/aptos2019-blindness-detection/train_images/\")\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe=train_csv, x_col=\"id_code\", y_col=\"diagnosis\", subset=\"training\",\n    batch_size=16, target_size=(224, 224))\nval_generator = train_datagen.flow_from_dataframe(\n    dataframe=train_csv, x_col=\"id_code\", y_col=\"diagnosis\",\n    subset=\"validation\", batch_size=16, target_size=(224, 224))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Work with the neural network model\n\nCreating a neural network mode"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, Input\nfrom keras.layers import Dense, GlobalAveragePooling2D\nfrom keras.applications.mobilenetv2 import MobileNetV2\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.layers import Average, Input\nfrom keras.applications.nasnet import NASNetLarge\nfrom keras.preprocessing import image\n\n\ndef create_model():\n    input_tensor = Input((224, 224, 3))\n    outputs = []\n    \n    mobilenet_model = MobileNetV2(weights=None, input_shape=(224, 224, 3),include_top=False, alpha=1.4)                          \n    mobilenet_model.load_weights(\"/kaggle/input/mobilenet-v2-keras-weights/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.4_224_no_top.h5\")\n    \n    #VGG16_model = VGG16(weights=None, input_shape = (224,224,3), include_top=False)\n    #VGG16_model.load_weights(\"/kaggle/input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n    \n    #InceptionV3_model = InceptionV3(weights=None, input_shape=(224,224,3),include_top = False)\n    #InceptionV3_model.load_weights(\"/kaggle/input/keras-pretrained-models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n    \n    NasNet_model = NASNetLarge(weights=None, input_shape=(224,224,3),include_top = False)\n    #NasNet_model.load_weights(\"/kaggle/input/pre-trained-models-for-keras/nasnet-large-no-top.h5\")\n    #NasNet_model = NASNetMobile(input_shape=(224,224,3), include_top=False, weights='imagenet', input_tensor=None, pooling=None)\n    \n    pretrained_models = [\n        mobilenet_model,NasNet_model\n    ]\n    for model in pretrained_models:\n        curr_output = model(input_tensor)\n        curr_output = GlobalAveragePooling2D()(curr_output)\n        curr_output = Dense(1024, activation=\"relu\")(curr_output)\n        outputs.append(curr_output)\n    output_tensor = Average()(outputs)\n    output_tensor = Dense(5, activation=\"softmax\")(output_tensor)\n\n    model = Model(input_tensor, output_tensor)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LRFinder(Callback):\n\n    def __init__(self, start_learning_rate, multiplier):\n        self.multiplier = multiplier\n        self.start_learning_rate = start_learning_rate\n        self.curr_train_step_number = 1\n        self.learning_rate = start_learning_rate\n        self.all_learning_rates = []\n        self.all_loss_values = []\n        super().__init__()\n\n    def on_batch_end(self, batch, logs=None):\n        self.learning_rate *= self.multiplier\n        K.set_value(self.model.optimizer.lr, self.learning_rate)\n        self.curr_train_step_number += 1\n        self.all_learning_rates.append(self.learning_rate)\n        self.all_loss_values.append(logs.get('loss'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_lr = 1e-10\nend_lr = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrfinder_generator = ImageDataGenerator(rescale=1./255, validation_split=0).flow_from_dataframe(\n    dataframe=train_csv, x_col=\"id_code\", y_col=\"diagnosis\", subset=\"training\",\n    batch_size=8, target_size=(224, 224))\nlrfinder_callback = LRFinder(1e-10, multiplier=(end_lr / start_lr) ** (1 / len(lrfinder_generator))) \nlrfinder_model = create_model()\nlrfinder_model.compile(optimizer=Adam(start_lr),\n                       loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrfinder_model.fit_generator(generator=lrfinder_generator,\n                             steps_per_epoch=len(lrfinder_generator),\n                             epochs=1,\n                             callbacks=[lrfinder_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def exponentionaly_weighted_average(data: list, beta: float) -> list:\n    processed_data = [data[0]]\n    for value in data[1:]:\n        processed_data.append(beta * value + (1 - beta) * processed_data[-1])\n    return processed_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"beta_values = [0.9, 0.5, 0.3, 0.1, 0.05]\nlosses = lrfinder_callback.all_loss_values\nlearning_rates = lrfinder_callback.all_learning_rates","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(1, figsize=(10, 5))\nplt.xscale('log')\nplt.xticks([10**(-i) for i in range(11)])\nplt.plot(learning_rates, losses)\nplt.xlabel(\"Learning rate (log scale, without EWMA)\")\nplt.ylabel(\"Loss\")\nplt.show()\n\nfor idx, beta_value in enumerate(beta_values):\n    plt.figure(idx + 2, figsize=(10, 5))\n    plt.xscale('log')\n    plt.xticks([10**(-i) for i in range(11)])\n    plt.plot(learning_rates, exponentionaly_weighted_average(losses, beta_value))\n    plt.xlabel(f\"Learning rate (log scale, with EWMA, beta={beta_value})\")\n    plt.ylabel(\"Loss\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    ModelCheckpoint(\n        \"best_weights.h5\",\n        monitor='val_acc',\n        verbose=1, save_best_only=True),\n    EarlyStopping(monitor='val_acc', patience=5)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model()\nmodel.compile(optimizer=Adam(1e-4),\n              loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\nmodel.fit_generator(generator=train_generator,\n                    steps_per_epoch=len(train_generator),\n                    validation_data=val_generator,\n                    validation_steps=len(val_generator),\n                    epochs=100,\n                    callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model(\"best_weights.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv = pd.read_csv(\"/kaggle/input/aptos2019-blindness-detection/test.csv\")\npredicted_csv = pd.DataFrame(columns=[\"id_code\", \"diagnosis\"])\n\nfor id_code in test_csv[\"id_code\"]:\n    filename = f\"/kaggle/input/aptos2019-blindness-detection/test_images/{id_code}.png\"\n    img = imread(filename)\n    img = cv2.resize(img, dsize=(224, 224), interpolation=cv2.INTER_CUBIC)\n    img = np.expand_dims(img, 0)\n    prediction = int(np.argmax(model.predict(img)[0]))\n    predicted_csv = predicted_csv.append(\n        {'id_code':id_code ,\"diagnosis\": prediction}, ignore_index=True)\n\nwith open(\"submission.csv\", \"w\") as f:\n    f.write(predicted_csv.to_csv(index=False))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}