{"cells":[{"metadata":{},"cell_type":"markdown","source":"* This Notebook is Simple fine-tuning Model for begginners.\n* I'm a beginner at Machine Learning, so I'm sorry if there are any mistakes.\n* I referred to the following documents for fine-tuning.\nhttps://keras.io/examples/vision/image_classification_efficientnet_fine_tuning/\n\n"},{"metadata":{},"cell_type":"markdown","source":"### Import Module"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport glob\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing import image\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense, Input, Activation, add, Add, Dropout, BatchNormalization, GlobalAveragePooling2D\nfrom keras.models import Sequential, Model\nfrom keras import optimizers\nfrom kaggle_datasets import KaggleDatasets\n\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.callbacks import ModelCheckpoint\n\nfrom tensorflow.keras.applications import EfficientNetB4\nfrom tensorflow.keras.applications.efficientnet import preprocess_input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set up variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 256\nIMG_WIDTH = 380\nIMG_HEIGHT = 380\nCHANNELS = 3\nCLASSES = 5\nSEED = 5\n\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set Up Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n    preprocessing_function=preprocess_input,\n    rotation_range=90,\n    width_shift_range = 0.2,\n    height_shift_range = 0.2,\n    shear_range = 0.2,\n    zoom_range = 0.2,\n    fill_mode='nearest',\n    cval=0xCC,\n    horizontal_flip=True,\n    vertical_flip=True,\n    validation_split = 0.2\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['path'] = '../input/cassava-leaf-disease-classification/train_images/' + df_train['image_id']\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images = glob.glob('../input/cassava-leaf-disease-classification/test_images/*.jpg')\ndf_test = pd.DataFrame(test_images, columns = ['path'])\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# If class_mode=\"categorical\", y_col=\"label\" column values must be type string, list or tuple.\ndf_train['label'] = df_train['label'].astype('str') \n\ntrain_generator = datagen.flow_from_dataframe(\n    df_train,\n    x_col = 'path',\n    y_col = 'label',\n    target_size=(IMG_WIDTH,IMG_HEIGHT),\n    batch_size=batch_size,\n    class_mode='categorical',\n    shuffle=True,\n    seed = SEED,\n    subset = 'training'\n    )\n\nvalidation_generator = datagen.flow_from_dataframe(\n    df_train,\n    x_col = 'path',\n    y_col = 'label',\n    target_size=(IMG_WIDTH,IMG_HEIGHT),\n    batch_size=batch_size,\n    class_mode='categorical',\n    shuffle=True,\n    seed = SEED,\n    subset = 'validation'\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import layers\ndef build_model():\n    inputs = layers.Input(shape=(IMG_WIDTH,IMG_HEIGHT,CHANNELS))\n    x = inputs\n    model = EfficientNetB4(include_top=False, input_tensor=x, \n                           weights='../input/tfkerasefficientnetimagenetnotop/efficientnetb4_notop.h5')\n    \n    # Freeze the pretrained weights\n    model.trainable = False\n    \n    x = layers.GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n    x = layers.BatchNormalization()(x)\n    top_dropout_rate = 0.2\n    x = layers.Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n    outputs = layers.Dense(CLASSES, activation=\"softmax\", name=\"pred\")(x)\n    \n    model = tf.keras.Model(inputs, outputs, name=\"EfficientNet\")\n    \n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pre Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss = tf.keras.losses.CategoricalCrossentropy(from_logits = False,\n                                               label_smoothing=0.0001,\n                                               name='categorical_crossentropy' )  \n\nmodel.compile(loss = loss,\n              optimizer = optimizers.Adam(learning_rate=1e-2),\n              metrics=['accuracy'])\n \nSTEP_SIZE_TRAIN=train_generator.n // train_generator.batch_size\nSTEP_SIZE_VALID=validation_generator.n  // validation_generator.batch_size\n\nmodelCheckpoint = ModelCheckpoint(filepath = 'cassava-efficientNetB4_bestParam_pre.h5',\n                                  monitor='val_loss',\n                                  verbose=1,\n                                  save_best_only=True,\n                                  save_weights_only=True,\n                                  mode='min',\n                                  period=1)\n\n\nepochs = 10\n\nhistory = model.fit_generator(train_generator,\n                             steps_per_epoch=STEP_SIZE_TRAIN,\n                             epochs=epochs,\n                             validation_data=validation_generator,\n                             validation_steps=STEP_SIZE_VALID,\n                             callbacks=[modelCheckpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('fivethirtyeight')\nplt.title('Pre Model Accuracy')\nplt.plot(range(1, epochs+1), history.history['accuracy'], label=\"training\")\nplt.plot(range(1, epochs+1), history.history['val_accuracy'], label=\"validation\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('fivethirtyeight')\nplt.title('Pre Model Loss')\nplt.plot(range(1, epochs+1), history.history['loss'], label=\"training\")\nplt.plot(range(1, epochs+1), history.history['val_loss'], label=\"validation\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fine Tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Pretrain-weights\nmodel.load_weights('cassava-efficientNetB4_bestParam_pre.h5', by_name=True)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unfreeze_model(model):\n    model.trainable = True\n    \n    for layer in model.layers:\n        layer.trainable = False\n\n    # unfreeze the top 20 layers while leaving BatchNorm layers frozen\n    for layer in model.layers[-20:]:\n        if not isinstance(layer, BatchNormalization):\n            layer.trainable = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Unfreezing\nunfreeze_model(model)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fine Tuning\nmodel.compile(loss = loss,\n              optimizer = optimizers.Adam(learning_rate=1e-4),\n              metrics=['accuracy'])\n \nSTEP_SIZE_TRAIN=train_generator.n // train_generator.batch_size\nSTEP_SIZE_VALID=validation_generator.n  // validation_generator.batch_size\n\nmodelCheckpoint = ModelCheckpoint(filepath = 'cassava-efficientNetB4_bestParam.h5',\n                                  monitor='val_loss',\n                                  verbose=1,\n                                  save_best_only=True,\n                                  save_weights_only=False,\n                                  mode='min',\n                                  period=1)\n\nepochs = 10\n\nhistory = model.fit_generator(train_generator,\n                             steps_per_epoch=STEP_SIZE_TRAIN,\n                             epochs=epochs,\n                             validation_data=validation_generator,\n                             validation_steps=STEP_SIZE_VALID,\n                             callbacks=[modelCheckpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('fivethirtyeight')\nplt.title('Model Accuracy')\nplt.plot(range(1, epochs+1), history.history['accuracy'], label=\"training\")\nplt.plot(range(1, epochs+1), history.history['val_accuracy'], label=\"validation\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('fivethirtyeight')\nplt.title('Model Loss')\nplt.plot(range(1, epochs+1), history.history['loss'], label=\"training\")\nplt.plot(range(1, epochs+1), history.history['val_loss'], label=\"validation\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('cassava-efficientNetB4_bestParam.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_datagen = image.ImageDataGenerator(preprocessing_function=preprocess_input)\ntest_generator = test_datagen.flow_from_dataframe(\n    df_test,\n    x_col = 'path',\n    y_col=None,\n    target_size=(IMG_WIDTH,IMG_HEIGHT),\n    batch_size=batch_size,\n    class_mode=None,\n    shuffle=False,\n    seed = SEED\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict_generator(test_generator, verbose = True)\npred_labels = np.argmax(pred, axis = -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['path'].str.split('/').str[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = df_test\ndf_sub['image_id'] = df_test['path'].str.split('/').str[-1]\ndf_sub['label'] = pred_labels\ndf_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.to_csv('submission.csv', index=False, columns=['image_id', 'label'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}