{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/noisystudent/noisy_student_efficientnet-b3.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:00.782137Z","iopub.execute_input":"2021-10-09T06:57:00.78243Z","iopub.status.idle":"2021-10-09T06:57:02.177883Z","shell.execute_reply.started":"2021-10-09T06:57:00.7824Z","shell.execute_reply":"2021-10-09T06:57:02.177089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -xf /kaggle/working/noisy_student_efficientnet-b3.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:02.180285Z","iopub.execute_input":"2021-10-09T06:57:02.180648Z","iopub.status.idle":"2021-10-09T06:57:03.918144Z","shell.execute_reply.started":"2021-10-09T06:57:02.180611Z","shell.execute_reply":"2021-10-09T06:57:03.917177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/python/keras/applications/efficientnet_weight_update_util.py","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:03.919887Z","iopub.execute_input":"2021-10-09T06:57:03.920243Z","iopub.status.idle":"2021-10-09T06:57:04.889937Z","shell.execute_reply.started":"2021-10-09T06:57:03.920205Z","shell.execute_reply":"2021-10-09T06:57:04.889019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ./efficientnet_weight_update_util.py --model b3 --notop --ckpt ./noisy_student_efficientnet-b3/model.ckpt --o ./efficientnetb3_notop.h5","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:04.893671Z","iopub.execute_input":"2021-10-09T06:57:04.893968Z","iopub.status.idle":"2021-10-09T06:57:20.055322Z","shell.execute_reply.started":"2021-10-09T06:57:04.893937Z","shell.execute_reply":"2021-10-09T06:57:20.054494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nROOT_DIR = '../input/cassava-leaf-disease-classification/'\nos.listdir(ROOT_DIR)\n\nimport json # to read in the 'label_num_to_disease_map.json' file","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-09T06:57:20.059903Z","iopub.execute_input":"2021-10-09T06:57:20.060171Z","iopub.status.idle":"2021-10-09T06:57:20.06791Z","shell.execute_reply.started":"2021-10-09T06:57:20.060142Z","shell.execute_reply":"2021-10-09T06:57:20.067183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\nimport cv2\nimport random","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:20.0716Z","iopub.execute_input":"2021-10-09T06:57:20.071901Z","iopub.status.idle":"2021-10-09T06:57:22.97455Z","shell.execute_reply.started":"2021-10-09T06:57:20.071866Z","shell.execute_reply":"2021-10-09T06:57:22.973791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:22.976167Z","iopub.execute_input":"2021-10-09T06:57:22.976553Z","iopub.status.idle":"2021-10-09T06:57:22.982557Z","shell.execute_reply.started":"2021-10-09T06:57:22.976515Z","shell.execute_reply":"2021-10-09T06:57:22.981528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nfrom tensorflow.keras.optimizers import Adam\n\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom tensorflow.keras.preprocessing.image import img_to_array\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.applications import EfficientNetB3, EfficientNetB5\nfrom tensorflow.keras.utils import plot_model","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:22.984128Z","iopub.execute_input":"2021-10-09T06:57:22.984887Z","iopub.status.idle":"2021-10-09T06:57:24.4765Z","shell.execute_reply.started":"2021-10-09T06:57:22.984847Z","shell.execute_reply":"2021-10-09T06:57:24.475765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set the training and test directory paths\nTRAIN_DIR = '../input/cassava-leaf-disease-classification/train_images/'\nTEST_DIR = '../input/cassava-leaf-disease-classification/test_images/'","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-10-09T06:57:24.477841Z","iopub.execute_input":"2021-10-09T06:57:24.478238Z","iopub.status.idle":"2021-10-09T06:57:24.488383Z","shell.execute_reply.started":"2021-10-09T06:57:24.478194Z","shell.execute_reply":"2021-10-09T06:57:24.487547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set seed\nseed = 42\n\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.489761Z","iopub.execute_input":"2021-10-09T06:57:24.490138Z","iopub.status.idle":"2021-10-09T06:57:24.497369Z","shell.execute_reply.started":"2021-10-09T06:57:24.490098Z","shell.execute_reply":"2021-10-09T06:57:24.496461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center><h1> Data exploration </h1></center> ","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(ROOT_DIR + 'train.csv')\nsample_df = pd.read_csv(ROOT_DIR + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.499053Z","iopub.execute_input":"2021-10-09T06:57:24.499636Z","iopub.status.idle":"2021-10-09T06:57:24.540828Z","shell.execute_reply.started":"2021-10-09T06:57:24.499596Z","shell.execute_reply":"2021-10-09T06:57:24.540185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape, sample_df.shape)\ndisplay(train_df.head())","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.542473Z","iopub.execute_input":"2021-10-09T06:57:24.54272Z","iopub.status.idle":"2021-10-09T06:57:24.561764Z","shell.execute_reply.started":"2021-10-09T06:57:24.542686Z","shell.execute_reply":"2021-10-09T06:57:24.560883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(ROOT_DIR + 'label_num_to_disease_map.json')\ndata = json.load(f)\nprint(json.dumps(data, indent = 2))","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.563031Z","iopub.execute_input":"2021-10-09T06:57:24.563533Z","iopub.status.idle":"2021-10-09T06:57:24.575542Z","shell.execute_reply.started":"2021-10-09T06:57:24.563495Z","shell.execute_reply":"2021-10-09T06:57:24.574725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z = train_df.sample(20)\ndisplay(z)\nimages, labels = z['image_id'].tolist(), z['label'].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.57715Z","iopub.execute_input":"2021-10-09T06:57:24.577512Z","iopub.status.idle":"2021-10-09T06:57:24.592563Z","shell.execute_reply.started":"2021-10-09T06:57:24.577475Z","shell.execute_reply":"2021-10-09T06:57:24.591793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center><h1> Split dataset for training and validation </h1></center> \n<center> Reserving 15% of data for validation </center>","metadata":{}},{"cell_type":"code","source":"train_df = train_df.astype({\"label\": str})","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.594371Z","iopub.execute_input":"2021-10-09T06:57:24.594766Z","iopub.status.idle":"2021-10-09T06:57:24.626278Z","shell.execute_reply.started":"2021-10-09T06:57:24.594701Z","shell.execute_reply":"2021-10-09T06:57:24.62523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test = train_test_split(train_df, test_size = 0.15, random_state = seed)\nprint(train.shape, test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.628188Z","iopub.execute_input":"2021-10-09T06:57:24.628804Z","iopub.status.idle":"2021-10-09T06:57:24.638209Z","shell.execute_reply.started":"2021-10-09T06:57:24.628763Z","shell.execute_reply":"2021-10-09T06:57:24.637202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center><h1> Creating ImageDataGenerator to generate data in batches and perform image augmentation. </h1></center> ","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 333\nsize = (IMG_SIZE,IMG_SIZE)\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.639749Z","iopub.execute_input":"2021-10-09T06:57:24.64029Z","iopub.status.idle":"2021-10-09T06:57:24.644962Z","shell.execute_reply.started":"2021-10-09T06:57:24.640252Z","shell.execute_reply":"2021-10-09T06:57:24.643815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n                    rotation_range = 30,\n                    width_shift_range = 0.2,\n                    height_shift_range = 0.2,\n                    shear_range = 0.2,\n                    zoom_range = 0.2,\n                    brightness_range = [0.5,1.5],\n                    horizontal_flip = True,\n                    vertical_flip = True,\n                    fill_mode = 'nearest'\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.646369Z","iopub.execute_input":"2021-10-09T06:57:24.646966Z","iopub.status.idle":"2021-10-09T06:57:24.653989Z","shell.execute_reply.started":"2021-10-09T06:57:24.646926Z","shell.execute_reply":"2021-10-09T06:57:24.652979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validgen = ImageDataGenerator()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.655343Z","iopub.execute_input":"2021-10-09T06:57:24.655795Z","iopub.status.idle":"2021-10-09T06:57:24.662486Z","shell.execute_reply.started":"2021-10-09T06:57:24.655756Z","shell.execute_reply":"2021-10-09T06:57:24.661588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = datagen.flow_from_dataframe(\n                    train,\n                    directory = TRAIN_DIR,\n                    x_col = \"image_id\",\n                    y_col = \"label\",\n                    target_size = size,\n                    class_mode = \"sparse\",\n                    batch_size = batch_size,\n                    shuffle = True,\n                    seed = seed,\n                    interpolation = \"nearest\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:24.66459Z","iopub.execute_input":"2021-10-09T06:57:24.6651Z","iopub.status.idle":"2021-10-09T06:57:44.12154Z","shell.execute_reply.started":"2021-10-09T06:57:24.665063Z","shell.execute_reply":"2021-10-09T06:57:44.120637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_generator = validgen.flow_from_dataframe(\n                    test,\n                    directory = TRAIN_DIR,\n                    x_col = \"image_id\",\n                    y_col = \"label\",\n                    target_size = size,\n                    class_mode = \"sparse\",\n                    batch_size = batch_size,\n                    shuffle = False,\n                    seed = seed,\n                    interpolation = \"nearest\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:44.122843Z","iopub.execute_input":"2021-10-09T06:57:44.123377Z","iopub.status.idle":"2021-10-09T06:57:47.521663Z","shell.execute_reply.started":"2021-10-09T06:57:44.123333Z","shell.execute_reply":"2021-10-09T06:57:47.518753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center><h1> Model creation and training </h1></center> ","metadata":{}},{"cell_type":"code","source":"NUM_CLASSES = 5","metadata":{"execution":{"iopub.status.busy":"2021-10-09T06:57:47.528292Z","iopub.execute_input":"2021-10-09T06:57:47.528569Z","iopub.status.idle":"2021-10-09T06:57:47.534992Z","shell.execute_reply.started":"2021-10-09T06:57:47.52854Z","shell.execute_reply":"2021-10-09T06:57:47.534076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    \n    model = models.Sequential()\n    # initialize EfficientNetB3 model with input shape as (300,300,3)\n    model.add(EfficientNetB3(input_shape = (IMG_SIZE, IMG_SIZE, 3), include_top = False, weights = \"./efficientnetb3_notop.h5\"))\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dense(256, activation = 'relu'))\n    model.add(layers.Dropout(0.4))\n    model.add(layers.Dense(NUM_CLASSES, activation = 'softmax'))\n    model.summary()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-09T07:05:13.178595Z","iopub.execute_input":"2021-10-09T07:05:13.178969Z","iopub.status.idle":"2021-10-09T07:05:13.186535Z","shell.execute_reply.started":"2021-10-09T07:05:13.178937Z","shell.execute_reply":"2021-10-09T07:05:13.185683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-09T07:05:22.702789Z","iopub.execute_input":"2021-10-09T07:05:22.703116Z","iopub.status.idle":"2021-10-09T07:05:28.737172Z","shell.execute_reply.started":"2021-10-09T07:05:22.703087Z","shell.execute_reply":"2021-10-09T07:05:28.736424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_modelFreezed():\n    print(\"gg\")\n    model = models.Sequential()\n    pre_trained_model = EfficientNetB3(input_shape = (IMG_SIZE, IMG_SIZE, 3), include_top = False, weights = \"./efficientnetb3_notop.h5\",drop_connect_rate=0.2)\n    pre_trained_model.trainable = True\n     # We unfreeze the top 25 layers while leaving BatchNorm layers frozen\n    for layer in pre_trained_model.layers:\n        if isinstance(layer, layers.BatchNormalization):\n            layer.trainable = False\n            \n    model.add(pre_trained_model)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dense(256, activation = 'relu'))\n    model.add(layers.Dropout(0.6))\n    model.add(layers.Dense(NUM_CLASSES, activation = 'softmax'))\n    plot_model(model, show_shapes = True)\n    model.summary()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-09T07:09:55.86329Z","iopub.execute_input":"2021-10-09T07:09:55.863621Z","iopub.status.idle":"2021-10-09T07:09:55.872924Z","shell.execute_reply.started":"2021-10-09T07:09:55.863593Z","shell.execute_reply":"2021-10-09T07:09:55.871825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_modelFreezed()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-09T07:10:13.053315Z","iopub.execute_input":"2021-10-09T07:10:13.053633Z","iopub.status.idle":"2021-10-09T07:10:17.373606Z","shell.execute_reply.started":"2021-10-09T07:10:13.053603Z","shell.execute_reply":"2021-10-09T07:10:17.372717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss = 'sparse_categorical_crossentropy',\n             optimizer = Adam(learning_rate = 1e-4),\n             metrics = ['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stop training when the validation loss metric has stopped decreasing for 5 epochs.\nearly_stopping = EarlyStopping(monitor = 'val_loss',\n                               patience = 5,\n                               mode = 'min',\n                               restore_best_weights = True)\n\n# Save the model with the maximum validation accuracy \ncheckpoint = ModelCheckpoint('best_model.hdf5', \n                             monitor = 'val_accuracy',\n                             verbose = 1,\n                             mode = 'max', \n                             save_best_only = True)\n# reduce learning rate\nreduce_lr = ReduceLROnPlateau(monitor = 'val_loss',\n                              factor = 0.2,\n                              patience = 2,\n                              mode = 'min',\n                              verbose = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 30\nSTEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n                    validation_data = valid_generator,\n                    epochs = EPOCHS,\n                    steps_per_epoch = STEP_SIZE_TRAIN,\n                    validation_steps = STEP_SIZE_VALID,\n                    callbacks = [early_stopping, checkpoint, reduce_lr]\n                   )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model, show_shapes = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<center><h1> Model evaluation </h1></center> ","metadata":{}},{"cell_type":"code","source":"model.evaluate_generator(generator = valid_generator, steps = STEP_SIZE_VALID)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'c-', label='Training accuracy')\nplt.plot(epochs, val_acc, 'y-', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'c-', label='Training Loss')\nplt.plot(epochs, val_loss, 'y-', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Short version history:\n* Baseline model - 0.601\n* EfficientNetB0 - image_size = (224,224), batch_size =\t64\t- 0.841\n* EfficientNetB3 - image_size =\t(300,300), batch_size =\t32\t- 0.856\n* EfficientNetB3 with more augmentations - 0.865\n","metadata":{}},{"cell_type":"markdown","source":"| Base model\t  |resolution |\n|-----------------|-----------|\n| EfficientNetB0  |\t224 |\n| EfficientNetB1  |\t240 |\n| EfficientNetB2  |\t260 |\n| EfficientNetB3  |\t300 |\n| EfficientNetB4  |\t380 |\n| EfficientNetB5  |\t456 |\n| EfficientNetB6  |\t528 |\n| EfficientNetB7  |\t600 |","metadata":{}},{"cell_type":"markdown","source":"### Some useful links:\n* The prediction and submission notebook can be found here : [Inference Notebook](https://www.kaggle.com/lavanyask/cassava-leaf-disease-inference)\n* More about keras EfficientNets: [here](https://keras.io/examples/vision/image_classification_efficientnet_fine_tuning/)","metadata":{}},{"cell_type":"markdown","source":"### Further experiments:\n* Trying out different network architecture, changing number of layers in the network\n* Hyperparameter tuning - changing epochs, batch size, number of neurons in hiddden layers, activation function ...\n* Cross Validation\n* More augmentation techniques\n* PyTorch","metadata":{}},{"cell_type":"markdown","source":"## Do consider upvoting if you found it useful :)\n### Thank you for reading the notebook.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}