{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Problem Statement\nPlant Pathology 2021 - FGVC8 is a [Kaggle competition](https://www.kaggle.com/c/plant-pathology-2021-fgvc8) launched on march 15 2021 and closed on mai 27 2021.\n\nExploration notebook can be find on [Kaggle](https://www.kaggle.com/xavierbarbier/plant-pathology-2021-fgvc8-eda) and the [full project on Github](https://github.com/xavierbarbier/Plant_Pathology_2021_FGVC8).\n\nThe goals of this notebook are:\n\n* Use a distributed approach (TPU) to optimise training time\n* Create a sample dataset for training\n* Compare differents pre-trained model\n* Optimise and tune the selected model\n* Train the optimised model on the full dataset","metadata":{}},{"cell_type":"code","source":"#-------------------\n# importing libraries\n#-------------------\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport PIL\nimport shutil\nimport csv\n\nimport matplotlib.image as img\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras import Model,layers","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:04:52.313049Z","iopub.execute_input":"2021-08-16T15:04:52.313417Z","iopub.status.idle":"2021-08-16T15:04:52.319947Z","shell.execute_reply.started":"2021-08-16T15:04:52.313388Z","shell.execute_reply":"2021-08-16T15:04:52.318824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow_addons\n\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:54:36.585207Z","iopub.execute_input":"2021-08-16T14:54:36.585512Z","iopub.status.idle":"2021-08-16T14:54:44.903877Z","shell.execute_reply.started":"2021-08-16T14:54:36.585483Z","shell.execute_reply":"2021-08-16T14:54:44.902945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the distributed strategy\nAUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:54:44.906043Z","iopub.execute_input":"2021-08-16T14:54:44.906337Z","iopub.status.idle":"2021-08-16T14:54:50.696249Z","shell.execute_reply.started":"2021-08-16T14:54:44.906306Z","shell.execute_reply":"2021-08-16T14:54:50.695397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()\nprint(GCS_DS_PATH)\n\nTRAIN_PATH = GCS_DS_PATH + \"/train_images/\"\n\ntrain = pd.read_csv(\"../input/plant-pathology-2021-fgvc8/train.csv\")\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:54:50.697653Z","iopub.execute_input":"2021-08-16T14:54:50.697924Z","iopub.status.idle":"2021-08-16T14:57:55.513944Z","shell.execute_reply.started":"2021-08-16T14:54:50.697898Z","shell.execute_reply":"2021-08-16T14:57:55.512940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a stratify sample","metadata":{}},{"cell_type":"markdown","source":"## Split","metadata":{}},{"cell_type":"code","source":"labels_counts = train[\"labels\"].value_counts()\n\nplt.barh(labels_counts.index,labels_counts)\nplt.title(\"Labels counts\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:55.515514Z","iopub.execute_input":"2021-08-16T14:57:55.515858Z","iopub.status.idle":"2021-08-16T14:57:55.766455Z","shell.execute_reply.started":"2021-08-16T14:57:55.515805Z","shell.execute_reply":"2021-08-16T14:57:55.765431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will use 10% of the full dataset avec use sklearn train_test_split to create our sample.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n\n# splitting on labels\nX_train, X_test, y_train, y_test = train_test_split(train['image'], train['labels'], test_size=0.1, random_state = 12,\n                                                      stratify =  train['labels'] )\n\n# using test set as training sample\ndata_sample = train.iloc[y_test.index]\n\nprint(\"sample shape\")\nprint(data_sample.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:55.767723Z","iopub.execute_input":"2021-08-16T14:57:55.768040Z","iopub.status.idle":"2021-08-16T14:57:56.462659Z","shell.execute_reply.started":"2021-08-16T14:57:55.768013Z","shell.execute_reply":"2021-08-16T14:57:56.461572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_counts = data_sample[\"labels\"].value_counts()\n\nplt.barh(labels_counts.index,labels_counts)\nplt.title(\"sample labels counts\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:56.463987Z","iopub.execute_input":"2021-08-16T14:57:56.464262Z","iopub.status.idle":"2021-08-16T14:57:56.894628Z","shell.execute_reply.started":"2021-08-16T14:57:56.464237Z","shell.execute_reply":"2021-08-16T14:57:56.893288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sample dataset seems to have same distribution as full dataset","metadata":{}},{"cell_type":"code","source":"data_sample[\"path\"] = TRAIN_PATH + data_sample[\"image\"]\n\nclass_dict = {\n    'scab': 0,\n    'frog_eye_leaf_spot' : 1,\n    'rust' : 2,\n    'complex' : 3,\n    'powdery_mildew' : 4,\n    \"healthy\" : 5\n}\nnum_classes = len(class_dict)    \nclass_names = dict([(value, key) for key, value in class_dict.items()])\ndata_sample[\"labels\"] = data_sample[\"labels\"].map(lambda x : [i for i in x.split(\" \")])\n#train_df[\"labels\"] = train_df[\"labels\"].map(lambda x : x.split(\" \"))\ndata_sample[\"labels\"] = data_sample[\"labels\"].map(lambda x : [class_dict[i] for i in x])\n\ndata_sample.head()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:56.896781Z","iopub.execute_input":"2021-08-16T14:57:56.897082Z","iopub.status.idle":"2021-08-16T14:57:56.936248Z","shell.execute_reply.started":"2021-08-16T14:57:56.897055Z","shell.execute_reply":"2021-08-16T14:57:56.935204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data prep","metadata":{}},{"cell_type":"code","source":"#--------------\n#initialize constants\n#--------------\nHEIGHT,WIDTH = 299,299\nCHANNELS = 3\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nSEED = 143\nSPLIT = int(0.8*len(data_sample))\nAUTO = tf.data.experimental.AUTOTUNE\n\ndef process_img(filepath,label):\n    image = tf.io.read_file(filepath)\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = tf.image.convert_image_dtype(image, tf.float32) \n    \n    image = tf.image.resize(image, [HEIGHT,WIDTH])\n    return image,label","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:56.937612Z","iopub.execute_input":"2021-08-16T14:57:56.937899Z","iopub.status.idle":"2021-08-16T14:57:56.944652Z","shell.execute_reply.started":"2021-08-16T14:57:56.937863Z","shell.execute_reply":"2021-08-16T14:57:56.943701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Spliting sample data to train and valid set\nX_train, X_test, y_train, y_test = train_test_split(data_sample[\"path\"], data_sample['labels'],\n                                                    test_size=0.33, random_state = 12,\n                                                      stratify =  data_sample['labels'] )\n\ntrain_ds = pd.concat([X_train, y_train], axis = 1)\nvalid_ds = pd.concat([X_test, y_test], axis = 1)\n\nfiles_ls = list(train_ds[\"path\"])\nlabels = np.zeros((len(train_ds),num_classes))\n\nfor i,file in enumerate(train_ds.values):\n    labels[i][train_ds.iloc[i][\"labels\"]] = 1\n    \ntrain_ds = tf.data.Dataset.from_tensor_slices((files_ls,labels))\ntrain_ds = train_ds.map(process_img,num_parallel_calls=AUTO)\n\n\nfiles_ls = list(valid_ds[\"path\"])\nlabels = np.zeros((len(valid_ds),num_classes))\n\nfor i,file in enumerate(valid_ds.values):\n    labels[i][valid_ds.iloc[i][\"labels\"]] = 1\n    \nval_ds = tf.data.Dataset.from_tensor_slices((files_ls,labels))\nval_ds = val_ds.map(process_img,num_parallel_calls=AUTO)\n\nprint(\"Nb obs train set:\",len(train_ds))\nprint(\"Nb obs valid set:\",len(val_ds))","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:56.945922Z","iopub.execute_input":"2021-08-16T14:57:56.946363Z","iopub.status.idle":"2021-08-16T14:57:57.254114Z","shell.execute_reply.started":"2021-08-16T14:57:56.946317Z","shell.execute_reply":"2021-08-16T14:57:57.252992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#--------------\n#initialize constants\n#--------------\n\nSTEPS_PER_EPOCH  = (len(train_ds))//BATCH_SIZE\nVALID_STEPS = (len(val_ds))//BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:57.255526Z","iopub.execute_input":"2021-08-16T14:57:57.255833Z","iopub.status.idle":"2021-08-16T14:57:57.270220Z","shell.execute_reply.started":"2021-08-16T14:57:57.255792Z","shell.execute_reply":"2021-08-16T14:57:57.269330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = train_ds.cache().repeat().shuffle(2048).batch(BATCH_SIZE).prefetch(AUTO)\nval_ds = val_ds.cache().repeat().batch(BATCH_SIZE).prefetch(AUTO)\nprint(\"Data Pipeline achieved !\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T14:57:57.271456Z","iopub.execute_input":"2021-08-16T14:57:57.272274Z","iopub.status.idle":"2021-08-16T14:57:57.288093Z","shell.execute_reply.started":"2021-08-16T14:57:57.272229Z","shell.execute_reply":"2021-08-16T14:57:57.286963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre trained models","metadata":{}},{"cell_type":"code","source":"# Define epochs for each training and scoring metric\nEPOCHS = 5\n\nmetrics = tfa.metrics.F1Score(num_classes = num_classes,average = \"macro\",name = \"f1_score\",\n                             threshold= 0.5)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:01:29.171727Z","iopub.execute_input":"2021-08-16T15:01:29.172337Z","iopub.status.idle":"2021-08-16T15:01:29.193192Z","shell.execute_reply.started":"2021-08-16T15:01:29.172291Z","shell.execute_reply":"2021-08-16T15:01:29.192350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compile_model(model, lr=1e-3):\n    \n    optimizer = tf.keras.optimizers.Adam(lr=lr)\n    \n    loss = tf.keras.losses.BinaryCrossentropy()\n        \n    metrics = tfa.metrics.F1Score(num_classes = num_classes,\n                                            average = \"macro\",name = \"f1_score\") \n\n    model.compile(optimizer=optimizer, loss=loss, metrics=[metrics])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:01:27.169694Z","iopub.execute_input":"2021-08-16T15:01:27.170291Z","iopub.status.idle":"2021-08-16T15:01:27.175825Z","shell.execute_reply.started":"2021-08-16T15:01:27.170245Z","shell.execute_reply":"2021-08-16T15:01:27.175071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### INCEPTION V3","metadata":{}},{"cell_type":"code","source":"\n\ndef create_model():\n    pre_trained_model = InceptionV3(input_shape = (HEIGHT,WIDTH, CHANNELS), \n                                  include_top = False, \n                                  weights = \"imagenet\")\n\n  # Setting pretrained model to no trainable\n    pre_trained_model.trainable = False\n\n    last_layer = pre_trained_model.get_layer('mixed7')\n\n    last_output = last_layer.output\n\n    x = layers.GlobalMaxPooling2D()(last_output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Flatten()(x)    \n    x = layers.Dense(num_classes, activation='sigmoid',dtype='float32')(x)           \n\n    model = Model( pre_trained_model.input, x )\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-01T10:24:50.093309Z","iopub.execute_input":"2021-06-01T10:24:50.093689Z","iopub.status.idle":"2021-06-01T10:24:50.104206Z","shell.execute_reply.started":"2021-06-01T10:24:50.093659Z","shell.execute_reply":"2021-06-01T10:24:50.103322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nVERBOSE =1\n\ntf.keras.backend.clear_session()\n\nwith strategy.scope():\n    \n    model = create_model()\n    model = compile_model(model, lr=1e-3)    \n    \n    history = model.fit(\n                        train_ds,\n                        epochs=EPOCHS,\n                        \n                        validation_data = val_ds,\n                        verbose=VERBOSE,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        validation_steps=VALID_STEPS\n                                           )","metadata":{"execution":{"iopub.status.busy":"2021-06-01T10:24:50.105242Z","iopub.execute_input":"2021-06-01T10:24:50.105677Z","iopub.status.idle":"2021-06-01T10:26:59.027966Z","shell.execute_reply.started":"2021-06-01T10:24:50.105648Z","shell.execute_reply":"2021-06-01T10:26:59.02693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting accuracy and val loss\nacc = history.history['f1_score']\nval_acc = history.history['val_f1_score']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(len(history.history['loss']))\n\nplt.figure(figsize=(14, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training f1_score')\nplt.plot(epochs_range, val_acc, label='Validation f1_score')\nplt.legend(loc='lower right')\nplt.title('Training and Validation f1_score')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T10:26:59.029059Z","iopub.execute_input":"2021-06-01T10:26:59.029297Z","iopub.status.idle":"2021-06-01T10:26:59.33526Z","shell.execute_reply.started":"2021-06-01T10:26:59.029274Z","shell.execute_reply":"2021-06-01T10:26:59.334371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('./my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-01T10:27:02.640874Z","iopub.execute_input":"2021-06-01T10:27:02.641399Z","iopub.status.idle":"2021-06-01T10:27:04.261902Z","shell.execute_reply.started":"2021-06-01T10:27:02.641357Z","shell.execute_reply":"2021-06-01T10:27:04.260979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MOBILENET","metadata":{}},{"cell_type":"code","source":"def create_model():\n    pre_trained_model = tf.keras.applications.MobileNetV2(input_shape = (HEIGHT,WIDTH, CHANNELS), \n                                include_top = False, \n                                weights = \"imagenet\")\n\n    # Setting pretrained model to no trainable\n    pre_trained_model.trainable = False\n\n    last_layer = pre_trained_model.get_layer('out_relu')\n    \n    last_output = last_layer.output\n\n    x = layers.GlobalMaxPooling2D()(last_output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Flatten()(x)    \n    x = layers.Dense(num_classes, activation='sigmoid',dtype='float32')(x)           \n\n    model = Model( pre_trained_model.input, x )\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nwith strategy.scope():\n    \n    model = create_model()\n    model = compile_model(model, lr=1e-3)\n   \n    \n    \n    history = model.fit(\n                        train_ds,\n                        epochs=EPOCHS,\n                        \n                        validation_data = val_ds,\n                        verbose=VERBOSE,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        validation_steps=VALID_STEPS\n                       )\n                       ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting accuracy and val loss\nacc = history.history['f1_score']\nval_acc = history.history['val_f1_score']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(len(history.history['loss']))\n\nplt.figure(figsize=(14, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training f1_score')\nplt.plot(epochs_range, val_acc, label='Validation f1_score')\nplt.legend(loc='lower right')\nplt.title('Training and Validation f1_score')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RESNET 50","metadata":{}},{"cell_type":"code","source":"def create_model():\n    pre_trained_model = tf.keras.applications.ResNet50(input_shape = (HEIGHT,WIDTH, CHANNELS), \n                                include_top = False, \n                                weights = \"imagenet\")\n\n    # Setting pretrained model to no trainable\n    pre_trained_model.trainable = False\n\n    last_layer = pre_trained_model.get_layer('conv5_block3_out')\n    \n    last_output = last_layer.output\n\n    x = layers.GlobalMaxPooling2D()(last_output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Flatten()(x)    \n    x = layers.Dense(num_classes, activation='sigmoid',dtype='float32')(x)           \n\n    model = Model( pre_trained_model.input, x )\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EPOCHS = 10\nVERBOSE =1\n\ntf.keras.backend.clear_session()\n\nwith strategy.scope():\n    \n    model = create_model()\n    model = compile_model(model, lr=1e-3)\n   \n    \n    \n    history = model.fit(\n                        train_ds,\n                        epochs=EPOCHS,\n                        \n                        validation_data = val_ds,\n                        verbose=VERBOSE,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        validation_steps=VALID_STEPS\n                       )\n                       ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting accuracy and val loss\nacc = history.history['f1_score']\nval_acc = history.history['val_f1_score']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(len(history.history['loss']))\n\nplt.figure(figsize=(14, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training f1_score')\nplt.plot(epochs_range, val_acc, label='Validation f1_score')\nplt.legend(loc='lower right')\nplt.title('Training and Validation f1_score')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-trained models conclusion\n\nMobileNet model seems to have better results","metadata":{}},{"cell_type":"markdown","source":"## MOBILNET Tuner\n\nWe will use Keras tuner Hyperband to tune :\n* A dropout layer (as there is clear overfitting)\n* Learning rate\n","metadata":{}},{"cell_type":"code","source":"def model_builder(hp):\n    pre_trained_model = tf.keras.applications.MobileNetV2(input_shape = (HEIGHT,WIDTH, CHANNELS), \n                                include_top = False, \n                                weights = \"imagenet\")\n\n    # Setting pretrained model to no trainable\n    pre_trained_model.trainable = False\n\n    last_layer = pre_trained_model.get_layer('out_relu')\n    \n    last_output = last_layer.output\n\n    x = layers.GlobalMaxPooling2D()(last_output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Flatten()(x)    \n    # Tune a drop out layer\n    # Choose an optimal value from 0.0 to 0.5\n    x = layers.Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.2))(x)\n    x = layers.Dense(num_classes, activation='sigmoid',dtype='float32')(x)           \n\n    model = Model( pre_trained_model.input, x )\n\n    # Tune the learning rate for the optimizer\n    # Choose an optimal value from 0.01, 0.001, or 0.0001\n    hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4, 1e-5])\n\n    loss = tf.keras.losses.BinaryCrossentropy()\n\n    metrics = tfa.metrics.F1Score(num_classes = num_classes,\n                                    average = \"macro\",name = \"f1_score\") \n\n\n    model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),\n            loss=loss,\n            metrics=[metrics])\n\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:02:32.866568Z","iopub.execute_input":"2021-08-16T15:02:32.867541Z","iopub.status.idle":"2021-08-16T15:02:32.879785Z","shell.execute_reply.started":"2021-08-16T15:02:32.867443Z","shell.execute_reply":"2021-08-16T15:02:32.878519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Keras hyperband tuner","metadata":{}},{"cell_type":"code","source":"import kerastuner as kt\n\ntf.keras.backend.clear_session()\n\ntuner = kt.Hyperband(\n    model_builder,\n    objective= kt.Objective(\"val_f1_score\", direction=\"max\"),\n    max_epochs=10,\n    distribution_strategy=strategy    )","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:05:00.398237Z","iopub.execute_input":"2021-08-16T15:05:00.398615Z","iopub.status.idle":"2021-08-16T15:05:08.167259Z","shell.execute_reply.started":"2021-08-16T15:05:00.398582Z","shell.execute_reply":"2021-08-16T15:05:08.166115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Search space summary\")\ntuner.search_space_summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:06:22.796591Z","iopub.execute_input":"2021-08-16T15:06:22.796974Z","iopub.status.idle":"2021-08-16T15:06:22.804223Z","shell.execute_reply.started":"2021-08-16T15:06:22.796943Z","shell.execute_reply":"2021-08-16T15:06:22.802501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.search(train_ds, epochs=5,steps_per_epoch = STEPS_PER_EPOCH,validation_steps = STEPS_PER_EPOCH,\n             validation_data = val_ds)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:06:30.168316Z","iopub.execute_input":"2021-08-16T15:06:30.168676Z","iopub.status.idle":"2021-08-16T15:26:35.808062Z","shell.execute_reply.started":"2021-08-16T15:06:30.168647Z","shell.execute_reply":"2021-08-16T15:26:35.807031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Search results summary\")\ntuner.results_summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:41:20.343290Z","iopub.execute_input":"2021-08-16T15:41:20.343697Z","iopub.status.idle":"2021-08-16T15:41:20.369079Z","shell.execute_reply.started":"2021-08-16T15:41:20.343664Z","shell.execute_reply":"2021-08-16T15:41:20.359955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]\ndropout = best_hps.get('dropout')\nlr = best_hps.get('learning_rate') ","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:07:59.511243Z","iopub.execute_input":"2021-08-16T16:07:59.511634Z","iopub.status.idle":"2021-08-16T16:07:59.515941Z","shell.execute_reply.started":"2021-08-16T16:07:59.511595Z","shell.execute_reply":"2021-08-16T16:07:59.515185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:07:55.306781Z","iopub.execute_input":"2021-08-16T16:07:55.307210Z","iopub.status.idle":"2021-08-16T16:07:55.312945Z","shell.execute_reply.started":"2021-08-16T16:07:55.307176Z","shell.execute_reply":"2021-08-16T16:07:55.311925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Full data\n\nWe will now train the tuned model on the full dataset","metadata":{}},{"cell_type":"markdown","source":"## Data prep","metadata":{}},{"cell_type":"code","source":"class_dict = {\n    'scab': 0,\n    'frog_eye_leaf_spot' : 1,\n    'rust' : 2,\n    'complex' : 3,\n    'powdery_mildew' : 4,\n    \"healthy\" : 5\n}\nnum_classes = len(class_dict)    \nclass_names = dict([(value, key) for key, value in class_dict.items()])\ntrain[\"labels\"] = train[\"labels\"].map(lambda x : [i for i in x.split(\" \")])\n#train_df[\"labels\"] = train_df[\"labels\"].map(lambda x : x.split(\" \"))\ntrain[\"labels\"] = train[\"labels\"].map(lambda x : [class_dict[i] for i in x])\n\ntrain[\"path\"] = TRAIN_PATH + train[\"image\"]\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:41:55.564998Z","iopub.execute_input":"2021-08-16T15:41:55.565347Z","iopub.status.idle":"2021-08-16T15:41:55.628378Z","shell.execute_reply.started":"2021-08-16T15:41:55.565316Z","shell.execute_reply":"2021-08-16T15:41:55.627344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting for a train and valid set\n\nX_train, X_test, y_train, y_test = train_test_split(train[\"path\"], train['labels'], test_size=0.2, random_state = 12,\n                                                      stratify =  train['labels'] )","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:41:58.356537Z","iopub.execute_input":"2021-08-16T15:41:58.356911Z","iopub.status.idle":"2021-08-16T15:41:58.408952Z","shell.execute_reply.started":"2021-08-16T15:41:58.356881Z","shell.execute_reply":"2021-08-16T15:41:58.407865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = pd.concat([X_train, y_train], axis = 1)\nvalid_ds = pd.concat([X_test, y_test], axis = 1)\n\nfiles_ls = list(train_ds[\"path\"])\nlabels = np.zeros((len(train_ds),num_classes))\n\nfor i,file in enumerate(train_ds.values):\n    labels[i][train_ds.iloc[i][\"labels\"]] = 1\n    \ntrain_ds = tf.data.Dataset.from_tensor_slices((files_ls,labels))\ntrain_ds = train_ds.map(process_img,num_parallel_calls=AUTO)\n\n\nfiles_ls = list(valid_ds[\"path\"])\nlabels = np.zeros((len(valid_ds),num_classes))\n\nfor i,file in enumerate(valid_ds.values):\n    labels[i][valid_ds.iloc[i][\"labels\"]] = 1\n    \nval_ds = tf.data.Dataset.from_tensor_slices((files_ls,labels))\nval_ds = val_ds.map(process_img,num_parallel_calls=AUTO)\n\nSTEPS_PER_EPOCH  = (len(train_ds))//BATCH_SIZE\nVALID_STEPS = (len(val_ds))//BATCH_SIZE\n\ntrain_ds = train_ds.cache().repeat().shuffle(2048).batch(BATCH_SIZE).prefetch(AUTO)\nval_ds = val_ds.cache().repeat().batch(BATCH_SIZE).prefetch(AUTO)\nprint(\"Data Pipeline achieved !\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:42:00.331720Z","iopub.execute_input":"2021-08-16T15:42:00.332114Z","iopub.status.idle":"2021-08-16T15:42:02.217130Z","shell.execute_reply.started":"2021-08-16T15:42:00.332080Z","shell.execute_reply":"2021-08-16T15:42:02.216016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Callbacks","metadata":{}},{"cell_type":"code","source":"# Create a callback that saves the model's weights\n\ncheckpoint_dir = \"./raw_model.h5\"\ncp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_dir,\n                                                 save_weights_only=True,\n                                                 save_best_only=True,\n                                                 verbose=1,\n                                                 monitor= \"val_f1_score\",\n        mode='max')\n\n# Create a callback that stops fitting when val loss do not decrease\ncallback = tf.keras.callbacks.EarlyStopping(monitor=\"val_f1_score\", patience=10, mode='max')\n\nreducelr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor= \"val_f1_score\",\n        mode='max',\n        factor=0.1,\n        patience=2,\n        verbose=1\n    )\n\ncallbacks=[callback,cp_callback,reducelr]","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:42:07.782630Z","iopub.execute_input":"2021-08-16T15:42:07.783017Z","iopub.status.idle":"2021-08-16T15:42:07.793357Z","shell.execute_reply.started":"2021-08-16T15:42:07.782986Z","shell.execute_reply":"2021-08-16T15:42:07.791824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef create_model():\n    pre_trained_model = tf.keras.applications.MobileNetV2(input_shape = (HEIGHT,WIDTH, CHANNELS), \n                                include_top = False, \n                                weights = \"imagenet\")\n\n  # Setting pretrained model to no trainable\n    pre_trained_model.trainable = False\n\n    last_layer = pre_trained_model.get_layer('out_relu')\n\n    last_output = last_layer.output\n\n    x = layers.GlobalMaxPooling2D()(last_output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Flatten()(x)   \n    x = layers.Dropout(dropout)(x)\n    x = layers.Dense(num_classes, activation='sigmoid',dtype='float32')(x)           \n\n    model = Model( pre_trained_model.input, x )\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:42:52.151711Z","iopub.execute_input":"2021-08-16T15:42:52.152118Z","iopub.status.idle":"2021-08-16T15:42:52.159927Z","shell.execute_reply.started":"2021-08-16T15:42:52.152078Z","shell.execute_reply":"2021-08-16T15:42:52.158692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training ","metadata":{}},{"cell_type":"code","source":"EPOCHS = 100\nVERBOSE =1\n\ntf.keras.backend.clear_session()\n\nwith strategy.scope():\n    \n    model = create_model()\n    model = compile_model(model, lr=lr) \n\n    history = model.fit(train_ds,\n            epochs=EPOCHS,\n            validation_data = val_ds,\n            verbose=VERBOSE,\n            steps_per_epoch = STEPS_PER_EPOCH,\n            validation_steps=STEPS_PER_EPOCH,\n            callbacks = callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T15:42:55.544698Z","iopub.execute_input":"2021-08-16T15:42:55.545102Z","iopub.status.idle":"2021-08-16T16:06:43.995895Z","shell.execute_reply.started":"2021-08-16T15:42:55.545066Z","shell.execute_reply":"2021-08-16T16:06:43.994668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting accuracy and val loss\nacc = history.history['f1_score']\nval_acc = history.history['val_f1_score']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(len(history.history['loss']))\n\nplt.figure(figsize=(14, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training f1_score')\nplt.plot(epochs_range, val_acc, label='Validation f1_score')\nplt.legend(loc='lower right')\nplt.title('Training and Validation f1_score')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:07:07.625264Z","iopub.execute_input":"2021-08-16T16:07:07.625633Z","iopub.status.idle":"2021-08-16T16:07:07.989697Z","shell.execute_reply.started":"2021-08-16T16:07:07.625603Z","shell.execute_reply":"2021-08-16T16:07:07.988962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights(checkpoint_dir)\n\nmodel.save('./my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:08:17.194240Z","iopub.execute_input":"2021-08-16T16:08:17.194739Z","iopub.status.idle":"2021-08-16T16:08:20.660544Z","shell.execute_reply.started":"2021-08-16T16:08:17.194709Z","shell.execute_reply":"2021-08-16T16:08:20.659037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predictions","metadata":{}},{"cell_type":"markdown","source":"We now want to make some prediction on a small sample of images.","metadata":{}},{"cell_type":"code","source":"# loading the model\nnew_model = tf.keras.models.load_model('./my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:08:44.812607Z","iopub.execute_input":"2021-08-16T16:08:44.813181Z","iopub.status.idle":"2021-08-16T16:08:46.602892Z","shell.execute_reply.started":"2021-08-16T16:08:44.813147Z","shell.execute_reply":"2021-08-16T16:08:46.601971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking 9 images as sample\n\ntrain = pd.read_csv(\"../input/plant-pathology-2021-fgvc8/train.csv\")\n\nfiles_ls = tf.io.gfile.glob(TRAIN_PATH + '*.jpg')\n\nfrom random import sample\n\nfiles_ls_sample = sample(files_ls,9)\n\ntrain[\"path\"] = TRAIN_PATH  +  train[\"image\"]\n\ntest_df = train[train[\"path\"].isin(files_ls_sample)]\n\nprint(\"Sample shape\")\ntest_df.shape\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:09:54.165750Z","iopub.execute_input":"2021-08-16T16:09:54.166313Z","iopub.status.idle":"2021-08-16T16:09:57.482950Z","shell.execute_reply.started":"2021-08-16T16:09:54.166282Z","shell.execute_reply":"2021-08-16T16:09:57.481776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(9)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:09:57.485014Z","iopub.execute_input":"2021-08-16T16:09:57.485487Z","iopub.status.idle":"2021-08-16T16:09:57.501491Z","shell.execute_reply.started":"2021-08-16T16:09:57.485440Z","shell.execute_reply":"2021-08-16T16:09:57.500346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocessing without labels\n\ndef process_img_test(filepath):\n    image = tf.io.read_file(filepath)\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = tf.image.convert_image_dtype(image, tf.float32) \n    image = tf.image.resize(image, [HEIGHT,WIDTH])\n    return image\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(files_ls_sample)\n    .map(process_img_test\n).batch(BATCH_SIZE)\n)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:09:57.503918Z","iopub.execute_input":"2021-08-16T16:09:57.504349Z","iopub.status.idle":"2021-08-16T16:09:57.531635Z","shell.execute_reply.started":"2021-08-16T16:09:57.504305Z","shell.execute_reply":"2021-08-16T16:09:57.530242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# making predictions\npredicts = new_model.predict(test_dataset) \n\nthreshold = 0.5\n\ndef get_labels(prediction):\n  pred = []\n  idx = np.where(prediction>threshold)[0]\n  for i in idx:\n    pred.append(class_names[i])\n  pred = ' '. join(pred)\n  if len(pred) == 0:\n    pred = []\n    idx = np.argmax(prediction)\n    pred.append(class_names[idx])\n    pred = ' '. join(pred)\n    return pred\n  else :\n    return pred\n\nlabels = []\nfor i in range(len(predicts)):\n  pred = predicts[i]\n\n  labels.append(get_labels(pred))\n    \ntest_df[\"pred\"] = labels","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:09:57.533547Z","iopub.execute_input":"2021-08-16T16:09:57.534006Z","iopub.status.idle":"2021-08-16T16:09:59.044463Z","shell.execute_reply.started":"2021-08-16T16:09:57.533969Z","shell.execute_reply":"2021-08-16T16:09:59.043114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Showing some predictions","metadata":{}},{"cell_type":"code","source":"test_df[\"path\"] = \"../input/plant-pathology-2021-fgvc8/train_images/\" + test_df[\"image\"]\n\n# Showing image sample\nplt.figure(figsize=(14,9))\nn=1\nfor i in test_df.index :\n    plt.subplot(3,3,n)\n    \n    testImage = img.imread(test_df[\"path\"][i])\n\n    # displaying the image\n    plt.imshow(testImage)\n    color = \"blue\" if test_df[\"pred\"][i] == test_df[\"labels\"][i] else \"red\"\n    \n    plt.title(test_df[\"pred\"][i].title(), color=color)\n    plt.axis(\"off\")\n    n+=1\n_ = plt.suptitle(\"Model predictions on sample set (blue: correct, red: incorrect)\")","metadata":{"execution":{"iopub.status.busy":"2021-08-16T16:09:59.046153Z","iopub.execute_input":"2021-08-16T16:09:59.046610Z","iopub.status.idle":"2021-08-16T16:10:09.761295Z","shell.execute_reply.started":"2021-08-16T16:09:59.046549Z","shell.execute_reply":"2021-08-16T16:10:09.760029Z"},"trusted":true},"execution_count":null,"outputs":[]}]}