{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Hello Kaggler!"},{"metadata":{},"cell_type":"markdown","source":"In this notebook, we will try to use **GPU** (Graphics Processing Unit) provided by **kaggle** to train a Convolutional Neural Network on the dataset provided by **Plant Pathology 2021** competition."},{"metadata":{},"cell_type":"markdown","source":"Check out the other notebook for TPU [TensorFlow-Custom-Distributed-Training-TPU](https://www.kaggle.com/mohammadasimbluemoon/tensorflow-custom-distributed-training-tpu)"},{"metadata":{},"cell_type":"markdown","source":"1. **To start with first we import all the required libraries**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nfrom matplotlib import image\nfrom matplotlib import pyplot\nimport os\nimport cv2\nimport random\nimport concurrent.futures\nimport time\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom keras import backend as K\nfrom kaggle_secrets import UserSecretsClient\nfrom kaggle_datasets import KaggleDatasets\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. **In the following code, the training list of names and labels are shown. We can see there are 12 different kinds of combinations. This is a multi-label problem, because we can see that the unique labels are the first 6 labels, and the rest are their combinations.**\n\n\n    label_names=['healthy', 'scab', 'frog_eye_leaf_spot', 'powdery_mildew', 'rust', 'complex']"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/plant-pathology-2021-fgvc8/train.csv\", dtype=str)\nprint(train['labels'].value_counts())\nprint(train['labels'].value_counts().plot.bar())\nprint(train['labels'].count())\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nlabel_names=['scab', 'frog_eye_leaf_spot', 'powdery_mildew', 'rust', 'complex']\nnames=[]\nlabels = []\nfor i in range(len(train)):\n    name = train['image'][i]\n    label = train['labels'][i]\n    splits = label.split()\n    vec = np.zeros(len(label_names))\n    for split in splits:\n        if(split!='healthy'):\n            vec[label_names.index(split)] = 1\n    labels.append(vec)\n    names.append(name)\n\nVAL_SPLIT = 0.3\ntrain_names, val_names, train_labels, val_labels = train_test_split(names, labels, \\\n                                                   test_size=VAL_SPLIT, random_state=42,\\\n                                                   stratify=labels)\ntrain_names = list(train_names)\nval_names = list(val_names)\nprint(\"Length of training set: \", len(train_names))\nprint(\"Length of validation set: \", len(val_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"strategy = tf.distribute.MirroredStrategy()\n\nTRAIN_BATCH_SIZE = 128\nTRAIN_SHUFFLE_BUFFER = 8912\nVAL_BATCH_SIZE = 32\nVAL_SHUFFLE_BUFFER = 3584\nGCS_PATH = '../input/plant-pathology-2021-fgvc8'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 1000\n\nrandom_rotation = tf.keras.layers.experimental.preprocessing.RandomRotation(3.142/2, seed=SEED)\nrandom_flip = tf.keras.layers.experimental.preprocessing.RandomFlip(mode=\"horizontal_and_vertical\", seed=SEED)\nrandom_zoom = tf.keras.layers.experimental.preprocessing.RandomZoom((-0.05, 0.25), seed=SEED)\nrandom_translate = tf.keras.layers.experimental.preprocessing.RandomTranslation((-0.1, 0.1), (-0.1, -0.1), seed=SEED)\n\nIMSIZE = 256\n\ndef _parse(imgs, label):\n    image_string = tf.io.read_file(GCS_PATH + '/train_images/' + imgs)\n    image_decoded = tf.image.decode_jpeg(image_string)\n    imgs = tf.image.resize(image_decoded, [IMSIZE, IMSIZE])\n    imgs = tf.cast(imgs, tf.uint8)\n    return imgs, label\ndef _normalize(imgs,label):\n    return imgs/255, label\n\n\n\ntrain_dataset = tf.data.Dataset.from_tensor_slices((tf.constant(train_names), tf.constant(train_labels)))\\\n                               .map(_parse, num_parallel_calls=tf.data.AUTOTUNE)\\\n                               .cache()\\\n                               .shuffle(TRAIN_SHUFFLE_BUFFER)\\\n                               .batch(TRAIN_BATCH_SIZE, drop_remainder=True)\\\n                               .map(_normalize, num_parallel_calls=tf.data.AUTOTUNE)\\\n                               .prefetch(tf.data.AUTOTUNE)\n\nval_dataset = tf.data.Dataset.from_tensor_slices((tf.constant(val_names), tf.constant(val_labels)))\\\n                             .map(_parse, num_parallel_calls=tf.data.AUTOTUNE)\\\n                             .cache()\\\n                             .shuffle(VAL_SHUFFLE_BUFFER)\\\n                             .batch(VAL_BATCH_SIZE, drop_remainder=True)\\\n                             .map(_normalize, num_parallel_calls=tf.data.AUTOTUNE)\\\n                             .prefetch(tf.data.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# def IResNet_brain_module(inp, layercount):\n#     n = layercount\n#     ############################################################################\n#     # Parallel Block 1\n#     x1_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x1_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x1_1)\n#     x1 = tf.keras.layers.add([x1_1, x1_2])\n#     ############################################################################\n#     # Parallel Block 2\n#     x2_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x2_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x2_1)\n#     x2_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x2_1)\n#     x2 = tf.keras.layers.add([x2_1, x2_2, x1])\n#     ############################################################################\n#     # Parallel Block 3\n#     x3_1 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x3_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x3 = tf.keras.layers.add([x3_1, x3_2, x2, x1])\n#     ############################################################################\n#     # Parallel Block 4\n#     x4_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x4_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x4_1)\n#     x4_1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x4_1)\n#     x4_2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x4_1)\n#     x4 = tf.keras.layers.add([x4_1, x4_2, x3, x2, x1])\n#     ############################################################################\n#     mod = tf.keras.layers.concatenate([x1, x2, x3, x4], axis = -1)\n#     mod = tf.keras.layers.BatchNormalization()(mod)\n#     return mod\n# def IResNet_connection_module(inp, layercount):\n#     n = layercount\n#     ############################################################################\n#     # Parallel Block 1\n#     x1 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     ############################################################################\n#     # Parallel Block 2\n#     x2 = tf.keras.layers.SeparableConv2D(n, (1, 1), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     ############################################################################\n#     # Parallel Block 3\n#     x3 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     x3 = tf.keras.layers.SeparableConv2D(n, (3, 3), activation = 'relu', padding = 'same', kernel_regularizer='l2', bias_regularizer='l1')(x3)\n#     ############################################################################\n#     mod = tf.keras.layers.concatenate([x1, x2, x3], axis = -1)\n#     mod = tf.keras.layers.BatchNormalization()(mod)\n#     return mod\n# def IResNet_reduction_module(inp, layercount):\n#     n = layercount\n#     ############################################################################\n#     # Reduction module\n#     R1= tf.keras.layers.Conv2D(n, (3, 3), strides = 2, activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(inp)\n#     R1 = tf.keras.layers.Conv2D(n, (1, 1), activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(R1)\n#     R1 = tf.keras.layers.MaxPool2D(pool_size=(3, 3))(R1)\n#     R1 = tf.keras.layers.Conv2D(n, (3, 3), strides = 2, activation = 'relu', kernel_regularizer='l2', bias_regularizer='l1')(R1)\n#     mod = tf.keras.layers.BatchNormalization()(R1)\n#     return mod\n# def IResNet_classifier_module(input, num_classes, activation):\n#     ############################################################################\n#     # Classifier module\n#     R1= tf.keras.layers.Flatten()(input)\n#     mod = tf.keras.layers.Dense(num_classes, activation=activation)(R1)\n#     return mod","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# IMSIZE = 256\n# CHANNEL = 3\n# with strategy.scope():\n#     inp = tf.keras.layers.Input(shape=(IMSIZE, IMSIZE, CHANNEL))\n#     img_inputs_1 = tf.keras.layers.experimental.preprocessing.RandomRotation(3.142/2, seed=SEED)(inp)\n#     img_inputs_2 = tf.keras.layers.experimental.preprocessing.RandomFlip(mode=\"horizontal_and_vertical\", seed=SEED)(img_inputs_1)\n#     img_inputs_3 = tf.keras.layers.experimental.preprocessing.RandomZoom((0, 0.35), seed=SEED)(img_inputs_2)\n#     module = IResNet_brain_module(img_inputs_3, 32) \n#     connect = IResNet_connection_module(module, 32)\n#     red = IResNet_reduction_module(connect, 64)\n#     red = IResNet_reduction_module(red, 64)\n#     red = IResNet_reduction_module(module, 128)\n#     red = IResNet_reduction_module(red, 128)\n#     connect = IResNet_connection_module(module, 256)\n#     cla = IResNet_classifier_module(connect, 6, 'sigmoid')\n#     model = tf.keras.models.Model(inputs=inp, outputs=cla, name = \"IResNetv1\")\n#     model.summary()\n#     tf.keras.utils.plot_model(model, show_shapes=True,to_file='./img.png')\n#     metric_auc = tf.keras.metrics.AUC(num_thresholds=200, name='auc')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nwith strategy.scope():\n    base_model = tf.keras.applications.ResNet50V2(include_top=False,weights='imagenet', pooling='max')\n    model = tf.keras.Sequential([\n        tf.keras.layers.Input((IMSIZE,IMSIZE, 3)),\n        random_rotation,\n        random_flip,\n        random_zoom,\n        random_translate,\n        base_model,\n        tf.keras.layers.Dense(5, activation='sigmoid')\n    ])\n    optimizer = tf.keras.optimizers.SGD(0.001)\n    epoch_auc = tf.keras.metrics.AUC(num_thresholds=200, multi_label=True)\n    val_epoch_auc = tf.keras.metrics.AUC(num_thresholds=200, multi_label=True)\n    loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\n    val_loss = tf.keras.metrics.Mean()\n    acc = tf.keras.metrics.BinaryAccuracy()\n    val_acc = tf.keras.metrics.BinaryAccuracy()\n    f1 = tfa.metrics.F1Score(num_classes=5, average='weighted', threshold=0.5)\n    val_f1 = tfa.metrics.F1Score(num_classes=5, average='weighted', threshold=0.5)\n\ntrain_loss_history = []\ntrain_acc_history = []\ntrain_f1_history = []\ntrain_auc_history = []\nval_loss_history = []\nval_acc_history = []\nval_f1_history = []\nval_auc_history = []\nlr_list = []\ndist_train_dataset = strategy.experimental_distribute_dataset(train_dataset)\ndist_val_dataset = strategy.experimental_distribute_dataset(val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"with strategy.scope():\n    def compute_loss(labels, predictions):\n        per_example_loss = loss_object(labels, predictions)\n        return tf.nn.compute_average_loss(per_example_loss, global_batch_size=TRAIN_BATCH_SIZE)\n    \ndef train_step(inputs):\n    images, labels = inputs\n    with tf.GradientTape() as tape:\n        logits = model(images)\n        loss_value = compute_loss(labels, logits)\n    grads = tape.gradient(loss_value, model.trainable_variables)\n    optimizer.apply_gradients(zip(grads, model.trainable_variables))\n    epoch_auc.update_state(labels, logits)\n    acc.update_state(labels, logits)\n    f1.update_state(labels, logits)\n    train_loss_history.append(loss_value)\n    return loss_value\n\n@tf.function\ndef distributed_train_step(dist_inputs):\n    per_replica_losses = strategy.run(train_step, args=(dist_inputs,))\n    loss = strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n                         axis=None)\n    return loss\n\ndef val_step(inputs):\n    images, labels = inputs\n    logits = model(images)\n    loss_value = loss_object(labels, logits)\n    \n    val_f1.update_state(labels, logits)\n    val_loss.update_state(loss_value)\n    val_acc.update_state(labels, logits)\n    val_epoch_auc.update_state(labels, logits)\n    \n@tf.function\ndef distributed_val_step(dist_inputs):\n    strategy.run(val_step, args=(dist_inputs,))\n    \ndef train(epochs,modelname, verbose=1, PATIENCE = 4, DECAY = 0.9):\n    \n    ########################## Epoch Loop ##########################\n    patience = 0\n    for epoch in range(epochs):\n        lr_list.append(optimizer.learning_rate.numpy())\n        ind = 0\n        start = time.time()\n        i = 0\n        print ('\\nEpoch {}/{} '.format(epoch+1, epochs))\n        \n        ####################### Train Loop #########################\n        num_batches = 0\n        loss = 0.0\n        for data in dist_train_dataset:\n            loss += distributed_train_step(data)\n            num_batches += 1\n            auc = epoch_auc.result()\n            accuracy = acc.result()\n            f1score = f1.result()\n            percent = float(i+1) * 100 / len(train_dataset)\n            arrow   = '-' * int(percent/100 * 20 - 1) + '>'\n            spaces  = ' ' * (20 - len(arrow))\n            if(verbose):    \n                print('\\rTraining: [%d/%d] [%s%s] %d %% - Training Loss: %f - Training AUC: %f - Training ACC: %f - Training F1: %f'% (num_batches, len(train_dataset), arrow, spaces, percent, loss/num_batches, auc, accuracy, f1score), end='', flush=True)\n            i += 1\n        \n        train_loss_history.append(loss.numpy()/num_batches)\n        train_acc_history.append(accuracy.numpy())\n        train_f1_history.append(f1score.numpy())\n        train_auc_history.append(auc.numpy())\n        if(not verbose):\n            print(' Epoch Loss: ', loss/num_batches)\n        i = 0\n        if(verbose):\n            print(\" -\", int(time.time()-start), \"s\", end=\"\")\n            print()\n        start = time.time()\n        \n        ####################### Validation Loop #########################\n        num_batches=0\n        for data in dist_val_dataset:\n            num_batches += 1\n            distributed_val_step(data)\n            auc = val_epoch_auc.result()\n            loss = val_loss.result()\n            accuracy = val_acc.result()\n            f1score = val_f1.result()\n            percent = float(i+1) * 100 / len(val_dataset)\n            arrow   = '-' * int(percent/100 * 20 - 1) + '>'\n            spaces  = ' ' * (20 - len(arrow))\n            if(verbose):    \n                print('\\rValidate: [%d/%d] [%s%s] %d %% - Validation Loss: %f - Validation AUC: %f - Validation ACC: %f - Validation F1: %f'% (num_batches, len(val_dataset), arrow, spaces, percent, loss, auc, accuracy, f1score), end='', flush=True)\n            i += 1\n            \n        if(epoch > 0):\n            if(loss.numpy() < min(val_loss_history)):\n                tf.keras.models.save_model(model, './' + modelname + '-best-loss-aug.h5', save_format='h5', include_optimizer=True, overwrite=True)\n\n            if(accuracy.numpy() > max(val_acc_history)):\n                tf.keras.models.save_model(model, './' + modelname + '-best-acc-aug.h5', save_format='h5', include_optimizer=True, overwrite=True)\n\n            if(f1score.numpy() > max(val_f1_history)):\n                tf.keras.models.save_model(model, './' + modelname + '-best-f1-aug.h5', save_format='h5', include_optimizer=True, overwrite=True)\n        \n            if(loss.numpy() >= min(val_loss_history)):\n                if(patience >= PATIENCE):\n                    patience = 0\n                    K.set_value(optimizer.learning_rate, optimizer.learning_rate.numpy()*DECAY)\n                    ind = 1\n                patience += 1\n        \n        val_loss_history.append(loss.numpy())\n        val_acc_history.append(accuracy.numpy())\n        val_f1_history.append(f1score.numpy())\n        val_auc_history.append(auc.numpy())\n        if(verbose):\n            print(\" -\", int(time.time()-start), \"s\")\n            if(ind):\n                print(\"\\nLearning rate reduced to: \", optimizer.learning_rate.numpy())\n            \n        epoch_auc.reset_states()\n        val_epoch_auc.reset_states()\n        val_loss.reset_states()\n        acc.reset_states()\n        val_acc.reset_states()\n        f1.reset_states()\n        val_f1.reset_states()\n    model.save(modelname + '-aug-final-epoch.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(100,'resnet50v2',1, 2, 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,15))\n\nplt.subplot(3,2,1)\nplt.plot(train_loss_history[2:], label = \"train_loss\")\nplt.plot(val_loss_history, label = \"val_loss\")\nplt.title('Loss Profile')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(3,2,2)\nplt.plot(train_acc_history, label = \"train_acc\")\nplt.plot(val_acc_history, label = \"val_acc\")\nplt.title('Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(3,2,3)\nplt.plot(train_f1_history, label = \"train_f1\")\nplt.plot(val_f1_history, label = \"val_f1\")\nplt.title('F1 Score')\nplt.ylabel('F1 Score')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(3,2,4)\nplt.plot(train_auc_history, label = \"train_auc\")\nplt.plot(val_auc_history, label = \"val_auc\")\nplt.title('AUC')\nplt.ylabel('AUC')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.subplot(3,2,5)\nplt.plot(lr_list, label=\"lr\")\nplt.title('Learning Rate')\nplt.ylabel('learning rate')\nplt.xlabel('Epoch')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}