{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import depandencies"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nimport math, os, re, warnings, random\nimport numpy as np \nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom plotly.subplots import make_subplots\nfrom sklearn.manifold import TSNE\nfrom tqdm.notebook import tqdm\nprint(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nIMAGE_SIZE = [512, 512]\nbatch_size = 16 * tpu_strategy.num_replicas_in_sync\nLEARNING_RATE = 3e-5 * tpu_strategy.num_replicas_in_sync\nEPOCHS = 30\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 5\nES_PATIENCE = 10\nN_FOLDS = 5\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-center-{HEIGHT}x{WIDTH}')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  # Auxiliary functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\nvalidation_split = 0.19\nfilenames = tf.io.gfile.glob(TRAINING_FILENAMES) \nsplit = int(0.9 * len(filenames))\ntrain_fns = filenames[:split]\nvalidation_fns = filenames[split:]\n\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n        \ndef parse_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames):\n  # Read from TFRecords. For optimal performance, we interleave reads from multiple files.\n  records = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n  return records.map(parse_tfrecord, num_parallel_calls=AUTO)\n\ndef get_training_dataset():\n  dataset = load_dataset(train_fns)\n\n  # Create some additional training images by randomly flipping and\n  # increasing/decreasing the saturation of images in the training set. \n  def data_augment(image, one_hot_class):\n    modified = tf.image.random_flip_left_right(image)\n    modified = tf.image.random_saturation(modified, 0, 2)\n    return modified, one_hot_class\n  augmented = dataset.map(data_augment, num_parallel_calls=AUTO)\n\n  # Prefetch the next batch while training (autotune prefetch buffer size).\n  return augmented.repeat().shuffle(2048).batch(batch_size).prefetch(AUTO) \n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = load_dataset(validation_fns).batch(batch_size).prefetch(AUTO)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\n\ndef display_one_flower(image, title, subplot, color):\n  plt.subplot(subplot)\n  plt.axis('off')\n  plt.imshow(image)\n  plt.title(title, fontsize=16, color=color)\n  \n# If model is provided, use it to generate predictions.\ndef display_nine_flowers(images, titles, title_colors=None):\n  subplot = 512\n  plt.figure(figsize=(13,13))\n  for i in range(9):\n    color = 'black' if title_colors is None else title_colors[i]\n    display_one_flower(images[i], titles[i], 512+i, color)\n  plt.tight_layout()\n  plt.subplots_adjust(wspace=0.1, hspace=0.1)\n  plt.show()\n\ndef get_dataset_iterator(dataset, n_examples):\n  return dataset.unbatch().batch(n_examples).as_numpy_iterator()\n\ntraining_viz_iterator = get_dataset_iterator(training_dataset, 9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*display images*"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', \n                  fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n    \ntrain_dataset = get_training_dataset()\ntrain_iter = iter(train_dataset.unbatch().batch(20))\n\ndisplay_batch_of_images(next(train_iter))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"#visualizations\n\n#reading training data\ntrain_df = pd.read_csv('../input/cassava-leaf-disease-tfrecords-center-512x512/train.csv')\ndisease_names = {0:'Cassava Bacterial Blight', \n           1:'Cassava Brown Streak Disease', \n           2:'Cassava Green Mottle', \n           3:'Cassava Mosaic Disease', \n           4:'Healthy'}\n\ntrain_df['disease_name'] = train_df['label'].apply(lambda x: disease_names[x])#str()\n\nfig = make_subplots(rows=1, cols=2,\n            specs=[[{\"type\": \"xy\"}, {\"type\": \"domain\"}]],)\n\nt1 = go.Bar(x=train_df['disease_name'].value_counts().index, \n            y=train_df['disease_name'].value_counts().values,\n            text=train_df['disease_name'].value_counts().values,\n            textposition='auto',name='Count',\n           marker_color='indianred')\n#Pie chart with labels and counts\nt2 = go.Pie(labels=train_df['disease_name'].value_counts().index,\n           values=train_df['disease_name'].value_counts().values,\n           hole=0.3)\nfig.add_trace(t1,row=1, col=1)\nfig.add_trace(t2,row=1, col=2)\nfig.update_layout(title='Distribution of Class Labels')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Model****"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n  pretrained_model = tf.keras.applications.DenseNet201(input_shape=[*IMAGE_SIZE, 3], include_top=False)\n  pretrained_model.trainable = True\n  model = tf.keras.Sequential([\n    pretrained_model,\n    tf.keras.layers.GlobalAveragePooling2D(),\n    tf.keras.layers.Dense(5, activation='softmax')\n  ])\n  model.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n      #'sparse_categorical_crossentropy'\n    metrics=['accuracy']\n  )\n  return model\n\nwith tpu_strategy.scope(): # creating the model in the TPUStrategy scope means we will train the model on the TPU\n model = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n  # The number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n  n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n  return np.sum(n)\n\nn_train = count_data_items(train_fns)\nn_valid = count_data_items(validation_fns)\ntrain_steps = count_data_items(train_fns) // batch_size\n\nprint(\"TRAINING IMAGES: \", n_train)\nprint(\"VALIDATION IMAGES: \", n_valid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"learning rate scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 12\n\nstart_lr = 0.00001\nmin_lr = 0.00001\nmax_lr = 0.00005 * tpu_strategy.num_replicas_in_sync\nrampup_epochs = 5\nsustain_epochs = 0\nexp_decay = .8\n\ndef lrfn(epoch):\n  if epoch < rampup_epochs:\n    return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n  elif epoch < rampup_epochs + sustain_epochs:\n    return max_lr\n  else:\n    return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nrang = np.arange(EPOCHS)\ny = [lrfn(x) for x in rang]\nplt.plot(rang, y)\nprint('Learning rate per epoch:')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training the model using 5-fold cross validation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\n\nseed = 0\n\nskf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    #TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])\n    #VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\n    np.random.shuffle(TRAINING_FILENAMES)\n    #ct_train = count_data_items(train_dataset)\n    ## MODEL\n    K.clear_session()\n    with tpu_strategy.scope():\n        model = create_model()\n        \n   \n    model_path =f'./Densenet_{fold}.h5'\n    es = EarlyStopping(monitor='val_sparse_categorical_accuracy', mode='max', \n                       patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n\n    ## TRAIN\n    \n    history = model.fit(train_dataset, validation_data=validation_dataset,steps_per_epoch=train_steps,epochs=EPOCHS, callbacks=[lr_callback]).history\n    history_list.append(history)\n    # Save last model weights\n    model.save_weights(model_path)\n\n    # OOF predictions\n    ds_valid = validation_dataset\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, image_name: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n    \n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_accuracy']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **plot the loss and accuracy in each fold**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\ndef plot_metrics(history):\n    metric_list = [m for m in list(history.keys()) if m is not 'lr']\n    size = len(metric_list)//2\n    fig, axes = plt.subplots(size, 1, sharex='col', figsize=(20, size * 4))\n    if size > 1:\n        axes = axes.flatten()\n    else:\n        axes = [axes]\n    \n    for index in range(len(metric_list)//2):\n        metric_name = metric_list[index]\n        val_metric_name = metric_list[index+size]\n        axes[index].plot(history[metric_name], label='Train %s' % metric_name)\n        axes[index].plot(history[val_metric_name], label='Validation %s' % metric_name)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric_name)\n        if 'loss' in metric_name:\n            axes[index].axvline(np.argmin(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmin(history[val_metric_name]), linestyle='dashed', color='orange')\n        else:\n            axes[index].axvline(np.argmax(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmax(history[val_metric_name]), linestyle='dashed', color='orange')\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()\n    \nfor fold, history in enumerate(history_list):\n    print(f'\\nFOLD: {fold+1}')\n    plot_metrics(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ***Model evaluation using classification report**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\ny_true = np.concatenate(oof_labels)\ny_preds = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_preds, target_names=CLASSES))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}