{"cells":[{"metadata":{},"cell_type":"markdown","source":"References:\n* https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n* https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-tensorflow-inference/data\n* https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-tensorflow-training/notebook#Training\n\nTraining Notebook:\nhttps://www.kaggle.com/ajaykumar7778/tpu-train-effnets/output?scriptVersionId=48358834"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses, metrics\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"GPU\" #or \"TPU\"\n\n# train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n# test  = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub   = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n\nFOLDS = 3\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768\nIMG_SIZES = [256, 384, 512]\n# INCLUDE OLD COMP DATA? YES=1 NO=0\n# INC2019 = [1,0,1,1,0,1,0,1]\n# INC2018 = [0,1,1,0,1,1,1,1]\nBATCH_SIZES = [64,64,64]   #[64,64,64]\nEPOCHS = [20, 20, 20]   #[10, 10, 10]\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [2, 2, 3]\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS#[1/FOLDS,1/FOLDS,1/FOLDS]\n# TEST TIME AUGMENTATION STEPS\nTTA_STEPS = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef build_model(dim=128, ef=0):\n    input_image = tf.keras.layers.Input(shape=(dim,dim,3))\n    #base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n    \n    base_model = EFNS[ef](input_tensor=input_image, \n                                    include_top=False, \n                                    #weights='noisy-student', \n                                    weights = None,\n                                    pooling='avg')\n    \n    model = Sequential([\n                base_model,\n                L.Dropout(.25),\n                L.Dense(N_CLASSES, activation='softmax', name='output')\n            ])\n    \n#     optimizer = optimizers.Adam(lr=LEARNING_RATE)\n#     model.compile(optimizer=optimizer, \n#                   loss=losses.SparseCategoricalCrossentropy(), \n#                   metrics=['sparse_categorical_accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\n\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nsubmission = pd.read_csv(f'{database_base_path}sample_submission.csv')\ndisplay(submission.head())\n\nTEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'GCS: test: {NUM_TEST_IMAGES}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\n\nmodel_path_list = glob.glob('../input/exp-1-3-fold/fold*.h5')\nmodel_path_list.sort()\n\n\nprint('Models to predict:')\nprint(*model_path_list, sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_path.split('/')[-1].split('.')[0].split('-')[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef get_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    name = parts[-1]\n    return name\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    \n#     image = center_crop(image)\n    return image\n\ndef center_crop(image, h, w):\n    image = tf.reshape(image, [600, 800, CHANNELS]) # Original shape\n    \n    h, w = image.shape[0], image.shape[1]\n    if h > w:\n        image = tf.image.crop_to_bounding_box(image, (h - w) // 2, 0, w, w)\n    else:\n        image = tf.image.crop_to_bounding_box(image, 0, (w - h) // 2, h, h)\n        \n    image = tf.image.resize(image, [h, w]) # Expected shape\n    return image\n\ndef resize_image(image, label, h, w):\n    image = tf.image.resize(image, [h, w])\n    image = tf.reshape(image, [h, w, CHANNELS])\n    return image, label\n\ndef process_path(file_path):\n    name = get_name(file_path)\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, name\n\ndef get_dataset(files_path, h, w, shuffled=False, tta=False, extension='jpg'):\n    dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTO)\n    if tta:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    #dataset = dataset.map(resize_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfiles_path = f'{database_base_path}/test_images/'\ntest_preds = np.zeros((len(os.listdir(files_path)), N_CLASSES))\n\n\nfor model_path in model_path_list:\n    fold = int(model_path.split('/')[-1].split('.')[0].split('-')[-1])\n    print(model_path)\n    h = IMG_SIZES[fold]\n    w = IMG_SIZES[fold]\n#     K.clear_session()\n#     model.load_weights(model_path)\n\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])\n    \n    print('Loading model...')\n    model.load_weights(model_path)\n    #model.load_weights('../input/exp-1-3-fold/fold-%i.h5'%fold)\n\n    if TTA_STEPS > 0:\n        test_ds = get_dataset(files_path, h, w, tta=True)\n        for step in range(TTA_STEPS):\n            print(f'TTA step {step+1}/{TTA_STEPS}')\n            x_test = test_ds.map(lambda image, image_name: image)\n            test_preds += model.predict(x_test) / (TTA_STEPS * len(model_path_list))\n    else:\n        test_ds = get_dataset(files_path, h, w, tta=False)\n        x_test = test_ds.map(lambda image, image_name: image)\n        test_preds += model.predict(x_test) / len(model_path_list)\n    \ntest_preds = np.argmax(test_preds, axis=-1)\nimage_names = [img_name.numpy().decode('utf-8') for img, img_name in iter(test_ds.unbatch())]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for fold, history in enumerate(history_list):\n#     print(f'\\nFOLD: {fold+1}')\n#     plot_metrics(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model evaluation\n\nNow we can evaluate the performance of the model, first, we can evaluate the usual metrics like, accuracy, precision, recall, and f1-score, scikit-learn provides the perfect function for this classification_report.\n\nWe are evaluating the model on the OOF predictions, it stands for Out Of Fold, since we are training using K-Fold our model will see all the data, and the correct way to evaluate each fold is by looking at the predictions that are not from that fold.\nOOF metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_true = np.concatenate(oof_labels)\n# y_preds = np.concatenate(oof_pred)\n\n# print(classification_report(y_true, y_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nConfusion matrix\n\nLet's also take a look at the confusion matrix, this will give us an idea about what classes the model is mixing or having a hard time.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# fig, ax = plt.subplots(1, 1, figsize=(20, 12))\n# train_cfn_matrix = confusion_matrix(y_true, y_preds, labels=range(len(CLASSES)))\n# train_cfn_matrix = (train_cfn_matrix.T / train_cfn_matrix.sum(axis=1)).T\n# train_df_cm = pd.DataFrame(train_cfn_matrix, index=CLASSES, columns=CLASSES)\n# ax = sns.heatmap(train_df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5).set_title('Train', fontsize=30)\n# plt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visualize predictions\n\nFinally, it is a good practice to always inspect some of the model's prediction by looking at the data, this can give an idea if the model is getting some predictions wrong because the data is really hard, of if it is because the model is actually bad.\nClass map\n\n0: Cassava Bacterial Blight (CBB)\n1: Cassava Brown Streak Disease (CBSD)\n2: Cassava Green Mottle (CGM)\n3: Cassava Mosaic Disease (CMD)\n4: Healthy"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_samp, y_samp = dataset_to_numpy_util(train_dataset, 25)\n\n# x_samp_1, y_samp_1 = x_samp[:9,:,:,:], y_samp[:9]\n# samp_preds_1 = model.predict(x_samp_1, batch_size=9)\n# display_9_images_with_predictions(x_samp_1, samp_preds_1, y_samp_1)\n\n# x_samp_2, y_samp_2 = x_samp[9:,:,:,:], y_samp[9:]\n# samp_preds_2 = model.predict(x_samp_2, batch_size=9)\n# display_9_images_with_predictions(x_samp_2, samp_preds_2, y_samp_2)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}