{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook objective is model performance from [Kernel mashup](https://www.kaggle.com/volody/kernel-mashup)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport tensorflow as tf\n\n# NOTE: internet should be ON\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [192, 192] \nIMAGE_CHANNELS = 3\n\n# related notebook output is using TPU\nBATCH_SIZE = 16 * 8 #strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n# predictions on this dataset should be submitted for the competition\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') \n\nCLASSES = [\n    'pink primrose',    'hard-leaved pocket orchid',\n    'canterbury bells',                 'sweet pea',\n    'wild geranium',                   'tiger lily',           \n    'moon orchid',               'bird of paradise',\n    'monkshood',                    'globe thistle',# 00 - 09\n    'snapdragon',                     \"colt's foot\",\n    'king protea',                  'spear thistle',\n    'yellow iris',                   'globe-flower',         \n    'purple coneflower',            'peruvian lily',\n    'balloon flower',       'giant white arum lily',# 10 - 19\n    'fire lily',                'pincushion flower',\n    'fritillary',                      'red ginger',\n    'grape hyacinth',                  'corn poppy',           \n    'prince of wales feathers',  'stemless gentian',\n    'artichoke',                    'sweet william',# 20 - 29\n    'carnation',                     'garden phlox',\n    'love in the mist',                    'cosmos',\n    'alpine sea holly',      'ruby-lipped cattleya', \n    'cape flower',               'great masterwort',\n    'siam tulip',                     'lenten rose',# 30 - 39\n    'barberton daisy',                   'daffodil',\n    'sword lily',                      'poinsettia',\n    'bolero deep blue',                'wallflower',\n    'marigold',                         'buttercup',\n    'daisy',                     'common dandelion',# 40 - 49\n    'petunia',                         'wild pansy',\n    'primula',                          'sunflower',\n    'lilac hibiscus',          'bishop of llandaff',\n    'gaura',                             'geranium',\n    'orange dahlia',           'pink-yellow dahlia',# 50 - 59\n    'cautleya spicata',          'japanese anemone',\n    'black-eyed susan',                'silverbush',\n    'californian poppy',             'osteospermum',         \n    'spring crocus',                         'iris',\n    'windflower',                      'tree poppy',# 60 - 69\n    'gazania',                             'azalea',\n    'water lily',                            'rose',\n    'thorn apple',                  'morning glory',     \n    'passion flower',                       'lotus',\n    'toad lily',                        'anthurium',# 70 - 79\n    'frangipani',                        'clematis',\n    'hibiscus',                         'columbine',\n    'desert-rose',                    'tree mallow',      \n    'magnolia',                         'cyclamen ',\n    'watercress',                      'canna lily',# 80 - 89\n    'hippeastrum ',                      'bee balm',\n    'pink quill',                        'foxglove',\n    'bougainvillea',                     'camellia',        \n    'mallow',                     'mexican petunia',\n    'bromelia',                    'blanket flower',# 90 - 99\n    'trumpet creeper',            'blackberry lily',\n    'common tulip',                     'wild rose']#100 -103","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\n\n# this notebook is running on CPU \n# hardcode AUTO for dataset to load in same way as it done in \n# kernel-mashup notebook\n\nAUTO = 24 # tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    # convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0  \n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS]) \n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    # returns a dataset of (image, label) pairs\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"id\": tf.io.FixedLenFeature([], tf.string),  \n        # class is missing, this competitions's challenge \n        # is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum \n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, \n    # reading from multiple files at once and\n    # disregarding data order. Order does not matter \n    # since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order) \n    dataset = dataset.map(read_labeled_tfrecord if labeled else \n                          read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True \n    # or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in \n    # the next function (below), this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while \n    # the TPU itself is computing gradients.\n    #image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=False, do_grid=False):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    # Rotation Augmentation GPU/TPU\n    #if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)\n    # grid mask\n    #if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)   \n    # the training dataset must repeat for several epochs\n    dataset = dataset.repeat() \n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef get_validation_dataset(dataset):\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files,\n    # i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.\n      format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load model names\nmashup_path = \"../input/kernel-mashup\" \nMODEL_NAME = ['Model1', 'Model2', 'Model3']\n\ntry:\n    MODEL_NAME = np.load(f'{mashup_path}/model_names.npy')\nexcept:\n    print('model names processing issue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nFOLDS = len(MODEL_NAME)\nSEED = 92\nMODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES\n\nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load predictions\nfold_predictions = []\ntry:\n    for ifold in range(FOLDS):\n        predict = np.load(f'{mashup_path}/cm_fold{ifold+1}.npy')\n        fold_predictions.append(predict)\nexcept:\n    print('KFold processing issue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom matplotlib import pyplot as plt\nfrom matplotlib.colors import LinearSegmentedColormap\n\ndef custom_cmap():\n    # custom cmap, replace 0 with gray\n    cmap = plt.cm.Reds\n    cmaplist = [cmap(i) for i in range(cmap.N)]\n    cmaplist[0] = (0, 0, 0, 0.2)\n    return LinearSegmentedColormap.from_list('mcm' ,cmaplist, cmap.N) \n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap=custom_cmap())\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, \n            'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef get_labels(idx):\n    tst_labels = []; val_labels = []\n\n    j, (trn_ind, val_ind) = list(enumerate(kfold.split(MODEL_FILENAMES)))[idx]\n\n    tdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[trn_ind]['FILENAMES'])\n    NUM_TRAIN_IMAGES = count_data_items(tdf)\n    train_dataset = load_dataset(tdf, labeled = True)\n    tdf_dataset = get_training_dataset(train_dataset, True)\n    tst_labels_ds = tdf_dataset.map(lambda image, label: label).unbatch()\n    tst_labels = next(iter(tst_labels_ds.batch(NUM_TRAIN_IMAGES))).numpy()\n    \n    vdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[val_ind]['FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(vdf)\n    val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n    cm_val_dataset = get_validation_dataset(val_dataset)\n    labels_ds = cm_val_dataset.map(lambda image, label: label).unbatch()\n    val_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n   \n    return tst_labels, val_labels\n    \ndef get_model_confusion_matrix_info(idx):\n    all_pred = []\n\n    tst_labels, val_labels = get_labels(idx)\n    predictions = np.argmax(fold_predictions[idx], axis=-1)\n    \n    cmat = confusion_matrix(val_labels, predictions, \n                     labels=range(len(CLASSES)))\n    score = f1_score(val_labels, predictions, \n                     labels=range(len(CLASSES)), average='macro')\n    precision = precision_score(val_labels, predictions, \n                     labels=range(len(CLASSES)), average='macro')\n    recall = recall_score(val_labels, predictions, \n                     labels=range(len(CLASSES)), average='macro')\n   \n    return (cmat, score, precision, recall)\n    \ndef get_confusion_matrix_info():\n    labels = []; pred = []\n\n    for j, (trn_ind, val_ind) in enumerate( kfold.split(MODEL_FILENAMES) ):\n        vdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[val_ind]['FILENAMES'])\n        NUM_VALIDATION_IMAGES = count_data_items(vdf)\n        val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n        cmdataset = get_validation_dataset(val_dataset)\n        labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n        try:\n            pred.append( np.argmax(fold_predictions[j], axis=-1) )\n            labels.append(next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy())\n        except:\n            print(\"unable to process \", j, \" fold\")\n\n    cm_labels = np.concatenate(labels)\n    cm_preds = np.concatenate(pred)\n\n    cmat = confusion_matrix(cm_labels, cm_preds, labels=range(len(CLASSES)))\n    score = f1_score(cm_labels, cm_preds, labels=range(len(CLASSES)), average='macro')\n    precision = precision_score(cm_labels, cm_preds, labels=range(len(CLASSES)), average='macro')\n    recall = recall_score(cm_labels, cm_preds, labels=range(len(CLASSES)), average='macro')\n   \n    return (cmat, score, precision, recall)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_fold_dataset(idx):\n    ta, va = get_labels(idx)\n    #\n    t_df = pd.DataFrame(data = ta ) \n    tc_df = pd.DataFrame(data = t_df[0].value_counts()).sort_index()\n    #\n    v_df = pd.DataFrame(data = va ) \n    vc_df = pd.DataFrame(data = v_df[0].value_counts()).sort_index()\n    # append pred\n    ts = pd.Series(va != np.argmax(fold_predictions[idx], axis=-1), dtype='int')\n    v_df.insert(1, \"error\", ts)\n    ec_df = v_df.groupby([0]).sum().replace(0, np.nan)\n    # Data\n    return pd.DataFrame({'x': range(len(CLASSES)), 'test': tc_df[0], \n                         'validation': vc_df[0], 'error': ec_df['error']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def try_get_data(idx):\n    try:\n        return count_fold_dataset(idx)\n    except:\n        return None   \n\ndf = []    \nfor i in range(FOLDS):\n    df.append(try_get_data(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# multiple line plot\nplt.figure(figsize=(30,10))\n\nif df[0] is not None:\n    plt.plot( 'x', 'test', data=df[0], marker='o', markerfacecolor='skyblue', \n             markersize=6, color='skyblue', linewidth=1)\n    plt.plot( 'x', 'validation', data=df[0], marker='o', markerfacecolor='olive', \n             markersize=6, color='olive', linewidth=1)\n\nif df[1] is not None:\n    plt.plot( 'x', 'test', data=df[1], marker='o', markerfacecolor='teal', \n             markersize=6, color='teal', linewidth=1)\n    plt.plot( 'x', 'validation', data=df[1], marker='o', markerfacecolor='gold', \n             markersize=6, color='gold', linewidth=1)\n\n# if df2 is not None:\n#     plt.plot( 'x', 'test', data=df2, marker='o', markerfacecolor='teal', \n#             markersize=6, color='teal', linewidth=1)\n#     plt.plot( 'x', 'validation', data=df2, marker='o', markerfacecolor='gold', \n#             markersize=6, color='gold', linewidth=1)\n\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_dataset_counts(idx):\n\n    font = {'family': 'serif',\n        'color':  'darkred',\n        'weight': 'normal',\n        'size': 16,\n        }\n    \n    if df[idx] is not None:\n        # multiple line plot\n        plt.figure(figsize=(30,10))\n        plt.plot( 'x', 'test', data=df[idx], marker='o', \n                 markerfacecolor='blue', \n                 markersize=6, color='skyblue', linewidth=1)\n        plt.plot( 'x', 'validation', data=df[idx], marker='o', \n                 markerfacecolor='olive', \n                 markersize=6, color='olive', linewidth=1)\n        plt.plot( 'x', 'error', data=df[idx], marker='o', \n                 markerfacecolor='red', \n                 markersize=6, color='red', linewidth=1)\n        plt.xlabel(MODEL_NAME[idx], fontdict=font)\n        plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for idx in range(FOLDS):\n    plot_dataset_counts(idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def try_get_model_confusion_matrix_info(idx):\n    try:\n        return get_model_confusion_matrix_info(idx)\n    except:\n        return None  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for j in range(FOLDS):\n    info = try_get_model_confusion_matrix_info(j)\n    if info is not None:\n        print('{4} f1 score: {1:.3f}, precision: {2:.3f}, recall: {3:.3f}'.format(*info, MODEL_NAME[j]))\n        display_confusion_matrix(*info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"info = get_confusion_matrix_info()\nprint('f1 score: {1:.3f}, precision: {2:.3f}, recall: {3:.3f}'.format(*info))\ndisplay_confusion_matrix(*info)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visual validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nimport math\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    # binary string in this case, these are image ID strings\n    if numpy_labels.dtype == object: \n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels \n    # (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    ch = u\"\\u2192\"\n    if correct:\n        return f\"{CLASSES[label]} [OK]\", correct\n    else:\n        return f\"{CLASSES[label]} [OK]\\n[NO{ch}{CLASSES[correct_label]}]\", correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        if not red:\n            plt.title(title, fontsize=int(titlesize), color='black', \n              fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n        else:\n            plt.title(title, fontsize=int(titlesize/1.2), color='red', \n              fontdict={'verticalalignment':'center'}, pad=int(titlesize/2))\n            \n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(images, labels = None, predictions=None, squaring=True):\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square \n    # or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n    \n    if not squaring:\n        cols = 5\n        rows = len(images)//cols\n        # limit by 100\n        if rows > 100:\n            rows = 100\n\n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    # figsize(width, height)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    elif not squaring:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        # magic formula tested to work from 1x1 to 10x10 images\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 \n        if not squaring:\n            dynamic_titlesize = FIGSIZE*SPACING/cols*40+3 \n        subplot = display_one_flower(image, title, subplot, not correct, \n                                     titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_model_images(idx, count=20):\n    j, (trn_ind, val_ind) = list(enumerate(kfold.split(MODEL_FILENAMES)))[idx]\n    vdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[val_ind]['FILENAMES'])\n    val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n    \n    dataset = get_validation_dataset(val_dataset)\n    dataset = dataset.unbatch().batch(count)\n    batch = iter(dataset)\n    \n    # run this cell again for next set of images\n    images, labels = next(batch)\n    predictions = np.argmax(fold_predictions[idx], axis=-1)[:count]\n\n    # data\n    images, labels = batch_to_numpy_images_and_labels((images, labels))\n    \n    display_batch_of_images(images, labels, predictions)\n    \ndef display_model_errors(idx, count=20):\n    j, (trn_ind, val_ind) = list(enumerate(kfold.split(MODEL_FILENAMES)))[idx]\n    vdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[val_ind]['FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(vdf)\n    \n    val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n    cm_val_dataset = get_validation_dataset(val_dataset)\n\n    images_ds = cm_val_dataset.map(lambda image, label: image).unbatch()\n    val_images = next(iter(images_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n    \n    labels_ds = cm_val_dataset.map(lambda image, label: label).unbatch()\n    val_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n    \n    prediction = np.argmax(fold_predictions[idx], axis=-1)\n    \n    images = []; labels = []; predictions = [];\n    \n    for i in range(NUM_VALIDATION_IMAGES):\n        if val_labels[i] != prediction[i]:\n            images.append(val_images[i])\n            labels.append(val_labels[i])\n            predictions.append(prediction[i])\n            if count != None and len(predictions) >= count:\n                break\n    \n    display_batch_of_images(images, labels, predictions, count != None)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    for idx in range(FOLDS):\n        print(f\"KFold{idx+1} model {MODEL_NAME[idx]} 20 images\")    \n        display_model_images(idx)\n        print(f\"KFold{idx+1} model {MODEL_NAME[idx]} errors \")    \n        display_model_errors(idx, None)\nexcept:\n    print('image processing issue')    \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}