{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:40.366853Z","iopub.execute_input":"2022-03-15T07:53:40.367542Z","iopub.status.idle":"2022-03-15T07:53:45.728982Z","shell.execute_reply.started":"2022-03-15T07:53:40.367448Z","shell.execute_reply":"2022-03-15T07:53:45.727362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/swintransformertf')\nfrom swintransformer import SwinTransformer","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:45.730537Z","iopub.execute_input":"2022-03-15T07:53:45.731887Z","iopub.status.idle":"2022-03-15T07:53:46.610551Z","shell.execute_reply.started":"2022-03-15T07:53:45.731846Z","shell.execute_reply":"2022-03-15T07:53:46.609789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU or GPU detection","metadata":{}},{"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:46.611669Z","iopub.execute_input":"2022-03-15T07:53:46.612627Z","iopub.status.idle":"2022-03-15T07:53:48.782751Z","shell.execute_reply.started":"2022-03-15T07:53:46.612595Z","shell.execute_reply":"2022-03-15T07:53:48.781661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition data access\nTPUs read data directly from Google Cloud Storage (GCS). This Kaggle utility will copy the dataset to a GCS bucket co-located with the TPU. If you have multiple datasets attached to the notebook, you can pass the name of a specific dataset to the get_gcs_path function. The name of the dataset is the name of the directory it is mounted in. Use `!ls /kaggle/input/` to list attached datasets.","metadata":{}},{"cell_type":"code","source":"# GCS_DS_PATH = KaggleDatasets().get_gcs_path(\"flower-classification\") # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.784803Z","iopub.execute_input":"2022-03-15T07:53:48.785065Z","iopub.status.idle":"2022-03-15T07:53:48.789478Z","shell.execute_reply.started":"2022-03-15T07:53:48.785031Z","shell.execute_reply":"2022-03-15T07:53:48.788799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [224, 224] # At this size, a GPU will run out of memory. Use the TPU.\n                        # For GPU training, please select 224 x 224 px image size.\nepochs = 15\nbatch_size = 16 * strategy.num_replicas_in_sync\n\n# GCS_PATH_SELECT = { # available image sizes\n#     192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n#     224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n#     331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n#     512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n# }\n# GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\n# VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n# TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition\n\n# CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n#            'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n#            'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n#            'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n#            'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n#            'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n#            'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n#            'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n#            'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n#            'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n#            'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.790990Z","iopub.execute_input":"2022-03-15T07:53:48.791582Z","iopub.status.idle":"2022-03-15T07:53:48.801169Z","shell.execute_reply.started":"2022-03-15T07:53:48.791509Z","shell.execute_reply":"2022-03-15T07:53:48.800410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization utilities\ndata -> pixels, nothing of much interest for the machine learning practitioner in this section.","metadata":{}},{"cell_type":"code","source":"# # numpy and matplotlib defaults\n# np.set_printoptions(threshold=15, linewidth=80)\n\n# def batch_to_numpy_images_and_labels(data):\n#     images, labels = data\n#     numpy_images = images.numpy()\n#     numpy_labels = labels.numpy()\n#     if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n#         numpy_labels = [None for _ in enumerate(numpy_images)]\n#     # If no labels, only image IDs, return None for labels (this is the case for test data)\n#     return numpy_images, numpy_labels\n\n# def title_from_label_and_target(label, correct_label):\n#     if correct_label is None:\n#         return CLASSES[label], True\n#     correct = (label == correct_label)\n#     return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n#                                 CLASSES[correct_label] if not correct else ''), correct\n\n# def display_one_flower(image, title, subplot, red=False, titlesize=16):\n#     plt.subplot(*subplot)\n#     plt.axis('off')\n#     plt.imshow(image)\n#     if len(title) > 0:\n#         plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n#     return (subplot[0], subplot[1], subplot[2]+1)\n    \n# def display_batch_of_images(databatch, predictions=None):\n#     \"\"\"This will work with:\n#     display_batch_of_images(images)\n#     display_batch_of_images(images, predictions)\n#     display_batch_of_images((images, labels))\n#     display_batch_of_images((images, labels), predictions)\n#     \"\"\"\n#     # data\n#     images, labels = batch_to_numpy_images_and_labels(databatch)\n#     if labels is None:\n#         labels = [None for _ in enumerate(images)]\n        \n#     # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n#     rows = int(math.sqrt(len(images)))\n#     cols = len(images)//rows\n        \n#     # size and spacing\n#     FIGSIZE = 13.0\n#     SPACING = 0.1\n#     subplot=(rows,cols,1)\n#     if rows < cols:\n#         plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n#     else:\n#         plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n#     # display\n#     for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n#         title = '' if label is None else CLASSES[label]\n#         correct = True\n#         if predictions is not None:\n#             title, correct = title_from_label_and_target(predictions[i], label)\n#         dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n#         subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n#     #layout\n#     plt.tight_layout()\n#     if label is None and predictions is None:\n#         plt.subplots_adjust(wspace=0, hspace=0)\n#     else:\n#         plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n#     plt.show()\n\n# def display_confusion_matrix(cmat, score, precision, recall):\n#     plt.figure(figsize=(15,15))\n#     ax = plt.gca()\n#     ax.matshow(cmat, cmap='Reds')\n#     ax.set_xticks(range(len(CLASSES)))\n#     ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n#     plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n#     ax.set_yticks(range(len(CLASSES)))\n#     ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n#     plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n#     titlestring = \"\"\n#     if score is not None:\n#         titlestring += 'f1 = {:.3f} '.format(score)\n#     if precision is not None:\n#         titlestring += '\\nprecision = {:.3f} '.format(precision)\n#     if recall is not None:\n#         titlestring += '\\nrecall = {:.3f} '.format(recall)\n#     if len(titlestring) > 0:\n#         ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n#     plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.802530Z","iopub.execute_input":"2022-03-15T07:53:48.802824Z","iopub.status.idle":"2022-03-15T07:53:48.815238Z","shell.execute_reply.started":"2022-03-15T07:53:48.802789Z","shell.execute_reply":"2022-03-15T07:53:48.814469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"code","source":"# def decode_image(image_data):\n#     image = tf.image.decode_jpeg(image_data, channels=3)  # image format uint8 [0,255]\n#     image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n#     return image\n\n# def read_labeled_tfrecord(example):\n#     LABELED_TFREC_FORMAT = {\n#         \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n#         \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n#     }\n#     example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n#     image = decode_image(example['image'])\n#     label = tf.cast(example['class'], tf.int32)\n#     return image, label # returns a dataset of (image, label) pairs\n\n# def read_unlabeled_tfrecord(example):\n#     UNLABELED_TFREC_FORMAT = {\n#         \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n#         \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n#         # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n#     }\n#     example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n#     image = decode_image(example['image'])\n#     idnum = example['id']\n#     return image, idnum # returns a dataset of image(s)\n\n# def load_dataset(filenames, labeled=True, ordered=False):\n#     # Read from TFRecords. For optimal performance, reading from multiple files at once and\n#     # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n#     ignore_order = tf.data.Options()\n#     if not ordered:\n#         ignore_order.experimental_deterministic = False # disable order, increase speed\n\n#     dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n#     dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n#     dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n#     # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n#     return dataset\n\n# def data_augment(image, label):\n#     # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n#     # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n#     # of the TPU while the TPU itself is computing gradients.\n#     image = tf.image.random_flip_left_right(image)\n#     #image = tf.image.random_saturation(image, 0, 2)\n#     return image, label   \n\n# def get_training_dataset():\n#     dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n#     dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n#     dataset = dataset.repeat() # the training dataset must repeat for several epochs\n#     dataset = dataset.shuffle(2048)\n#     dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n#     dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n#     return dataset\n\n# def get_validation_dataset(ordered=False):\n#     dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n#     dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n#     dataset = dataset.cache()\n#     dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n#     return dataset\n\n# def get_test_dataset(ordered=False):\n#     dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n#     return dataset\n\n# def count_data_items(filenames):\n#     # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n#     n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n#     return np.sum(n)\n\n# NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n# NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\n# NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n# STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n# VALIDATION_STEPS = -(-NUM_VALIDATION_IMAGES // BATCH_SIZE) # The \"-(-//)\" trick rounds up instead of down :-)\n# TEST_STEPS = -(-NUM_TEST_IMAGES // BATCH_SIZE)             # The \"-(-//)\" trick rounds up instead of down :-)\n# print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-03-15T07:53:48.816642Z","iopub.execute_input":"2022-03-15T07:53:48.817021Z","iopub.status.idle":"2022-03-15T07:53:48.829440Z","shell.execute_reply.started":"2022-03-15T07:53:48.816987Z","shell.execute_reply":"2022-03-15T07:53:48.828782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset visualizations","metadata":{}},{"cell_type":"code","source":"# # data dump\n# print(\"Training data shapes:\")\n# for image, label in get_training_dataset().take(3):\n#     print(image.numpy().shape, label.numpy().shape)\n# print(\"Training data label examples:\", label.numpy())\n# print(\"Validation data shapes:\")\n# for image, label in get_validation_dataset().take(3):\n#     print(image.numpy().shape, label.numpy().shape)\n# print(\"Validation data label examples:\", label.numpy())\n# print(\"Test data shapes:\")\n# for image, idnum in get_test_dataset().take(3):\n#     print(image.numpy().shape, idnum.numpy().shape)\n# print(\"Test data IDs:\", idnum.numpy().astype('U')) # U=unicode string","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.831975Z","iopub.execute_input":"2022-03-15T07:53:48.832159Z","iopub.status.idle":"2022-03-15T07:53:48.841283Z","shell.execute_reply.started":"2022-03-15T07:53:48.832136Z","shell.execute_reply":"2022-03-15T07:53:48.840566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Peek at training data\n# training_dataset = get_training_dataset()\n# training_dataset = training_dataset.unbatch().batch(20)\n# train_batch = iter(training_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.842591Z","iopub.execute_input":"2022-03-15T07:53:48.843097Z","iopub.status.idle":"2022-03-15T07:53:48.850460Z","shell.execute_reply.started":"2022-03-15T07:53:48.843005Z","shell.execute_reply":"2022-03-15T07:53:48.849710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # run this cell again for next set of images\n# display_batch_of_images(next(train_batch))","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.854312Z","iopub.execute_input":"2022-03-15T07:53:48.854533Z","iopub.status.idle":"2022-03-15T07:53:48.858533Z","shell.execute_reply.started":"2022-03-15T07:53:48.854501Z","shell.execute_reply":"2022-03-15T07:53:48.857754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # peer at test data\n# test_dataset = get_test_dataset()\n# test_dataset = test_dataset.unbatch().batch(20)\n# test_batch = iter(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.860154Z","iopub.execute_input":"2022-03-15T07:53:48.860468Z","iopub.status.idle":"2022-03-15T07:53:48.866280Z","shell.execute_reply.started":"2022-03-15T07:53:48.860432Z","shell.execute_reply":"2022-03-15T07:53:48.865536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # run this cell again for next set of images\n# display_batch_of_images(next(test_batch))","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.868348Z","iopub.execute_input":"2022-03-15T07:53:48.869035Z","iopub.status.idle":"2022-03-15T07:53:48.874452Z","shell.execute_reply.started":"2022-03-15T07:53:48.868997Z","shell.execute_reply":"2022-03-15T07:53:48.873666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\ndef load_split(file: str, is_jsonl):\n    \"\"\"\n    function that loads a .json file in a dictionary, used to load Monte Carlo splits.\n    :param case_based: whether to use cases.\n    :param manual_tags: whether to use manual tags.\n    :param file: the file to be loaded\n    :param path: the path of the file\n    :return: the dictionary of the data and the unique concepts of the file\n    \"\"\"\n\n    with open(file) as json_file:\n\n        if is_jsonl:\n            data_imgs = [json.loads(line)[\"id\"] for line in open(file)] \n            data_labels = [json.loads(line)[\"label\"] for line in open(file)]\n            data_captions = [json.loads(line)[\"text\"] for line in open(file)]\n            data = {}\n            text = {}\n            print(len(data_imgs))\n            \n            for i in range(len(data_imgs)):\n                text[data_imgs[i]+\".jpg\"]  = data_captions[i]\n                if data_labels[i] == '':\n                    data[data_imgs[i]+\".jpg\"] = ['others']\n                else:\n                    labels = data_labels[i].replace(\"'\",\"\").split(\", \")\n                    if len(labels)>1 and \"No Finding\" in labels and \"Support Devices\" in labels:\n                        labels = [\"support devices\"]\n                    if len(labels)==1 and \"No Finding\" in labels:\n                        labels = [\"normal\"]\n                        \n                    data[data_imgs[i]+\".jpg\"] = [x.lower() for x in labels]\n        else:\n            data = json.load(json_file)\n        \n        print('Loaded from: ', file)\n        keys = list(data.keys())\n        partition = {}  # train/val/test partition\n        concepts = []  # concepts for train/val/test\n        for key in keys:\n            partition[key] = data[key]\n            concepts.extend(data[key])\n        concepts = list(set(concepts))\n    return partition, concepts, text  # maybe concepts can be omitted in test set.\n\n\n#from keras.utils import CustomObjectScope\nimport math\n#from Attention import Attention\n# import utilities as u\nimport os\nfrom tqdm import tqdm\nimport numpy as np\nimport random\n\n\ndef load_data( data, concepts_list):\n    \"\"\"\n    function that loads the images and tags in NumPy arrays.\n    Tags become one-hot encoded.\n    :param data: the data dictionary to be loaded into the arrays.\n    :param concepts_list: the list of tags.\n    :return: data as (X,y) that can be used in training.\n    \"\"\"\n    x_data, y_data = [], []\n    # read the data file\n    for img_id in tqdm(data.keys()):\n        image_path = os.path.join(images_dir, img_id)\n        img = image.load_img(image_path, target_size=(224, 224))  # load PIL image.\n        x = image.img_to_array(img)  # turn the PIL image to NumPy array.\n        x = preprocess_input(x)  # mean and std of ImageNet, also [0,1] values.\n        # encode the tags\n        concepts = np.zeros(len(concepts_list), dtype=int)\n        if len(data[img_id]) != 0:\n            image_concepts = data[img_id]\n        else:\n            image_concepts = []\n        for i in range(0, len(concepts_list)):\n            # if the tag is assigned to the image put 1 in its position in the true binary vector\n            if concepts_list[i] in image_concepts:\n                concepts[i] = 1  # 1-hot encoding.\n        x_data.append(x)\n        y_data.append(concepts)\n    # creates images and labels\n    return np.array(x_data), np.array(y_data)\n\n\ndef tune_threshold(model, x_val, y_val, generator, multilabel: bool):\n    \"\"\"\n    tune the threshold in validation data.\n    :param x_val: validation images as NumPy array.\n    :param split: the number of Monte Carlo current split.\n    :return: the best threshold and its validation score.\n    \"\"\"\n\n    # 2D NumPy array(rows=images,columns=prediction for each tag)\n    if generator != None:\n        predictions = model.predict(generator)\n    else:\n        predictions = model.predict(x_val, batch_size=16, verbose=1)\n    steps = 100\n    f1_scores = {}\n    for i in tqdm(range(steps)):\n        threshold = float(i + 1) / steps  # ImageCLEF 2020 tests.\n        y_pred_val =  ( predictions >= threshold).astype('int')\n        if multilabel:\n            y_pred_val = add_column(y_pred_val)\n            val_data = add_column(y_val)\n            f1_scores[threshold] = f1_score(val_data, y_pred_val, average=\"micro\")\n        else:\n            f1_scores[threshold] = f1_score(y_val, y_pred_val)\n\n    best_threshold = max(f1_scores, key=f1_scores.get)  # get key with max value.\n    print('The best F1 score on validation data for split #' + \n          ' is ' + str(f1_scores[best_threshold]) +\n          ' achieved with threshold = ' + str(best_threshold) + '\\n')\n\n    return best_threshold, f1_scores[best_threshold]\n\n\ndef removeNormal(data, concepts):\n    for vals in data.values():\n        if \"normal\" in vals:\n            vals.remove(\"normal\")\n    \n    concepts.remove(\"normal\")\n    \n    return data, concepts\n    \n\nsplitDir = \"../input/mimicmedvillsplit/\"\n\ntrain_data, train_concepts, train_captions = load_split(splitDir+'train.json',True)\n# train_data, train_concepts = removeNormal(train_data, train_concepts)\n\nval_data, val_concepts, val_captions = load_split(splitDir+'valid.json',True)\n# val_data, val_concepts = removeNormal(val_data, val_concepts)\n\n\ntest_data, test_concepts, test_captions = load_split(splitDir+'test.json',True)\n# test_data, test_concepts = removeNormal(test_data, test_concepts)\n\n\nprint('Total tags: ', len(train_concepts))\n\nprint(\"normal\" in train_concepts)\n\n\ntrain_data, train_concepts = removeNormal(train_data, train_concepts)\n\nval_data, val_concepts = removeNormal(val_data, val_concepts)\n\ntest_data, test_concepts = removeNormal(test_data, test_concepts)\n\n\nprint('Total tags: ', len(train_concepts))\n\nimport pandas as pd \n\ndef dict_to_df(data_dict, text_dict):\n    df = pd.DataFrame.from_dict([data_dict]).T\n    df.reset_index(inplace=True)\n    df.columns = [\"filename\",\"labels\"]\n\n    df2 = pd.DataFrame.from_dict([text_dict]).T\n    df2.reset_index(inplace=True)\n    df2.columns = [\"filename\",\"captions\"]\n    \n    return df.join(df2.set_index('filename'), on='filename')\n\n\n\ntrain_df = dict_to_df(train_data, train_captions)\nval_df = dict_to_df(val_data, val_captions)\ntest_df = dict_to_df(test_data, test_captions) \n\ntrain_df.drop('captions', inplace=True, axis=1)\nval_df.drop('captions', inplace=True, axis=1)\ntest_df.drop('captions', inplace=True, axis=1)  \n\ndf = pd.concat([train_df, val_df, test_df]).copy()\ndf = df.sample(frac=1).reset_index(drop=True)\ndel train_df,val_df,test_df\n\nfrom sklearn.preprocessing import MultiLabelBinarizer\nmlb = MultiLabelBinarizer()\ndf.labels = mlb.fit_transform(df.labels.tolist()).tolist()\n# train_df.labels = mlb.fit_transform(train_df.labels.tolist()).tolist()\n# val_df.labels = mlb.fit_transform(val_df.labels.tolist()).tolist()\n# test_df.labels = mlb.fit_transform(test_df.labels.tolist()).tolist() \n\nidx1 = int(0.65 * len(df))\nidx2 = int(0.8 * len(df))\ntrain_df = df[0:idx1].reset_index(drop=True).copy()\nval_df = df[idx1:idx2].reset_index(drop=True).copy()\ntest_df = df[idx2:].reset_index(drop=True).copy()\n\ndel df\nprint(mlb.classes_)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:48.876991Z","iopub.execute_input":"2022-03-15T07:53:48.877187Z","iopub.status.idle":"2022-03-15T07:53:59.502192Z","shell.execute_reply.started":"2022-03-15T07:53:48.877156Z","shell.execute_reply":"2022-03-15T07:53:59.500694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os.path import exists\n\nclass CustomDataGen(tf.keras.utils.Sequence):\n\n    def __init__(self, df, batch_size, input_size, preprocess_input, path, shuffle, modality):\n        self.df = df.copy()\n        self.batch_size = batch_size\n        self.input_size = input_size\n        self.shuffle = shuffle\n        self.n = len(self.df)\n        self.preprocess_input = preprocess_input\n        self.path = path\n        self.modality = modality\n\n    def on_epoch_end(self):\n        if self.shuffle:\n            self.df = self.df.sample(frac=1).reset_index(drop=True)\n\n    def __get_input(self, path, target_size):\n       \n        if exists(self.path+\"Train/\"+path):\n            path = self.path+\"Train/\"+path\n        elif exists(self.path+\"Test/\"+path):\n            path = self.path+\"Test/\"+path\n        else:\n            path = self.path+\"Valid/\"+path\n        \n        image = tf.keras.preprocessing.image.load_img(path, target_size=(target_size[0], target_size[1]))\n        image = tf.keras.preprocessing.image.img_to_array(image)\n#         image = preprocess_input(image)\n        return image \n\n    def __get_data(self, batches):\n        # Generates data containing batch_size samples\n        y_batch = np.asarray([np.asarray(y) for y in batches.labels])\n\n        if self.modality==\"image\":\n            image_batch = np.asarray([self.__get_input(x, self.input_size) for x in batches.filename])\n            return image_batch, y_batch\n        elif self.modality==\"text\":\n            text_batch = np.asarray([np.asarray(x) for x in batches.captions])\n            return text_batch, y_batch\n        elif self.modality==\"multimodal\":\n            image_batch = np.asarray([self.__get_input(x, self.input_size) for x in batches.filename])\n            text_batch = np.asarray([np.asarray(x) for x in batches.captions])\n            return tuple([image_batch, text_batch]), y_batch\n\n    def __getitem__(self, index):\n        batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n        X, y = self.__get_data(batches)        \n        return X, y\n\n    def __len__(self):\n        return self.n // self.batch_size\n        \nimages_dir = \"../input/medvilmimic/mimic/re_512_3ch/\"\n\ntrain_generator = CustomDataGen(train_df, batch_size=batch_size, input_size=(224,224,3), preprocess_input=None, path=images_dir, shuffle=True, modality='image')\nval_generator =  CustomDataGen(val_df, batch_size=batch_size, input_size=(224,224,3), preprocess_input=None, path=images_dir, shuffle=True, modality='image')\n# test_generator =  CustomDataGen(test_df, batch_size=batch_size, input_size=(224,224,3), preprocess_input=None, path=images_dir+'Test/', shuffle=False, modality='image')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:59.503703Z","iopub.execute_input":"2022-03-15T07:53:59.503976Z","iopub.status.idle":"2022-03-15T07:53:59.521148Z","shell.execute_reply.started":"2022-03-15T07:53:59.503935Z","shell.execute_reply":"2022-03-15T07:53:59.520380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = train_generator.n\nNUM_VALIDATION_IMAGES = val_generator.n\n# NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // batch_size\nVALIDATION_STEPS = -(-NUM_VALIDATION_IMAGES // batch_size) # The \"-(-//)\" trick rounds up instead of down :-)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:59.523331Z","iopub.execute_input":"2022-03-15T07:53:59.523859Z","iopub.status.idle":"2022-03-15T07:53:59.530443Z","shell.execute_reply.started":"2022-03-15T07:53:59.523810Z","shell.execute_reply":"2022-03-15T07:53:59.529667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\nYou can select these models:  \n`swin_tiny_224`    \n`swin_small_224`  \n`swin_base_224`  \n`swin_base_384`  \n`swin_large_224`  \n`swin_large_384`  ","metadata":{}},{"cell_type":"code","source":"!pip install tensorflow-addons\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:53:59.531980Z","iopub.execute_input":"2022-03-15T07:53:59.532573Z","iopub.status.idle":"2022-03-15T07:54:08.405647Z","shell.execute_reply.started":"2022-03-15T07:53:59.532535Z","shell.execute_reply":"2022-03-15T07:54:08.404877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_tfa =  tfa.metrics.F1Score(num_classes=len(mlb.classes_), average=\"micro\",threshold=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:54:08.407344Z","iopub.execute_input":"2022-03-15T07:54:08.407611Z","iopub.status.idle":"2022-03-15T07:54:08.441743Z","shell.execute_reply.started":"2022-03-15T07:54:08.407577Z","shell.execute_reply":"2022-03-15T07:54:08.441039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:54:08.444510Z","iopub.execute_input":"2022-03-15T07:54:08.444706Z","iopub.status.idle":"2022-03-15T07:54:08.452414Z","shell.execute_reply.started":"2022-03-15T07:54:08.444681Z","shell.execute_reply":"2022-03-15T07:54:08.451655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nearly_stopping = EarlyStopping(monitor='val_loss', patience=3, mode='auto', restore_best_weights=True, verbose=1)\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=1, verbose=1, mode='min')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:54:08.453879Z","iopub.execute_input":"2022-03-15T07:54:08.454303Z","iopub.status.idle":"2022-03-15T07:54:08.460415Z","shell.execute_reply.started":"2022-03-15T07:54:08.454256Z","shell.execute_reply":"2022-03-15T07:54:08.459746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with strategy.scope():\nimg_adjust_layer = tf.keras.layers.Lambda(lambda data: tf.keras.applications.imagenet_utils.preprocess_input(tf.cast(data, tf.float32), mode=\"torch\"), input_shape=[*IMAGE_SIZE, 3])\n#     pretrained_model = SwinTransformer('swin_large_224', len(mlb.classes_), include_top=False, pretrained=True, use_tpu=True)\npretrained_model = SwinTransformer('swin_large_224', len(mlb.classes_), include_top=False, pretrained=True, use_tpu=False)\n\nmodel = tf.keras.Sequential([\n    img_adjust_layer,\n    pretrained_model,\n    tf.keras.layers.Dense(len(mlb.classes_), activation='sigmoid')\n])\n\nmodel.compile(\noptimizer=tf.keras.optimizers.Adam(learning_rate=1e-4, epsilon=1e-8),\nloss = 'binary_crossentropy',\nmetrics=[f1_tfa]\n)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:54:08.461829Z","iopub.execute_input":"2022-03-15T07:54:08.462188Z","iopub.status.idle":"2022-03-15T07:54:34.218134Z","shell.execute_reply.started":"2022-03-15T07:54:08.462151Z","shell.execute_reply":"2022-03-15T07:54:34.217419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"history = model.fit(train_generator,  epochs=epochs,\n                    validation_data=val_generator, callbacks=[early_stopping, reduce_lr])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:54:34.219321Z","iopub.execute_input":"2022-03-15T07:54:34.219577Z","iopub.status.idle":"2022-03-15T07:56:18.313577Z","shell.execute_reply.started":"2022-03-15T07:54:34.219540Z","shell.execute_reply":"2022-03-15T07:56:18.312815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.314803Z","iopub.execute_input":"2022-03-15T07:56:18.315222Z","iopub.status.idle":"2022-03-15T07:56:18.528126Z","shell.execute_reply.started":"2022-03-15T07:56:18.315184Z","shell.execute_reply":"2022-03-15T07:56:18.527409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['f1_score'])\nplt.plot(history.history['val_f1_score'])","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.529412Z","iopub.execute_input":"2022-03-15T07:56:18.529823Z","iopub.status.idle":"2022-03-15T07:56:18.713999Z","shell.execute_reply.started":"2022-03-15T07:56:18.529784Z","shell.execute_reply":"2022-03-15T07:56:18.713293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix\n\n\ndef add_column(y):\n    return np.hstack((y,(np.sum(y, axis=1) == 0).astype('int').reshape(-1, 1) ))","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.715150Z","iopub.execute_input":"2022-03-15T07:56:18.716030Z","iopub.status.idle":"2022-03-15T07:56:18.721688Z","shell.execute_reply.started":"2022-03-15T07:56:18.715991Z","shell.execute_reply":"2022-03-15T07:56:18.720778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val = np.array(val_df.labels.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.723218Z","iopub.execute_input":"2022-03-15T07:56:18.723924Z","iopub.status.idle":"2022-03-15T07:56:18.733470Z","shell.execute_reply.started":"2022-03-15T07:56:18.723884Z","shell.execute_reply":"2022-03-15T07:56:18.732641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_generator2 =  CustomDataGen(val_df, batch_size=1, input_size=(224,224,3), preprocess_input=None, path=images_dir, shuffle=False, modality='image')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.735790Z","iopub.execute_input":"2022-03-15T07:56:18.736443Z","iopub.status.idle":"2022-03-15T07:56:18.743617Z","shell.execute_reply.started":"2022-03-15T07:56:18.736398Z","shell.execute_reply":"2022-03-15T07:56:18.742831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(val_generator2)\n\nbest_threshold = 0\nbest_f1 = 0\nfor thres in np.arange(0,0.7,0.05):\n    y_p = ( y_pred >= thres).astype('int')\n    f1 =  f1_score(y_val, y_p, average=\"micro\")\n    print(thres,f1 )\n    if f1>best_f1:\n        best_f1 = f1\n        best_threshold = thres","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:18.745601Z","iopub.execute_input":"2022-03-15T07:56:18.746521Z","iopub.status.idle":"2022-03-15T07:56:35.229146Z","shell.execute_reply.started":"2022-03-15T07:56:18.746479Z","shell.execute_reply":"2022-03-15T07:56:35.228415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred =  ( model.predict(val_generator2) >= best_threshold).astype('int')\ny_pred2 = add_column(y_pred)\ny_val2 = add_column(y_val)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:35.233243Z","iopub.execute_input":"2022-03-15T07:56:35.233454Z","iopub.status.idle":"2022-03-15T07:56:46.730149Z","shell.execute_reply.started":"2022-03-15T07:56:35.233420Z","shell.execute_reply":"2022-03-15T07:56:46.729352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for avg_type in ['micro','macro']:\n    print(avg_type, f1_score(y_pred2, y_val2, average=avg_type)  )\n#     print(\"Samples\",avg_type, samples_f1(y_pred2, y_val2,avg_type)  )","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.731425Z","iopub.execute_input":"2022-03-15T07:56:46.731954Z","iopub.status.idle":"2022-03-15T07:56:46.745141Z","shell.execute_reply.started":"2022-03-15T07:56:46.731906Z","shell.execute_reply":"2022-03-15T07:56:46.744413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\ntarget_names = train_concepts.copy()\ntarget_names.append(\"normal\")\n\nprint( classification_report(y_val2, y_pred2, target_names=target_names) )","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.746377Z","iopub.execute_input":"2022-03-15T07:56:46.746815Z","iopub.status.idle":"2022-03-15T07:56:46.767186Z","shell.execute_reply.started":"2022-03-15T07:56:46.746779Z","shell.execute_reply":"2022-03-15T07:56:46.766408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.dummy import DummyClassifier\ndummy_clf = DummyClassifier(strategy=\"most_frequent\")\ny_train = np.array(train_df.labels.tolist())\ndummy_clf.fit(y_train,y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.768321Z","iopub.execute_input":"2022-03-15T07:56:46.768638Z","iopub.status.idle":"2022-03-15T07:56:46.784442Z","shell.execute_reply.started":"2022-03-15T07:56:46.768599Z","shell.execute_reply":"2022-03-15T07:56:46.783774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for avg_type in ['micro','macro']:\n    print(avg_type, f1_score(y_pred2, y_val2, average=avg_type)  )","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.786764Z","iopub.execute_input":"2022-03-15T07:56:46.787011Z","iopub.status.idle":"2022-03-15T07:56:46.799526Z","shell.execute_reply.started":"2022-03-15T07:56:46.786979Z","shell.execute_reply":"2022-03-15T07:56:46.798779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( classification_report(y_val2,y_pred2, target_names=target_names) )","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.800691Z","iopub.execute_input":"2022-03-15T07:56:46.801343Z","iopub.status.idle":"2022-03-15T07:56:46.818349Z","shell.execute_reply.started":"2022-03-15T07:56:46.801302Z","shell.execute_reply":"2022-03-15T07:56:46.817586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.mkdir(\"saved\")","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.819654Z","iopub.execute_input":"2022-03-15T07:56:46.820490Z","iopub.status.idle":"2022-03-15T07:56:46.825057Z","shell.execute_reply.started":"2022-03-15T07:56:46.820449Z","shell.execute_reply":"2022-03-15T07:56:46.823958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights(\"saved/ckpt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:46.826667Z","iopub.execute_input":"2022-03-15T07:56:46.827208Z","iopub.status.idle":"2022-03-15T07:56:49.438311Z","shell.execute_reply.started":"2022-03-15T07:56:46.827164Z","shell.execute_reply":"2022-03-15T07:56:49.437268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.make_archive(\"saved\", 'zip', 'saved')","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:56:49.443873Z","iopub.execute_input":"2022-03-15T07:56:49.447343Z","iopub.status.idle":"2022-03-15T07:57:13.903349Z","shell.execute_reply.started":"2022-03-15T07:56:49.447294Z","shell.execute_reply":"2022-03-15T07:57:13.902290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix","metadata":{}},{"cell_type":"code","source":"# cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\n# images_ds = cmdataset.map(lambda image, label: image)\n# labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n# cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n# cm_probabilities = model.predict(images_ds, steps=VALIDATION_STEPS)\n# cm_predictions = np.argmax(cm_probabilities, axis=-1)\n# print(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\n# print(\"Predicted labels: \", cm_predictions.shape, cm_predictions)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:57:13.904426Z","iopub.status.idle":"2022-03-15T07:57:13.906078Z","shell.execute_reply.started":"2022-03-15T07:57:13.905811Z","shell.execute_reply":"2022-03-15T07:57:13.905839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\n# score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# cmat = (cmat.T / cmat.sum(axis=1)).T # normalized\n# display_confusion_matrix(cmat, score, precision, recall)\n# print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","metadata":{"execution":{"iopub.status.busy":"2022-03-15T07:57:13.907488Z","iopub.status.idle":"2022-03-15T07:57:13.908202Z","shell.execute_reply.started":"2022-03-15T07:57:13.907927Z","shell.execute_reply":"2022-03-15T07:57:13.907980Z"},"trusted":true},"execution_count":null,"outputs":[]}]}