{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, gc, random, cv2 \nfrom datetime import datetime \nimport csv \nimport numpy as np\nfrom numpy.random import seed \nimport pandas as pd\nfrom skimage import measure #scikit-image conda-forge \nfrom skimage.transform import resize \nfrom sklearn.metrics import roc_auc_score #scikit-learn conda-forge \nimport tensorflow as tf \nfrom tensorflow import keras \nfrom tensorflow.keras.utils import Sequence \nfrom matplotlib import pyplot as plt \nimport matplotlib.pyplot as plt \nimport matplotlib.image as mpimg \nimport PIL \nfrom tqdm import tqdm\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob('../input/cassava-leaf-disease-classification/train_tfrecords/ld_train00-1338.tfrec')))\n\nDEVICE = \"TPU\"\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\nif tpu:\n    try:\n        print(\"initializing  TPU ...\")\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"TPU initialized\")\n    except _:\n        print(\"failed to initialize TPU\")\nelse:\n    DEVICE = \"GPU\"\nif DEVICE != \"TPU\": \n    print(\"Using default strategy for CPU and single GPU\") \n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\": \n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\n\nCFG = dict( batch_size = 32,\n    read_size         = 600, \n    crop_size         = 300, \n    net_size          = 600)\n           \ndef prepare_image(img, cfg=None, augment=True):\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])\n    return img\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef read_labeled_tfrecord_metadata(example):\n    tfrec_format = {\n        'image'                : tf.io.FixedLenFeature([], tf.string),\n        'image_name'           : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    metadata = [example['image_name']]\n    target = [example['target']]\n    return example['image'], metadata, target, example['image_name']\n\ndef get_dataset_metadata(files, cfg, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord_metadata, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord_metadata(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, metadata, imgname_or_label, SOP: (prepare_image(img, augment=augment, cfg=cfg), metadata, imgname_or_label, SOP), \\\n                num_parallel_calls=AUTO)\n    ds = ds.map(lambda img, metadata, imgname_or_label, SOP: (tuple([img, metadata]), imgname_or_label, SOP), num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds\nprint(\"done\") \n\ndef show_dataset_metadata(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n    for idx, data in enumerate(iter(ds)):\n        inputs, target_or_imgid, SOP = data\n        img = inputs[0]\n        metadata = inputs[1]\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        meta = metadata.numpy()\n        meta = meta[0]\n        meta = meta.decode()\n        print(meta)\n        print(target_or_imgid)\n        imgjpg = cv2.imread('../input/cassava-leaf-disease-classification/train_images/' + meta)\n        plt.imshow(imgjpg)\n        plt.show()\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n        \n    display(mosaic)\nds = get_dataset_metadata(files_train[0], CFG, shuffle=False,augment=False, labeled=True).unbatch().take(1)   \nshow_dataset_metadata(224, 2, 2, ds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}