{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Yum or Yuck Butterfly Mimics 2022 – Load Test Images\n\n**Author:** [Keith Pinson](https://github.com/keithpinson)<br>\n**Date created:** 2022/07/29<br>\n**Version:** 1.0.0001<br>\n**Description:** Example of how to load test images into a TensorFlow Dataset<br>\n**Platform:** Kaggle Packages including Tensorflow 2.6.4 with GPU support<br>\n<br>\n","metadata":{"papermill":{"duration":0.007088,"end_time":"2022-07-29T21:19:49.352655","exception":false,"start_time":"2022-07-29T21:19:49.345567","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import datetime\n\nprint(\"executed\",datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\"local time\")","metadata":{"papermill":{"duration":0.02209,"end_time":"2022-07-29T21:19:49.380849","exception":false,"start_time":"2022-07-29T21:19:49.358759","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:11.958410Z","iopub.execute_input":"2022-08-01T00:11:11.958868Z","iopub.status.idle":"2022-08-01T00:11:11.993090Z","shell.execute_reply.started":"2022-08-01T00:11:11.958778Z","shell.execute_reply":"2022-08-01T00:11:11.991562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## Set Environment\n---\n","metadata":{"papermill":{"duration":0.005816,"end_time":"2022-07-29T21:19:49.392224","exception":false,"start_time":"2022-07-29T21:19:49.386408","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import os\nimport platform\nimport random\n\nimport tensorflow as tf\n\nimport pandas as pd\nimport matplotlib as mpl\nfrom matplotlib import pyplot as plt, patches\n\nprint(f\"Tensorflow {tf.__version__}\", \"with GPU support\" if len(tf.config.list_physical_devices('GPU')) > 0 else \"for CPU only\")","metadata":{"papermill":{"duration":6.368652,"end_time":"2022-07-29T21:19:55.766661","exception":false,"start_time":"2022-07-29T21:19:49.398009","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:12.016188Z","iopub.execute_input":"2022-08-01T00:11:12.016954Z","iopub.status.idle":"2022-08-01T00:11:18.352468Z","shell.execute_reply.started":"2022-08-01T00:11:12.016908Z","shell.execute_reply":"2022-08-01T00:11:18.351115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <u>Dataset paths and names</u>\n","metadata":{"papermill":{"duration":0.00539,"end_time":"2022-07-29T21:19:55.777952","exception":false,"start_time":"2022-07-29T21:19:55.772562","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"os_system = platform.system()  # 'Windows', 'Linux'\n\nhosted_by = 'Kaggle' if os.environ.get('KAGGLE_URL_BASE') else \\\n            ('Windows' if os.environ.get('WINDIR') else \\\n            'Unknown')\n\nif hosted_by == 'Kaggle':\n    dataset_name = \"yum-or-yuck-butterfly-mimics-2022\"\n\n    # Setting the variables assuming a Kaggle platform\n    base_dir = \"/kaggle\"\n    dataset_dir = os.path.join(base_dir, 'input', dataset_name)\n    data_dir = os.path.join(dataset_dir, 'data', 'butterfly_mimics')\n    working_dir = os.path.join(base_dir, 'working')\n    temp_dir = os.path.join(base_dir, 'temp')\n\ntrain_dir = os.path.join(data_dir, 'images')\ntest_dir = os.path.join(data_dir, 'image_holdouts')\ntrain_csv = os.path.join(data_dir, 'images.csv')\ntest_csv = os.path.join(data_dir, 'image_holdouts.csv')\nsubmit_csv = os.path.join(working_dir, 'submission.csv')\n\nclass_names = ['black', 'monarch', 'pipevine', 'spicebush', 'tiger', 'viceroy']\nclass_count = len(class_names)\n","metadata":{"papermill":{"duration":0.020549,"end_time":"2022-07-29T21:19:55.804281","exception":false,"start_time":"2022-07-29T21:19:55.783732","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:18.355069Z","iopub.execute_input":"2022-08-01T00:11:18.355703Z","iopub.status.idle":"2022-08-01T00:11:18.368995Z","shell.execute_reply.started":"2022-08-01T00:11:18.355665Z","shell.execute_reply":"2022-08-01T00:11:18.367483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## Build Dataset Pipeline\n---\n","metadata":{"papermill":{"duration":0.005219,"end_time":"2022-07-29T21:19:55.815220","exception":false,"start_time":"2022-07-29T21:19:55.810001","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### <u>Load and Map Functions</u>","metadata":{"papermill":{"duration":0.005889,"end_time":"2022-07-29T21:19:55.826656","exception":false,"start_time":"2022-07-29T21:19:55.820767","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"IMAGE_WIDTH = IMAGE_HEIGHT = 224\nIMAGE_SIZE = (IMAGE_HEIGHT, IMAGE_WIDTH)  # Row,Column order\nIMAGE_DEPTH = 3\n\ndef get_feature_function(image_id):\n    _image_id = image_id[0].decode('UTF-8')\n\n    _img = tf.io.read_file(os.path.join(\n        test_dir, _image_id + '.jpg'))\n\n    _img = tf.image.decode_jpeg(_img,\n        channels=IMAGE_DEPTH,\n        dct_method='INTEGER_ACCURATE',\n        name=_image_id)\n\n    _img = tf.image.resize(_img,IMAGE_SIZE)\n\n    _img = tf.cast(_img, tf.float32)/255.0\n\n    return _img, image_id\n\ndef get_feature(x):\n\n    features_labels = tf.numpy_function(\n        get_feature_function,\n        [x],\n        [tf.float32,tf.string]\n    )\n\n    # numpy_function() loses the shapes, we will need to restore them\n\n    features_labels[0].set_shape(\n        tf.TensorShape([IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_DEPTH])\n    )\n\n    features_labels[1].set_shape(tf.TensorShape([1]))\n    tf.cast(features_labels[1], tf.string, name='image_id')\n\n    return features_labels\n\ndef load_tests(test_data : pd.core.frame.DataFrame):\n\n    # Test Dataset\n    _test_ds = tf.data.Dataset.from_tensor_slices((test_data))\n    _test_ds = _test_ds.map(get_feature)\n\n\n    return _test_ds\n\ndef decode_image(image):\n    # image.shape == tf.TensorShape([IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_DEPTH])\n    return tf.keras.utils.array_to_img(image.numpy())\n\ndef decode_image_id(image_id):\n    # image_id.shape == tf.TensorShape([1])\n    return image_id.numpy()[0].decode('UTF-8')\n","metadata":{"papermill":{"duration":0.023111,"end_time":"2022-07-29T21:19:55.855502","exception":false,"start_time":"2022-07-29T21:19:55.832391","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:18.374348Z","iopub.execute_input":"2022-08-01T00:11:18.374909Z","iopub.status.idle":"2022-08-01T00:11:18.391325Z","shell.execute_reply.started":"2022-08-01T00:11:18.374859Z","shell.execute_reply":"2022-08-01T00:11:18.390250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <u>Load the Test Images</u>","metadata":{"papermill":{"duration":0.003257,"end_time":"2022-07-29T21:19:55.862479","exception":false,"start_time":"2022-07-29T21:19:55.859222","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"test_ds_encoded = load_tests(pd.read_csv(test_csv))\ntest_ds = [(decode_image(m), decode_image_id(id)) for (m, id) in test_ds_encoded]\n\n\ntest_ds[:5]","metadata":{"papermill":{"duration":5.399001,"end_time":"2022-07-29T21:20:01.264989","exception":false,"start_time":"2022-07-29T21:19:55.865988","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:18.393912Z","iopub.execute_input":"2022-08-01T00:11:18.395174Z","iopub.status.idle":"2022-08-01T00:11:23.044745Z","shell.execute_reply.started":"2022-08-01T00:11:18.395125Z","shell.execute_reply":"2022-08-01T00:11:23.043455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{test_ds[1][1]}.jpg\")\ntest_ds[1][0]","metadata":{"papermill":{"duration":0.060577,"end_time":"2022-07-29T21:20:01.329664","exception":false,"start_time":"2022-07-29T21:20:01.269087","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:23.046362Z","iopub.execute_input":"2022-08-01T00:11:23.047184Z","iopub.status.idle":"2022-08-01T00:11:23.090078Z","shell.execute_reply.started":"2022-08-01T00:11:23.047135Z","shell.execute_reply":"2022-08-01T00:11:23.088839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <u>Show the Images</u>","metadata":{"papermill":{"duration":0.007638,"end_time":"2022-07-29T21:20:01.345587","exception":false,"start_time":"2022-07-29T21:20:01.337949","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"j = 0","metadata":{"papermill":{"duration":0.017517,"end_time":"2022-07-29T21:20:01.371128","exception":false,"start_time":"2022-07-29T21:20:01.353611","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:23.091405Z","iopub.execute_input":"2022-08-01T00:11:23.091888Z","iopub.status.idle":"2022-08-01T00:11:23.098118Z","shell.execute_reply.started":"2022-08-01T00:11:23.091843Z","shell.execute_reply":"2022-08-01T00:11:23.097001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this cell repeatedly to step through the images\n\nrows = 5\ncols = 5\n\nplt.style.use(\"default\")\n\nfig, axs = plt.subplots(nrows=rows, ncols=cols, figsize=(13, 13))\n\nplt.suptitle(\"Test Images\\n\", fontsize=20)\n\nmaxloop = len(test_ds)//(rows*cols)\n\nif j <= maxloop:\n    for i, ax in enumerate(axs.flat):\n        k = (j*rows*cols) + i\n\n        if k < len(test_ds):\n\n            ax.grid(None)\n            ax.axis('on')\n\n            ax.imshow(test_ds[k][0], interpolation_stage='rgb')\n            ax.set(xticks=[], yticks=[], xlabel = test_ds[k][1])\n        else:\n            ax.set_visible(False)\n\n    fig.text(.83, .965, f\"page {j+1}\", color='grey', fontsize=14)\n\n\n    j = j + 1 if j < maxloop else 0\n    plt.show()","metadata":{"papermill":{"duration":2.195198,"end_time":"2022-07-29T21:20:03.573858","exception":false,"start_time":"2022-07-29T21:20:01.378660","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-01T00:11:23.099697Z","iopub.execute_input":"2022-08-01T00:11:23.100103Z","iopub.status.idle":"2022-08-01T00:11:25.279346Z","shell.execute_reply.started":"2022-08-01T00:11:23.100069Z","shell.execute_reply":"2022-08-01T00:11:25.278073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{"papermill":{"duration":0.033302,"end_time":"2022-07-29T21:20:03.639846","exception":false,"start_time":"2022-07-29T21:20:03.606544","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}}]}