{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71885,"databundleVersionId":8143495,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Image Matching Challenge 2024 - Hexathlon**\n\n## **Notebook Published by:** Aarish Asif Khan\n\n## **Date:** 20th April 2024","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\nfrom tensorflow.keras.models import Sequential\n\nimport os\nimport pandas as pd \n\nfrom matplotlib import pyplot as plt\nimport time","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.267241Z","iopub.execute_input":"2024-04-21T06:28:59.267608Z","iopub.status.idle":"2024-04-21T06:28:59.273377Z","shell.execute_reply.started":"2024-04-21T06:28:59.267578Z","shell.execute_reply":"2024-04-21T06:28:59.272445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining Constants \nImage_size = (128, 128)\n\nImg_width = 128\nImg_height = 128\n\nbatch_size = 32\nepochs = 15\nchannels = 3\n\n# Define image dimensions\nnum_classes = 7","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.278199Z","iopub.execute_input":"2024-04-21T06:28:59.278466Z","iopub.status.idle":"2024-04-21T06:28:59.286670Z","shell.execute_reply.started":"2024-04-21T06:28:59.278443Z","shell.execute_reply":"2024-04-21T06:28:59.285844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the datasets\n# 1. sample_submission.csv\nsample_submission = pd.read_csv(\"/kaggle/input/image-matching-challenge-2024/sample_submission.csv\")\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.288123Z","iopub.execute_input":"2024-04-21T06:28:59.288673Z","iopub.status.idle":"2024-04-21T06:28:59.308424Z","shell.execute_reply.started":"2024-04-21T06:28:59.288641Z","shell.execute_reply":"2024-04-21T06:28:59.307539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. train_labels.csv\ntrain_labels = pd.read_csv(\"/kaggle/input/image-matching-challenge-2024/train/train_labels.csv\")\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.309822Z","iopub.execute_input":"2024-04-21T06:28:59.310061Z","iopub.status.idle":"2024-04-21T06:28:59.334035Z","shell.execute_reply.started":"2024-04-21T06:28:59.310041Z","shell.execute_reply":"2024-04-21T06:28:59.333102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3. categories.csv\ncategories = pd.read_csv(\"/kaggle/input/image-matching-challenge-2024/train/categories.csv\")\ncategories.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.335169Z","iopub.execute_input":"2024-04-21T06:28:59.335609Z","iopub.status.idle":"2024-04-21T06:28:59.346460Z","shell.execute_reply.started":"2024-04-21T06:28:59.335577Z","shell.execute_reply":"2024-04-21T06:28:59.345540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting\n# 1. sample_submission.csv\nsample_submission['dataset'].value_counts().plot(kind='bar')\nplt.xlabel('Category')\nplt.ylabel('Vals')\nplt.title('Sample Submission')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.348347Z","iopub.execute_input":"2024-04-21T06:28:59.348650Z","iopub.status.idle":"2024-04-21T06:28:59.531880Z","shell.execute_reply.started":"2024-04-21T06:28:59.348627Z","shell.execute_reply":"2024-04-21T06:28:59.531013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for missing values in the three datasets\nprint(\"Missing values in Sample-Submission:\", sample_submission.isnull().sum())\n\nprint(\"Missing values in Train-labels:\", train_labels.isnull().sum())\n\nprint(\"Missing values in Categories:\", categories.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.533153Z","iopub.execute_input":"2024-04-21T06:28:59.533496Z","iopub.status.idle":"2024-04-21T06:28:59.544023Z","shell.execute_reply.started":"2024-04-21T06:28:59.533463Z","shell.execute_reply":"2024-04-21T06:28:59.543116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for duplicated values in the three datasets\nprint(\"Duplicates in Sample-Submission:\", sample_submission.duplicated().any())\n\nprint(\"Duplicates in Train-labels:\", train_labels.duplicated().any())\n\nprint(\"Duplicates in Categories:\", categories.duplicated().any())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.546615Z","iopub.execute_input":"2024-04-21T06:28:59.547010Z","iopub.status.idle":"2024-04-21T06:28:59.561707Z","shell.execute_reply.started":"2024-04-21T06:28:59.546980Z","shell.execute_reply":"2024-04-21T06:28:59.560685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Path to the train and test directories\ntrain_dir = '/kaggle/input/image-matching-challenge-2024/train'\ntest_dir = '/kaggle/input/image-matching-challenge-2024/test'","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.562878Z","iopub.execute_input":"2024-04-21T06:28:59.563227Z","iopub.status.idle":"2024-04-21T06:28:59.567533Z","shell.execute_reply.started":"2024-04-21T06:28:59.563195Z","shell.execute_reply":"2024-04-21T06:28:59.566769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Path for the training images and testing\ntrain_images = tf.data.Dataset.list_files(train_dir + '/*')\ntest_images = tf.data.Dataset.list_files(test_dir + '/*')","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.568466Z","iopub.execute_input":"2024-04-21T06:28:59.568735Z","iopub.status.idle":"2024-04-21T06:28:59.594104Z","shell.execute_reply.started":"2024-04-21T06:28:59.568713Z","shell.execute_reply":"2024-04-21T06:28:59.593367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filtering the training and testing images to exclude .csv\ntrain_images = train_images.filter(lambda x: not tf.strings.regex_full_match(x, '.*\\.csv'))\ntest_images = test_images.filter(lambda x: not tf.strings.regex_full_match(x, '.*\\.csv'))","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.595151Z","iopub.execute_input":"2024-04-21T06:28:59.595389Z","iopub.status.idle":"2024-04-21T06:28:59.615397Z","shell.execute_reply.started":"2024-04-21T06:28:59.595368Z","shell.execute_reply":"2024-04-21T06:28:59.614715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file_path in train_images:\n    print(file_path.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.616367Z","iopub.execute_input":"2024-04-21T06:28:59.616645Z","iopub.status.idle":"2024-04-21T06:28:59.631944Z","shell.execute_reply.started":"2024-04-21T06:28:59.616622Z","shell.execute_reply":"2024-04-21T06:28:59.631102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file_path in test_images:\n    print(file_path.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.635880Z","iopub.execute_input":"2024-04-21T06:28:59.636195Z","iopub.status.idle":"2024-04-21T06:28:59.649349Z","shell.execute_reply.started":"2024-04-21T06:28:59.636173Z","shell.execute_reply":"2024-04-21T06:28:59.648586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shuffing\ntrain_images = train_images.shuffle(500)\ntest_images = test_images.shuffle(500)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.650481Z","iopub.execute_input":"2024-04-21T06:28:59.650814Z","iopub.status.idle":"2024-04-21T06:28:59.657614Z","shell.execute_reply.started":"2024-04-21T06:28:59.650791Z","shell.execute_reply":"2024-04-21T06:28:59.656791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file_path in train_images:\n    print(file_path.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.658575Z","iopub.execute_input":"2024-04-21T06:28:59.658818Z","iopub.status.idle":"2024-04-21T06:28:59.675868Z","shell.execute_reply.started":"2024-04-21T06:28:59.658796Z","shell.execute_reply":"2024-04-21T06:28:59.675074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classes in our dataset both of training and testing\ntrain_classes = [\"church\", \"dioscuri\", \"lizard\", \"multi-temporal-temple-baalshar\", \"pond\", \"transp_obj_glass_cup\", \"transp_obj_glass_cylinder\"]\ntest_classes = [\"church\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.676893Z","iopub.execute_input":"2024-04-21T06:28:59.677126Z","iopub.status.idle":"2024-04-21T06:28:59.682153Z","shell.execute_reply.started":"2024-04-21T06:28:59.677099Z","shell.execute_reply":"2024-04-21T06:28:59.681321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding the count of classes for training\ntrain_images_count = 0\nfor _ in train_images:\n    train_images_count += 1\n\nprint(train_images_count)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.683100Z","iopub.execute_input":"2024-04-21T06:28:59.683360Z","iopub.status.idle":"2024-04-21T06:28:59.699425Z","shell.execute_reply.started":"2024-04-21T06:28:59.683338Z","shell.execute_reply":"2024-04-21T06:28:59.698605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding the count of classes for testing\ntest_images_count = 0\nfor _ in test_images:\n    test_images_count += 1\n\nprint(test_images_count)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.700645Z","iopub.execute_input":"2024-04-21T06:28:59.701178Z","iopub.status.idle":"2024-04-21T06:28:59.715222Z","shell.execute_reply.started":"2024-04-21T06:28:59.701145Z","shell.execute_reply":"2024-04-21T06:28:59.714417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(train_images_count + test_images_count * 0.8)\n\ntrain_ds = train_images.take(train_size)\ntest_ds = train_images.skip(train_size)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.716219Z","iopub.execute_input":"2024-04-21T06:28:59.716468Z","iopub.status.idle":"2024-04-21T06:28:59.722379Z","shell.execute_reply.started":"2024-04-21T06:28:59.716445Z","shell.execute_reply":"2024-04-21T06:28:59.721577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_label(file_path):\n    return tf.strings.split(file_path, os.path.sep)[-2]","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.723487Z","iopub.execute_input":"2024-04-21T06:28:59.723773Z","iopub.status.idle":"2024-04-21T06:28:59.731710Z","shell.execute_reply.started":"2024-04-21T06:28:59.723742Z","shell.execute_reply":"2024-04-21T06:28:59.730858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_image(file_path):\n    label = get_label(file_path)\n    \n    img = tf.io.read_file(file_path)\n    img = tf.image.decode_jpeg(img)\n    img = tf.image.resize(img, [128, 128])\n    \n    return img, label","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.732763Z","iopub.execute_input":"2024-04-21T06:28:59.733053Z","iopub.status.idle":"2024-04-21T06:28:59.742373Z","shell.execute_reply.started":"2024-04-21T06:28:59.733030Z","shell.execute_reply":"2024-04-21T06:28:59.741591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in train_ds.take(4):\n    print (t.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.743471Z","iopub.execute_input":"2024-04-21T06:28:59.743887Z","iopub.status.idle":"2024-04-21T06:28:59.766318Z","shell.execute_reply.started":"2024-04-21T06:28:59.743857Z","shell.execute_reply":"2024-04-21T06:28:59.765516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.image as mpimg","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.767445Z","iopub.execute_input":"2024-04-21T06:28:59.767780Z","iopub.status.idle":"2024-04-21T06:28:59.771807Z","shell.execute_reply.started":"2024-04-21T06:28:59.767757Z","shell.execute_reply":"2024-04-21T06:28:59.770899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_image(file_path, label):\n\n    img = tf.io.read_file(file_path)\n \n    img = tf.image.decode_jpeg(img, channels=3)  # Adjust channels if needed\n    \n    img = tf.image.resize(img, [img_height, img_width])  \n   \n    img = img / 255.0\n    return img, label","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.772937Z","iopub.execute_input":"2024-04-21T06:28:59.773752Z","iopub.status.idle":"2024-04-21T06:28:59.781775Z","shell.execute_reply.started":"2024-04-21T06:28:59.773726Z","shell.execute_reply":"2024-04-21T06:28:59.780926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Showing Random Images from Directory (Train and Test)**","metadata":{}},{"cell_type":"markdown","source":"# **Loading Images from Train directory**\n\nYou can show random images by applying the code below from the training directory with matplotlib or seaborn.","metadata":{}},{"cell_type":"code","source":"image_path = \"/kaggle/input/image-matching-challenge-2024/train/church/images/00004.png\"\n\n# Load the image\nimage = mpimg.imread(image_path)\n\n# Display the image\nplt.imshow(image)\nplt.axis('off')  # Turn off axis\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:28:59.782770Z","iopub.execute_input":"2024-04-21T06:28:59.783022Z","iopub.status.idle":"2024-04-21T06:29:00.037466Z","shell.execute_reply.started":"2024-04-21T06:28:59.783000Z","shell.execute_reply":"2024-04-21T06:29:00.036300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = \"/kaggle/input/image-matching-challenge-2024/train/church/images/00005.png\"\n\n# Load the image\nimage = mpimg.imread(image_path)\n\n# Display the image\nplt.imshow(image)\nplt.axis('off')  # Turn off axis\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:00.038829Z","iopub.execute_input":"2024-04-21T06:29:00.039128Z","iopub.status.idle":"2024-04-21T06:29:00.350991Z","shell.execute_reply.started":"2024-04-21T06:29:00.039103Z","shell.execute_reply":"2024-04-21T06:29:00.350106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Loading Images from Test directory**","metadata":{}},{"cell_type":"code","source":"image_path = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00018.png\"\n\n# Load the image\nimage = mpimg.imread(image_path)\n\n# Display the image\nplt.imshow(image)\nplt.axis('off')  # Turn off axis\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:00.352289Z","iopub.execute_input":"2024-04-21T06:29:00.352861Z","iopub.status.idle":"2024-04-21T06:29:00.606323Z","shell.execute_reply.started":"2024-04-21T06:29:00.352826Z","shell.execute_reply":"2024-04-21T06:29:00.605284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = \"/kaggle/input/image-matching-challenge-2024/test/church/images/00026.png\"\n\n# Load the image\nimage = mpimg.imread(image_path)\n\n# Display the image\nplt.imshow(image)\nplt.axis('off')  # Turn off axis\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:00.615988Z","iopub.execute_input":"2024-04-21T06:29:00.616603Z","iopub.status.idle":"2024-04-21T06:29:00.925134Z","shell.execute_reply.started":"2024-04-21T06:29:00.616544Z","shell.execute_reply":"2024-04-21T06:29:00.924243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Applying Prefetch and Cache for performance enhancement**","metadata":{}},{"cell_type":"markdown","source":"# 1. **Prefetch**","metadata":{}},{"cell_type":"markdown","source":"You can add prefetch with buffer_size=1 and buffer_size=2 and even buffer_size=tf.data.experimental.AUTOTUNE, with applying these buffer sizes, your time can change and performance can maybe decrease or increase.","metadata":{}},{"cell_type":"code","source":"def process_image(file_path, label):\n    img = tf.io.read_file(file_path)\n    img = tf.image.decode_jpeg(img, channels=3) \n    img = tf.image.resize(img, [128, 128])  \n    img = img / 255.0\n    return img, label","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:00.926427Z","iopub.execute_input":"2024-04-21T06:29:00.926875Z","iopub.status.idle":"2024-04-21T06:29:00.932745Z","shell.execute_reply.started":"2024-04-21T06:29:00.926837Z","shell.execute_reply":"2024-04-21T06:29:00.931663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = train_images.take(train_size).map(lambda x: process_image(x, get_label(x)), num_parallel_calls=tf.data.experimental.AUTOTUNE)\ntest_ds = train_images.skip(train_size).map(lambda x: process_image(x, get_label(x)), num_parallel_calls=tf.data.experimental.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:00.933876Z","iopub.execute_input":"2024-04-21T06:29:00.934137Z","iopub.status.idle":"2024-04-21T06:29:01.129136Z","shell.execute_reply.started":"2024-04-21T06:29:00.934114Z","shell.execute_reply":"2024-04-21T06:29:01.128375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Before prefetching\nstart_time = time.time()\n\n# Define and preprocess the dataset without prefetching\ntrain_ds = train_images.take(train_size).map(lambda x: process_image(x, get_label(x)))","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.130373Z","iopub.execute_input":"2024-04-21T06:29:01.130953Z","iopub.status.idle":"2024-04-21T06:29:01.186135Z","shell.execute_reply.started":"2024-04-21T06:29:01.130919Z","shell.execute_reply":"2024-04-21T06:29:01.185488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply prefetching\ntrain_ds = train_ds.prefetch(buffer_size=1)\n\n# Calculate time taken\ntime_after_prefetch1 = time.time() - start_time","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.187453Z","iopub.execute_input":"2024-04-21T06:29:01.188181Z","iopub.status.idle":"2024-04-21T06:29:01.193689Z","shell.execute_reply.started":"2024-04-21T06:29:01.188147Z","shell.execute_reply":"2024-04-21T06:29:01.192775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Time taken after prefetching buffer_size 1:\", time_after_prefetch1)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.194854Z","iopub.execute_input":"2024-04-21T06:29:01.195201Z","iopub.status.idle":"2024-04-21T06:29:01.203459Z","shell.execute_reply.started":"2024-04-21T06:29:01.195170Z","shell.execute_reply":"2024-04-21T06:29:01.202561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply prefetching\ntrain_ds = train_ds.prefetch(buffer_size=2)\n\n# Calculate time taken\ntime_after_prefetch2 = time.time() - start_time","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.204470Z","iopub.execute_input":"2024-04-21T06:29:01.204773Z","iopub.status.idle":"2024-04-21T06:29:01.214978Z","shell.execute_reply.started":"2024-04-21T06:29:01.204749Z","shell.execute_reply":"2024-04-21T06:29:01.214250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Time taken after prefetching buffer size 2:\", time_after_prefetch2)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.216197Z","iopub.execute_input":"2024-04-21T06:29:01.216465Z","iopub.status.idle":"2024-04-21T06:29:01.224414Z","shell.execute_reply.started":"2024-04-21T06:29:01.216442Z","shell.execute_reply":"2024-04-21T06:29:01.223587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply prefetching\ntrain_ds = train_ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n# Calculate time taken\ntime_after_prefetch_AUTOTUNE = time.time() - start_time","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.225276Z","iopub.execute_input":"2024-04-21T06:29:01.225539Z","iopub.status.idle":"2024-04-21T06:29:01.235676Z","shell.execute_reply.started":"2024-04-21T06:29:01.225517Z","shell.execute_reply":"2024-04-21T06:29:01.234707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Time taken after prefetching buffer size AUTOTUNE:\", time_after_prefetch_AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.236880Z","iopub.execute_input":"2024-04-21T06:29:01.237207Z","iopub.status.idle":"2024-04-21T06:29:01.244628Z","shell.execute_reply.started":"2024-04-21T06:29:01.237178Z","shell.execute_reply":"2024-04-21T06:29:01.243698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. **Cache**","metadata":{}},{"cell_type":"code","source":"# Before caching\nstart_time = time.time()\n\n# Define and preprocess the dataset without prefetching and caching\ntrain_ds_no_cache = train_images.take(train_size).map(lambda x: process_image(x, get_label(x)), num_parallel_calls=tf.data.experimental.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.245693Z","iopub.execute_input":"2024-04-21T06:29:01.246073Z","iopub.status.idle":"2024-04-21T06:29:01.304328Z","shell.execute_reply.started":"2024-04-21T06:29:01.246044Z","shell.execute_reply":"2024-04-21T06:29:01.303676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate time taken\ntime_before_cache = time.time() - start_time\n\n# Apply caching\ntrain_ds_cached = train_ds_no_cache.cache()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.305285Z","iopub.execute_input":"2024-04-21T06:29:01.305533Z","iopub.status.idle":"2024-04-21T06:29:01.310631Z","shell.execute_reply.started":"2024-04-21T06:29:01.305511Z","shell.execute_reply":"2024-04-21T06:29:01.309808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate time taken\ntime_after_cache = time.time() - start_time","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.311962Z","iopub.execute_input":"2024-04-21T06:29:01.312220Z","iopub.status.idle":"2024-04-21T06:29:01.319610Z","shell.execute_reply.started":"2024-04-21T06:29:01.312198Z","shell.execute_reply":"2024-04-21T06:29:01.318667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Time taken before caching:\", time_before_cache)\nprint(\"Time taken after caching:\", time_after_cache)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:29:01.320457Z","iopub.execute_input":"2024-04-21T06:29:01.320706Z","iopub.status.idle":"2024-04-21T06:29:01.329880Z","shell.execute_reply.started":"2024-04-21T06:29:01.320685Z","shell.execute_reply":"2024-04-21T06:29:01.328990Z"},"trusted":true},"execution_count":null,"outputs":[]}]}