{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":19991,"databundleVersionId":1117522,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2023-11-20T01:28:53.134509Z","iopub.execute_input":"2023-11-20T01:28:53.134733Z","iopub.status.idle":"2023-11-20T01:28:53.141748Z","shell.execute_reply.started":"2023-11-20T01:28:53.134707Z","shell.execute_reply":"2023-11-20T01:28:53.141119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math, re, os\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\n# from tensorflow.keras.applications.efficientnet import EfficientNetB3\nfrom tensorflow.keras.applications.efficientnet_v2 import EfficientNetV2B0, EfficientNetV2B3, EfficientNetV2S\n# from tf.keras.applications.efficientnet import EfficientNetB3\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nprint(\"hi\")\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-11-25T19:39:26.602802Z","iopub.execute_input":"2023-11-25T19:39:26.603560Z","iopub.status.idle":"2023-11-25T19:39:43.400951Z","shell.execute_reply.started":"2023-11-25T19:39:26.603525Z","shell.execute_reply":"2023-11-25T19:39:43.399897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"hi\")\n\ntf.random.set_seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:43.402372Z","iopub.execute_input":"2023-11-25T19:39:43.402815Z","iopub.status.idle":"2023-11-25T19:39:43.406934Z","shell.execute_reply.started":"2023-11-25T19:39:43.402786Z","shell.execute_reply":"2023-11-25T19:39:43.406319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/working/hi.txt', 'w') as file:\n    file.write(\"hi\")\n    file.close()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:43.407982Z","iopub.execute_input":"2023-11-25T19:39:43.408235Z","iopub.status.idle":"2023-11-25T19:39:43.432052Z","shell.execute_reply.started":"2023-11-25T19:39:43.408209Z","shell.execute_reply":"2023-11-25T19:39:43.431408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LSB","metadata":{}},{"cell_type":"code","source":"# s = \"AB\"\n# b = bytearray()\n# b.extend(s.encode())\n# print(b)\n# print(b[0] & 1)\n\n# binary_data_string = ''.join(format(ord(char), '08b') for char in s)\n\n# print(format(ord(s[0]), '08b'))\n# print(binary_data_string)\n\n# from PIL import Image\n# img = Image.open('/kaggle/input/alaska2-image-steganalysis/Cover/00001.jpg')\n# pixels = list(img.getdata())\n\n# print(img.size)\n# # print(img.)\n#                # Encode data into the least significant bit of each pixel\n# encoded_pixels = []\n# # w, h = img.size\n# pixel_index = 0\n# num_bits_per_channel = 2\n# data_index = 0\n\n# def print_binary(char):\n#     print(format(ord(string), '08b'))\n\n# while data_index < len(binary_data_string):\n#     pixel = pixels[pixel_index]\n#     new_pixel = list(pixel)\n#     print(\"orig: \", new_pixel)\n#     for channel in range (3):\n#         if data_index >= len(binary_data_string):\n#             break;\n#         print(f'channel {channel}')\n#         print(f\"orig: {format(new_pixel[channel], 'b')}\")\n#         new_pixel[channel] = pixel[channel] & ~(2**num_bits_per_channel - 1)\n#         print(f\"masked: {format(new_pixel[channel], 'b')}\")\n#         for j in range(num_bits_per_channel):\n#             if data_index >= len(binary_data_string):\n#                 break;\n#             new_pixel[channel] |= (int(binary_data_string[data_index]) << j)\n#             print(f'adding bit {int(binary_data_string[data_index])} to position {j}')\n#             data_index += 1\n#         print(f\"final: {format(new_pixel[channel], 'b')}\")\n#     pixels[pixel_index] = tuple(new_pixel)\n#     pixel_index += 1\n#     print(\"new: \", new_pixel);\n#     print(\"-\" * 10)\n\n# print(pixels[:3])\n    \n# encoded_image_path = '/kaggle/working/test.jpg'\n# encoded_img = Image.new(img.mode, img.size)\n# encoded_img.putdata(pixels)\n# encoded_img.save(encoded_image_path, quality=100)\n# print(f'saved at {encoded_image_path}')\n# encoded_img = Image.open(encoded_image_path)\n# pixels = list(encoded_img.getdata())\n    \n# print(pixels[:3])\n    \n# recovered = ''\n# pixel_index = 0\n# data_index = 0\n\n# while data_index < len(binary_data_string):\n#     pixel = pixels[pixel_index]\n#     for channel in range(3):\n#         if data_index >= len(binary_data_string):\n#             break;\n#         for shift in range (num_bits_per_channel): # if num_bits_per_channel is 2 -> range is 1, 0\n#             if data_index >= len(binary_data_string):\n#                 break;\n#             bit = (pixel[channel] & (1 << shift)) >> shift\n#             recovered += str(bit)\n#             data_index += 1\n# print(binary_data_string)\n# print(recovered)\n# print(binary_data_string == recovered)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:43.433722Z","iopub.execute_input":"2023-11-25T19:39:43.433994Z","iopub.status.idle":"2023-11-25T19:39:43.452013Z","shell.execute_reply.started":"2023-11-25T19:39:43.433938Z","shell.execute_reply":"2023-11-25T19:39:43.451385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n\n# def encode_lsb(image_path, data, num_bits_per_channel):\n#     # Open the image\n#     img = Image.open(image_path)\n#     pixels = list(img.getdata())\n    \n#     # Convert data to binary\n#     binary_data_string = ''.join(format(ord(char), '08b') for char in data)\n    \n#     # Encode data into the least significant bit of each pixel\n#     encoded_pixels = []\n#     w, h = img.size\n#     pixel_index = 0\n\n#     for i in range(len(binary_data_string)):\n#         pixel = pixels[pixel_index]\n#         new_pixel = list(pixel)\n#         for i in range (3):\n#             if i >= len(binary_data_string):\n#                 break;\n#             new_pixel[i] = pixel[i] & ~(2**num_bits_per_channel - 1)\n#             for j in num_bits_per_channel:\n#                 new_pixel[i] |= (int(binary_data[data_index]) << j)\n#                 data_index += 1\n            \n    \n#     for pixel in pixels:\n#         new_pixel = list(pixel)\n#         for i in range(3):  # Loop through RGB channels\n            \n#         encoded_pixels.append(tuple(new_pixel))\n    \n#     # Create a new image with the encoded pixels\n#     encoded_img = Image.new(img.mode, img.size)\n#     encoded_img.putdata(encoded_pixels)\n#     encoded_img.save('/kaggle/working/encoded_image.jpg')\n\n# def decode_lsb(encoded_image_path, data_len):\n#     # Open the encoded image\n#     encoded_img = Image.open(encoded_image_path)\n#     encoded_pixels = list(encoded_img.getdata())\n    \n#     # Extract the LSB from each pixel to retrieve the hidden data\n#     binary_data = ''\n#     for pixel in encoded_pixels:\n#         for i in range(3):  # Loop through RGB channels\n#             binary_data += str(pixel[i] & 1)\n\n#     # Convert binary data back to string\n#     decoded_data = ''.join([chr(int(binary_data[i:i+8], 2)) for i in range(0, len(binary_data), 8)])\n#     return decoded_data\n\n# # Example usage\n# data_to_hide = \"Hello, LSB!\"\n# encode_lsb('/kaggle/input/alaska2-image-steganalysis/Cover/00001.jpg', data_to_hide)\n# decoded_data = decode_lsb('/kaggle/working/encoded_image.jpg')\n# print(\"Decoded Data:\", decoded_data)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:43.453048Z","iopub.execute_input":"2023-11-25T19:39:43.453283Z","iopub.status.idle":"2023-11-25T19:39:43.515503Z","shell.execute_reply.started":"2023-11-25T19:39:43.453258Z","shell.execute_reply":"2023-11-25T19:39:43.514892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TPU Strategy and other configs ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nprint('hi')\n\n# Detect and initialize TPU\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()  # Default strategy for CPU and single GPU\n\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:43.516414Z","iopub.execute_input":"2023-11-25T19:39:43.516862Z","iopub.status.idle":"2023-11-25T19:39:52.112438Z","shell.execute_reply.started":"2023-11-25T19:39:43.516833Z","shell.execute_reply":"2023-11-25T19:39:52.111641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load label and paths","metadata":{}},{"cell_type":"code","source":"# !rm -r /kaggle/input/alaska2-image-steganalysis/Test\n# !rm /kaggle/input/alaska2-image-steganalysis/sample_submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:39:52.113357Z","iopub.execute_input":"2023-11-25T19:39:52.113604Z","iopub.status.idle":"2023-11-25T19:39:52.116870Z","shell.execute_reply.started":"2023-11-25T19:39:52.113576Z","shell.execute_reply":"2023-11-25T19:39:52.116173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport os\n\n# Constants\nbase_dir = '/kaggle/input/alaska2-image-steganalysis/'\nclass_names = ['Cover', 'JMiPOD', 'UERD', 'JUNIWARD']\n\nimage_size = (512, 512)\nbatch_size = 32 * strategy.num_replicas_in_sync\naugmented_batch_size = 4\n        \ndef get_paths(split_at, end_at, multiclass=False):\n    train_data_paths = []\n    val_data_paths = []\n    train_labels = []\n    val_labels = []\n    \n    for class_name in class_names:\n        class_dir = os.path.join(base_dir, class_name)\n        all_images = os.listdir(class_dir) \n\n        selected_images = all_images[:end_at]  \n\n        for img in selected_images[:split_at]:\n            train_data_paths.append(os.path.join(class_dir, img))\n            if multiclass:\n                train_labels.append(tf.one_hot(class_names.index(class_name), len(class_names)))\n            else:\n                train_labels.append(0 if class_name == 'Cover' else 1)\n\n        for img in selected_images[split_at:end_at]:\n            val_data_paths.append(os.path.join(class_dir, img))\n            if multiclass:\n                val_labels.append(tf.one_hot(class_names.index(class_name), len(class_names)))\n            else:\n                val_labels.append(0 if class_name == 'Cover' else 1)\n    \n    return train_data_paths, val_data_paths, train_labels, val_labels\n\nsplit_at = 16000\nend_at = 20000\n\ntrain_data_paths, val_data_paths, train_labels, val_labels = get_paths(split_at, end_at)\n\ntrain_len = len(train_data_paths)\nval_len = len(val_data_paths)\n\npreprocessor = tf.keras.applications.resnet50.preprocess_input\npreprocess = False # If preprocessing is needed, EfficientNetV2 has a preprocessing layer doesn't need it\n    \n# Function to load and preprocess an image\ndef load_and_preprocess_image(img_path):\n    img = tf.io.read_file(img_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32)\n    if preprocess:\n        img = preprocessor(img)  # Resnet preprocessing\n        img = img / 255.0\n    return img\n\n# Function to apply data augmentation to an image\ndef augment_image(img):\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_flip_up_down(img)\n    # Add more augmentation techniques as needed\n    return img\n\n# Function to create a batch of augmented images\ndef create_augmented_batch(img):\n    augmented_images = [augment_image(img) for _ in range(augmented_batch_size)]  # Create 4 augmented images\n    return tf.stack(augmented_images)\n\n# for x, y in train_data_paths:\n#     print(x, y)\n#     load_and_preprocess_image(x, y)\n\n# Create datasets inside strategy.scope()\nwith strategy.scope():\n    train_dataset = tf.data.Dataset.from_tensor_slices((train_data_paths, train_labels))    \n    train_dataset = train_dataset.map(lambda img_path, label: (load_and_preprocess_image(img_path), label))\n    #     train_dataset = train_dataset.flat_map(lambda img, label: tf.data.Dataset.from_tensor_slices(create_augmented_batch(img)))\n    train_dataset = train_dataset.shuffle(split_at)  # Shuffle the dataset\n    train_dataset = train_dataset.batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\n\n    val_dataset = tf.data.Dataset.from_tensor_slices((val_data_paths, val_labels))\n    val_dataset = val_dataset.map(lambda img_path, label: (load_and_preprocess_image(img_path), label))\n    val_dataset = val_dataset.shuffle(end_at - split_at).batch(batch_size).prefetch(buffer_size=tf.data.AUTOTUNE)\n\n    # Check the dataset shapes\n    for image, label in train_dataset.take(1):\n        print(\"Training Batch - Image Shape:\", image.shape, \"Label:\", label)\n        print(tf.math.reduce_max(image))\n        print(tf.math.reduce_min(image))        \n    for image, label in val_dataset.take(1):\n        print(\"Validation Batch - Image Shape:\", image.shape, \"Label:\", label)\n        print(tf.math.reduce_max(image))\n        print(tf.math.reduce_min(image)) \n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:51:21.036301Z","iopub.execute_input":"2023-11-25T19:51:21.036732Z","iopub.status.idle":"2023-11-25T19:51:35.847563Z","shell.execute_reply.started":"2023-11-25T19:51:21.036696Z","shell.execute_reply":"2023-11-25T19:51:35.846354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import tensorflow as tf\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# # Define the base directory containing the 'Cover', 'JMiPOD', 'UERD', and 'JUNIWARD' folders\n# base_dir = '/kaggle/input/alaska2-image-steganalysis'\n\n# # Specify image size and batch size\n# img_size = (512, 512)\n# batch_size = 32 * strategy.num_replicas_in_sync\n\n# # Configuration\n# # epochs = 20\n\n# # Define data augmentation parameters\n# datagen = ImageDataGenerator(\n# #     rescale=1./255,\n#     horizontal_flip=True,\n#     vertical_flip=True,\n#     validation_split=0.2  # 20% of the data will be used for validation\n# )\n\n# with strategy.scope():\n\n#     print(\"starting train gen\")\n#     # Create a generator for training data\n#     train_generator = datagen.flow_from_directory(\n#         base_dir,\n#         target_size=img_size,\n#         batch_size=batch_size,\n#         classes=['Cover', 'JMiPOD', 'JUNIWARD', 'UERD'],\n#         class_mode='categorical',\n#         subset='training',  # Specify that this is the training subset\n#     )\n    \n#     # Create a generator for validation data\n#     validation_generator = datagen.flow_from_directory(\n#         base_dir,\n#         target_size=img_size,\n#         batch_size=batch_size,\n#         classes=['Cover', 'JMiPOD', 'JUNIWARD', 'UERD'],\n#         class_mode='categorical',\n#         subset='validation',  # Specify that this is the validation subset\n#     )\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:51:42.042528Z","iopub.execute_input":"2023-11-25T19:51:42.042977Z","iopub.status.idle":"2023-11-25T19:51:42.048405Z","shell.execute_reply.started":"2023-11-25T19:51:42.042927Z","shell.execute_reply":"2023-11-25T19:51:42.047393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def generate_dummy_image(shape=(512, 512, 3)):\n#     return np.random.rand(*shape).astype('float32') / 255.0\n\n# def generate_dummy_label(num_classes=10):\n#     return np.random.randint(num_classes, size=1)[0]\n\n# def create_dummy_dataset(num_samples, num_classes=10, batch_size=32):\n#     images = [generate_dummy_image() for _ in range(num_samples)]\n#     labels = [generate_dummy_label(num_classes) for _ in range(num_samples)]\n#     labels_one_hot = [tf.one_hot(label, num_classes) for label in labels]\n    \n#     dataset = tf.data.Dataset.from_tensor_slices((images, labels_one_hot))\n#     dataset = dataset.shuffle(buffer_size=num_samples)\n#     dataset = dataset.batch(batch_size)\n    \n#     return dataset\n\n# # Example usage:\n# train_samples = 128\n# validation_samples = 64\n# num_classes = 4\n# batch_size = 32\n\n# with strategy.scope():\n\n#     train_generator = create_dummy_dataset(train_samples, num_classes, batch_size)\n#     validation_generator = create_dummy_dataset(validation_samples, num_classes, batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:40:17.358857Z","iopub.execute_input":"2023-11-25T19:40:17.359123Z","iopub.status.idle":"2023-11-25T19:40:17.373854Z","shell.execute_reply.started":"2023-11-25T19:40:17.359096Z","shell.execute_reply":"2023-11-25T19:40:17.373116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images, labels = next(train_generator)\n\n# print(images.shape)\n# print(images[0])\n# print(labels)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:40:17.374802Z","iopub.execute_input":"2023-11-25T19:40:17.375062Z","iopub.status.idle":"2023-11-25T19:40:17.387290Z","shell.execute_reply.started":"2023-11-25T19:40:17.375036Z","shell.execute_reply":"2023-11-25T19:40:17.386636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleAttention(tf.keras.layers.Layer):\n    def __init__(self, **kwargs):\n        super(SimpleAttention, self).__init__(**kwargs)\n    \n    def build(self, input_shape):\n        # Trainable weight variable\n        self.attention_weights = self.add_weight(\n            name='attention_weights',\n            shape=(1280,), # should be 1280 for EfficientNetV2-S\n            initializer='glorot_uniform',\n            trainable=True\n        )\n    \n    def call(self, x):\n        # Compute attention scores:\n        # each feature map (1280 feature maps) multipled by corresponding attention weight, \n        # then reduced (summed) to one feature map\n        # (16, 16, 1280) -> (16, 16, 1)\n        attention_scores = tf.reduce_sum(x * self.attention_weights,axis=-1, keepdims=True)\n        \n        # Apply softmax (so that weights sum to 1)\n        softmax_attention_scores = tf.nn.softmax(attention_scores, axis=-1)\n        \n        # Apply attention weights to input\n        attended_input = x * softmax_attention_scores\n        \n        # Sum along feature maps dimention\n        output = tf.reduce_sum(attended_input, axis=-1, keepdims=True)                \n        \n        return output\n    \n    def compute_output_shape(self, input_shape):\n        return input_shape[:-1] + (1,)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:40:17.388163Z","iopub.execute_input":"2023-11-25T19:40:17.388390Z","iopub.status.idle":"2023-11-25T19:40:17.398253Z","shell.execute_reply.started":"2023-11-25T19:40:17.388366Z","shell.execute_reply":"2023-11-25T19:40:17.397601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base_model = EfficientNetV2(weights='imagenet', include_top=False, input_shape=(512, 512, 3))\nfrom tensorflow.keras.applications.efficientnet_v2 import EfficientNetV2B0, EfficientNetV2B3, EfficientNetV2S\nfrom tensorflow.keras.applications.resnet50 import ResNet50\n\nwith strategy.scope():\n\n#     tf.keras.applications.efficientnet_v2.EfficientNetV2B3(\n#         include_top=False,\n#         weights='imagenet',\n#         input_tensor=None,\n#         input_shape=None,\n#         pooling=None,\n#     #     classes=1000,\n#         classifier_activation='softmax',\n# #         include_preprocessing=True # should rescale for us\n#     )\n    \n    # Image preprocessing (normalization) included in implementation\n    # https://www.tensorflow.org/api_docs/python/tf/keras/applications/efficientnet/preprocess_input \n    base_model = EfficientNetV2S(\n        include_top=False,\n        weights='imagenet',\n#         input_tensor=None,\n        input_shape=(512,512, 3),\n#         pooling=None,\n#         classes=num_classes,\n#         classifier_activation='softmax',\n        include_preprocessing=True \n    )\n    \n#     base_model = ResNet50(\n#         include_top=False,\n#         weights=None,\n# #         input_tensor=None,\n#         input_shape=(512, 512, 3),\n# #         pooling='avg',\n# #         classes=4,\n#     )\n\n    # If we want to fine-tune\n#     for layer in base_model.layers:\n#         layer.trainable = False   \n#     model_name = 'resnet50'\n    model_name = 'efficientnetv2_s'\n    pretrained = True    \n    num_classes = 4\n    \n    model = tf.keras.Sequential([\n        base_model,\n        L.GlobalAveragePooling2D(),\n        L.Dropout(0.2),\n        L.Dense(1, activation='softmax')\n#         tf.keras.layers.Dense(num_classes, activation='softmax')\n#         SimpleAttention(),\n#         L.Flatten(),\n#         L.Dense(units=256), # one for each pixel (16 x 16)\n#         L.Dropout(0.5),\n#         L.Dense(num_classes, activation='softmax') \n    ])\n\n\n    print(model.summary())\n\n    model.compile(optimizer='adam',\n                  loss='binary_crossentropy',\n                  metrics=['accuracy'])\n#     model.compile(optimizer='adam',\n#                   loss='categorical_crossentropy',\n#                   metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:53:42.160835Z","iopub.execute_input":"2023-11-25T19:53:42.161311Z","iopub.status.idle":"2023-11-25T19:54:07.953040Z","shell.execute_reply.started":"2023-11-25T19:53:42.161271Z","shell.execute_reply":"2023-11-25T19:54:07.951942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nfrom tensorflow.keras.callbacks import Callback\n\nclass SaveHistoryCallback(Callback):\n    def __init__(self, filename=f'/kaggle/working/{model_name}' + ('.pretrained' if pretrained else '') + f'.train-n-{train_len}.running_history.pkl'):\n        super(SaveHistoryCallback, self).__init__()\n        self.filename = filename\n        self.history = {\n            \"loss\": [],\n            \"accuracy\": [],\n            \"val_loss\": [],\n            \"val_accuracy\": []\n        }\n    \n    def on_epoch_end(self, epoch, logs=None):\n#         current_epoch_data = {\n#             'epoch': epoch,\n#             'logs': logs.copy() if logs else None\n#         }        \n        if logs is not None:\n            # Access training metrics for the current epoch\n            loss = logs.get('loss')\n            accuracy = logs.get('accuracy')\n            \n            # Access validation metrics if available\n            val_loss = logs.get('val_loss')\n            val_accuracy = logs.get('val_accuracy')\n\n            self.history[\"loss\"].append(loss)\n            self.history[\"accuracy\"].append(accuracy)\n            self.history[\"val_loss\"].append(val_loss)\n            self.history[\"val_accuracy\"].append(val_accuracy)\n\n#             with open(self.filename, 'w') as file:\n#                 json.dump(self.history, file)\n#                 print(f'saved to {self.filename}')\n            with open(self.filename, 'wb') as file:\n                print(f'saved history to {self.filename}')\n                pickle.dump(self.history, file)\n        else:\n            print('logs unexpectedly none in SaveHistoryCallback')\n\n#             # Custom actions based on metrics\n#             print(f' Epoch {epoch + 1}: Val Loss={val_loss:.4f}, Val Accuracy={val_accuracy:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:54:20.419949Z","iopub.execute_input":"2023-11-25T19:54:20.420313Z","iopub.status.idle":"2023-11-25T19:54:20.428375Z","shell.execute_reply.started":"2023-11-25T19:54:20.420279Z","shell.execute_reply":"2023-11-25T19:54:20.427532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\nnum_epochs = 10\n\nwith strategy.scope():\n#     checkpoint = ModelCheckpoint('/kaggle/input/weights.{epoch:02d}-{val_loss:.2f}.h5', save_freq=1, monitor='val_loss', save_best_only=True)\n    \n    checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n        filepath='/kaggle/working/' + model_name + ('.pretrained' if pretrained else '') + '.train-n-' + str(train_len) + '.weights.{epoch:02d}-{val_accuracy:.4f}.h5',\n        save_weights_only=True,\n        monitor='val_accuracy',\n        mode='max',\n        save_best_only=True\n    )\n    \n    history = model.fit(\n        train_dataset,\n        steps_per_epoch=train_len // batch_size,\n        epochs=num_epochs,\n        validation_data=val_dataset,\n        validation_steps=val_len // batch_size,\n        callbacks=[checkpoint_callback, SaveHistoryCallback()]\n    )\n    \n    # Dummy model fitting\n#     history = model.fit(\n#         train_generator,\n#         steps_per_epoch=train_samples // batch_size,\n#         epochs=num_epochs,\n#         validation_data=validation_generator,\n#         validation_steps=validation_samples // batch_size,\n#         callbacks=[checkpoint, SaveHistoryCallback()]\n#     )\n    \n    # Save the training history to a file\n#     import pickle\n\n    with open(f'/kaggle/working/{model_name}' + ('.pretrained' if pretrained else '') + f'.train-n-{train_len}.final_history.pkl', 'wb') as file:\n        pickle.dump(history.history, file)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:54:21.782884Z","iopub.execute_input":"2023-11-25T19:54:21.783250Z","iopub.status.idle":"2023-11-25T20:32:48.709127Z","shell.execute_reply.started":"2023-11-25T19:54:21.783219Z","shell.execute_reply":"2023-11-25T20:32:48.707491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for images, labels in train_dataset.take(1):\n    print(model.predict(images))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T19:52:30.809313Z","iopub.execute_input":"2023-11-25T19:52:30.809723Z","iopub.status.idle":"2023-11-25T19:52:56.008436Z","shell.execute_reply.started":"2023-11-25T19:52:30.809686Z","shell.execute_reply":"2023-11-25T19:52:56.007176Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(f'/kaggle/working/{model_name}.{'pretrained' if pretrained else ''}_model.keras')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:06:24.167947Z","iopub.execute_input":"2023-11-22T07:06:24.168797Z","iopub.status.idle":"2023-11-22T07:06:27.200738Z","shell.execute_reply.started":"2023-11-22T07:06:24.168750Z","shell.execute_reply":"2023-11-22T07:06:27.199512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loads the weights\nload_weights = True\nload_path = '/kaggle/working/efficientnetv2_s.pretrained.train-n-32000.weights.04-0.7500.h5'\n\nif load_weights:\n    model.load_weights(load_path)\n    print(f'loaded weights from {load_path}')\n\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc, ConfusionMatrixDisplay\n\n# y_true = np.empty()\n# y_preds = np.empty()\n\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport matplotlib.pyplot as plt\n\n# Initialize empty lists for true labels and predicted probabilities\ny_true = []\ny_pred_prob = []\n\n# Iterate over the validation dataset to get true labels and predicted probabilities\nfor images, labels in val_dataset:    \n    y_true.append(labels.numpy()) \n    y_pred_prob.append(model.predict(images))\n\n# Concatenate the lists to obtain numpy arrays\ny_true = np.concatenate(y_true, axis=0)\ny_pred_prob = np.concatenate(y_pred_prob, axis=0)\n\n# Convert predicted probabilities to binary predictions (0 or 1)\ny_pred = np.round(y_pred_prob)\n\nprint(y_true)\nprint(y_pred)\n\n# Confusion Matrix\nconf_matrix = confusion_matrix(y_true, y_pred)\n\n# Plot Confusion Matrix\nplt.imshow(conf_matrix, interpolation='nearest', cmap=plt.cm.Blues)\nplt.title('Confusion Matrix')\nplt.colorbar()\nclasses = ['Class 0', 'Class 1']  # Replace with your class names\ntick_marks = np.arange(len(classes))\nplt.xticks(tick_marks, classes)\nplt.yticks(tick_marks, classes)\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.show()\n\n# ROC Curve\nfpr, tpr, thresholds = roc_curve(y_true, y_pred_prob)\nroc_auc = auc(fpr, tpr)\n\n# Plot ROC Curve\nplt.figure()\nplt.plot(fpr, tpr, label=f'AUC = {roc_auc:.2f}')\nplt.plot([0, 1], [0, 1], 'k--', lw=2)\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Curve')\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T20:45:34.243823Z","iopub.execute_input":"2023-11-25T20:45:34.244296Z","iopub.status.idle":"2023-11-25T20:46:35.363510Z","shell.execute_reply.started":"2023-11-25T20:45:34.244257Z","shell.execute_reply":"2023-11-25T20:46:35.362273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_name = 'resnet50'\n# train_len = 100\n# pretrained = True\n# print('/kaggle/working/' + model_name + '.train-n-' + str(train_len) + '.weights.{epoch:02d}-{val_accuracy:.4f}.h5')\n# print('/kaggle/working/' + model_name + ('.pretrained' if pretrained else '') + '.train-n-' + str(train_len) + '.weights.{epoch:02d}-{val_accuracy:.4f}.h5')\n# print(f'/kaggle/working/{model_name}.train-n-{train_len}.final_history.pkl')\n# print(f'/kaggle/working/{model_name}.train-n-{train_len}.running_history.json')","metadata":{"execution":{"iopub.status.busy":"2023-11-23T01:55:16.134785Z","iopub.execute_input":"2023-11-23T01:55:16.135137Z","iopub.status.idle":"2023-11-23T01:55:16.142109Z","shell.execute_reply.started":"2023-11-23T01:55:16.135108Z","shell.execute_reply":"2023-11-23T01:55:16.140837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, auc, ConfusionMatrixDisplay, confusion_matrix\n\n# Accuracy and Loss Graphs\n# history = model.fit(train_data, epochs=num_epochs, validation_data=validation_data)\nimport pickle\n\nnum_epochs = 8\n\nrunning_history = True\n# running_history_path = f'/kaggle/working/{model_name}' + ('.pretrained' if pretrained else '') + f'.train-n-{train_len}.' + ('running_history.pkl' if running_history else 'final_history.pkl')\nrunning_history_path = '/kaggle/working/efficientnetv2_s.pretrained.train-n-32000.running_history.pkl'\n# running_history_path = '/kaggle/working/resnet50.pretrained.train-n-32000.running_history.pkl'\n\nwith open(running_history_path, 'rb') as file:\n    history = pickle.load(file)\n# # else:\n# #     history = history.history\n\nprint(history)\n\n\nplt.figure()\nplt.plot(range(1, num_epochs + 1), history['accuracy'], label='Training Accuracy')\nplt.plot(range(1, num_epochs + 1), history['val_accuracy'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\nplt.figure()\nplt.plot(range(1, num_epochs + 1), history['loss'], label='Training Loss')\nplt.plot(range(1, num_epochs + 1), history['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T20:50:20.938422Z","iopub.execute_input":"2023-11-25T20:50:20.938857Z","iopub.status.idle":"2023-11-25T20:50:21.294524Z","shell.execute_reply.started":"2023-11-25T20:50:20.938820Z","shell.execute_reply":"2023-11-25T20:50:21.293464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"markdown","source":"Unhide below to see helper function `display_training_curves`:","metadata":{}},{"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-20T01:46:34.705724Z","iopub.status.idle":"2023-11-20T01:46:34.706066Z","shell.execute_reply.started":"2023-11-20T01:46:34.705897Z","shell.execute_reply":"2023-11-20T01:46:34.705915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(\n    history['loss'], \n    history['val_loss'], \n    'loss', 211)\ndisplay_training_curves(\n    history['accuracy'], \n    history['val_accuracy'], \n    'accuracy', 212)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-20T01:46:34.707394Z","iopub.status.idle":"2023-11-20T01:46:34.707680Z","shell.execute_reply.started":"2023-11-20T01:46:34.707538Z","shell.execute_reply":"2023-11-20T01:46:34.707553Z"},"trusted":true},"execution_count":null,"outputs":[]}]}