{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8033468,"sourceType":"datasetVersion","datasetId":4735360}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport keras\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nfrom keras import layers\n\n\n  \ndirectory = \"/kaggle/input/birdclef-2024-mel-spectrograms/train_images\"\n\nIMG_SIZE = 224\n# Load the dataset from the directory\ndataset = tf.keras.preprocessing.image_dataset_from_directory(\n    directory,\n    label_mode='categorical',\n    image_size=(IMG_SIZE, IMG_SIZE),\n    batch_size=1,\n    shuffle=True,\n    seed=123,\n    validation_split=0.2,\n    color_mode = 'rgb',\n    subset=\"both\",  # Use 'training' or 'validation' as needed\n)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-14T05:22:24.649667Z","iopub.execute_input":"2024-05-14T05:22:24.650358Z","iopub.status.idle":"2024-05-14T05:22:41.348835Z","shell.execute_reply.started":"2024-05-14T05:22:24.650327Z","shell.execute_reply":"2024-05-14T05:22:41.348034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import data as tf_data\nfrom tensorflow import image as tf_image\n\nAUTO = tf_data.AUTOTUNE\nBATCH_SIZE = 32\n\ntrain_ds = dataset[0]  ## For training\ndef preprocess_image(image, label):\n    image = tf_image.convert_image_dtype(image, \"float32\") / 255.0\n    return image, label\n\ntrain_ds = (\n    train_ds\n    .batch(BATCH_SIZE)\n    .shuffle(1024)\n    .map(preprocess_image, num_parallel_calls=AUTO)\n)\n\n\n\ntest_ds = dataset[1]\n\n\n\ntest_ds = (\n    test_ds.map(preprocess_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T05:22:41.350806Z","iopub.execute_input":"2024-05-14T05:22:41.351166Z","iopub.status.idle":"2024-05-14T05:22:41.416396Z","shell.execute_reply.started":"2024-05-14T05:22:41.351134Z","shell.execute_reply":"2024-05-14T05:22:41.415659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_cv\n\n\nrand_augment = keras_cv.layers.RandAugment(\n    value_range=(0, 1),\n    augmentations_per_image=3,\n    magnitude=0.3,\n    magnitude_stddev=0.2,\n    rate=1.0,\n)\n\n\ndef apply_rand_augment(image,label):\n    image = rand_augment(image)\n    return image,label\n\n\ncut_mix = keras_cv.layers.CutMix()\nmix_up = keras_cv.layers.MixUp()\n\n\n\ndef cut_mix_and_mix_up(image, label):\n    # Convert tuple to dictionary\n    samples_dict = {'images': image, 'labels': label}\n    \n    # Apply CutMix\n    mixed_samples = cut_mix(samples_dict)\n    \n    # Apply MixUp\n    mixed_samples = mix_up(mixed_samples)\n    \n    # Convert dictionary back to tuple\n    return (mixed_samples['images'], mixed_samples['labels'])\n\n\n\n\ndef preprocess(image, label):\n    # Squeeze the unnecessary dimension out\n    image = tf.squeeze(image, axis=[1])\n    label = tf.squeeze(label,axis=[1])# Squeeze out the second dimension which is 1\n    return image, label\n\n# Apply the preprocessing function to training and testing datasets\ntrain_ds = train_ds.map(preprocess)\ntest_ds = test_ds.map(preprocess) # Assu\n\n#train_ds = train_ds.map(apply_rand_augment, num_parallel_calls=tf.data.AUTOTUNE)\n\ntrain_ds_cmu = train_ds.map(cut_mix_and_mix_up, num_parallel_calls=tf.data.AUTOTUNE)\n\ntrain_ds = train_ds.concatenate(train_ds_cmu)\n\ntrain_ds = train_ds.shuffle(1000)\n\n#train_ds = train_ds.take(1500)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T05:22:41.417347Z","iopub.execute_input":"2024-05-14T05:22:41.417589Z","iopub.status.idle":"2024-05-14T05:22:47.593046Z","shell.execute_reply.started":"2024-05-14T05:22:41.417567Z","shell.execute_reply":"2024-05-14T05:22:47.592234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_batch, label_batch = next(iter(train_ds))\nplt.figure(figsize=(10, 10))\nfor i in range(9):\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(image_batch[i])\n    plt.show()\n    plt.axis(\"off\")\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-14T05:22:47.595393Z","iopub.execute_input":"2024-05-14T05:22:47.596138Z","iopub.status.idle":"2024-05-14T05:23:29.016816Z","shell.execute_reply.started":"2024-05-14T05:22:47.596103Z","shell.execute_reply":"2024-05-14T05:23:29.015679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using MobileNetV2 ","metadata":{}},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n# from tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.applications import MobileNetV2\n# from tensorflow.keras.applications.mobilenet_v2 import preprocess_input\n# from tensorflow.keras.callbacks import ModelCheckpoint\n# import keras\n# import matplotlib.pyplot as plt\n# import numpy as np\n# from tensorflow.keras.models import Model\n\n\n# from tensorflow.keras import backend as K\n# from tensorflow.keras.mixed_precision import experimental as mixed_precision\n\n# # Enable mixed precision\n# policy = mixed_precision.Policy('mixed_float16')\n# mixed_precision.set_policy(policy)\n# K.clear_session()\n\n# # Assuming the rest of your code sets up the dataset and model\n# batch_size = 32  # Adjust batch size if needed\n# AUTOTUNE = tf.data.AUTOTUNE\n\n# # Setup your data pipeline\n# train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\n# # Setup your data pipeline\n# test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\n# # Clear the Keras session periodically if needed\n\n\n\n# checkpoint = ModelCheckpoint(\n#     'best_model.keras',  # The saved directory will be named 'best_model'\n#     monitor='val_accuracy',\n#     save_best_only=True,\n#     save_weights_only=False,\n#     mode='max',\n#     verbose=1)\n\n# inputs = Input(shape=(224, 224, 3))\n# base_model = MobileNetV2(weights='imagenet', include_top=False, input_tensor=inputs)  # Load pre-trained MobileNetV2\n# x = GlobalAveragePooling2D()(base_model.output)\n# x = Dropout(0.2)(x)\n# x = Dense(256, activation='relu')(x)\n# x = Dropout(0.2)(x)\n# outputs = Dense(182, activation='softmax')(x)\n\n\n# for layer in base_model.layers:\n#     layer.trainable = True\n\n# # Create the model\n# model = Model(inputs=inputs, outputs=outputs)\n\n# import math\n\n# def get_lr_callback(batch_size=32, mode='cos', epochs=25, plot=False):\n#     lr_start, lr_max, lr_min = 5e-5, 8e-6 * batch_size, 1e-5\n#     lr_ramp_ep, lr_sus_ep, lr_decay = 3, 0, 0.75\n\n#     def lrfn(epoch):  # Learning rate update function\n#         if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n#         elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n#         elif mode == 'exp': lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n#         elif mode == 'step': lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n#         elif mode == 'cos':\n#             decay_total_epochs, decay_epoch_index = epochs - lr_ramp_ep - lr_sus_ep + 3, epoch - lr_ramp_ep - lr_sus_ep\n#             phase = math.pi * decay_epoch_index / decay_total_epochs\n#             lr = (lr_max - lr_min) * 0.5 * (1 + math.cos(phase)) + lr_min\n#         return lr\n\n#     if plot:  # Plot lr curve if plot is True\n#         plt.figure(figsize=(10, 5))\n#         plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n#         plt.xlabel('epoch'); plt.ylabel('lr')\n#         plt.title('LR Scheduler')\n#         plt.show()\n\n#     return keras.callbacks.LearningRateScheduler(lrfn, verbose=False)  # Create lr callback\n\n# lr_cb = get_lr_callback(25, plot=True)\n\n# # Compile the model\n# model.compile(optimizer=Adam(learning_rate=0.0003),\n#               loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.02),\n#               metrics=['accuracy'])\n\n\n# ckpt_cb = keras.callbacks.ModelCheckpoint(\"best_model.weights.h5\",\n#                                          monitor='val_auc',\n#                                          save_best_only=True,\n#                                          save_weights_only=True,\n#                                          mode='max')\n\n\n# history = model.fit(\n#     train_ds,\n#     validation_data=test_ds,\n#     epochs=25,  # you can adjust this according to your needs,\n#     callbacks = [lr_cb,checkpoint]\n# )\n\n\n# print(\"Model trained and saved successfully.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T05:23:29.018522Z","iopub.execute_input":"2024-05-14T05:23:29.018949Z","iopub.status.idle":"2024-05-14T05:23:29.027950Z","shell.execute_reply.started":"2024-05-14T05:23:29.018913Z","shell.execute_reply":"2024-05-14T05:23:29.026649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.applications.mobilenet_v2 import preprocess_input\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import mixed_precision\n\n# Enable mixed precision\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_global_policy(policy)\nK.clear_session()\n\n# Assuming the rest of your code sets up the dataset and model\nbatch_size = 32  # Adjust batch size if needed\nAUTOTUNE = tf.data.AUTOTUNE\n\n# Setup your data pipeline\ntrain_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\ntest_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\n# Define the model architecture\ninputs = Input(shape=(224, 224, 3))\nbase_model = MobileNetV2(weights='imagenet', include_top=False, input_tensor=inputs)  # Load pre-trained MobileNetV2\nx = GlobalAveragePooling2D()(base_model.output)\nx = Dropout(0.2)(x)\nx = Dense(256, activation='relu')(x)\nx = Dropout(0.2)(x)\noutputs = Dense(182, activation='softmax')(x)\n\n# Make all layers in the base model trainable\nfor layer in base_model.layers:\n    layer.trainable = True\n\n# Create the model\nmodel = Model(inputs=inputs, outputs=outputs)\n\n# Define a learning rate scheduler\nimport math\n\ndef get_lr_callback(batch_size=32, mode='cos', epochs=50, plot=False):\n    lr_start, lr_max, lr_min = 5e-5, 8e-6 * batch_size, 1e-5\n    lr_ramp_ep, lr_sus_ep, lr_decay = 3, 0, 0.75\n\n    def lrfn(epoch):  # Learning rate update function\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n        elif mode == 'exp':\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        elif mode == 'step':\n            lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n        elif mode == 'cos':\n            decay_total_epochs = epochs - lr_ramp_ep - lr_sus_ep + 3\n            decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            lr = (lr_max - lr_min) * 0.5 * (1 + math.cos(phase)) + lr_min\n        return lr\n\n    if plot:  # Plot lr curve if plot is True\n        plt.figure(figsize=(10, 5))\n        plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('lr')\n        plt.title('LR Scheduler')\n        plt.show()\n\n    return keras.callbacks.LearningRateScheduler(lrfn, verbose=False)  # Create lr callback\n\nlr_cb = get_lr_callback(25, plot=True)\n\n# Compile the model with a mixed precision optimizer\n# base_optimizer = Adam(learning_rate=0.0003)\n# optimizer = mixed_precision.LossScaleOptimizer(base_optimizer, dynamic=True)\n\nmodel.compile(optimizer=Adam(learning_rate=0.0003),\n              loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.02),\n              metrics=['accuracy'])\n\n\n# Define callbacks\ncheckpoint = ModelCheckpoint(\n    'best_model.keras',  # The saved directory will be named 'best_model'\n    monitor='val_accuracy',\n    save_best_only=True,\n    save_weights_only=False,\n    mode='max',\n    verbose=1)\n\nckpt_cb = keras.callbacks.ModelCheckpoint(\n    \"best_model.weights.h5\",\n    monitor='val_auc',\n    save_best_only=True,\n    save_weights_only=True,\n    mode='max')\n\n# Train the model\nhistory = model.fit(\n    train_ds,\n    validation_data=test_ds,\n    epochs=50,  # Adjust this according to your needs\n    callbacks=[lr_cb, checkpoint]\n)\n\nprint(\"Model trained and saved successfully.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T05:26:47.371707Z","iopub.execute_input":"2024-05-14T05:26:47.372336Z","iopub.status.idle":"2024-05-14T05:28:19.671643Z","shell.execute_reply.started":"2024-05-14T05:26:47.372306Z","shell.execute_reply":"2024-05-14T05:28:19.670093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}