{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8212088,"sourceType":"datasetVersion","datasetId":4797817,"isSourceIdPinned":true}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport keras\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport keras_cv\n\n\ntrain_path = \"/kaggle/input/birdclef24-preprocess-data/train_images\"\n\nIMAGE_SIZE = (224,224)\nBATCH_SIZE = 32\nSEED = 123\nEPOCH=60\nLEARNING_RATE=0.00001\n\ntrain_set = tf.keras.utils.image_dataset_from_directory(\n    train_path, \n    validation_split = 0.2,\n    labels='inferred',\n    label_mode='int',\n    subset='training',\n    seed =SEED, \n    image_size = IMAGE_SIZE,\n    batch_size = BATCH_SIZE)\n\nvalidate_set = tf.keras.utils.image_dataset_from_directory(\n    train_path,\n    validation_split = 0.2,\n    labels='inferred',\n    label_mode='int',\n    subset='validation',\n    seed = SEED,\n    image_size= IMAGE_SIZE,\n    batch_size = BATCH_SIZE)\n\nclass_names = sorted(os.listdir('/kaggle/input/birdclef-2024/train_audio/'))\nnum_classes = len(class_names)\nclass_labels = list(range(num_classes))\nlabel2name = dict(zip(class_labels, class_names))\nname2label = {v:k for k,v in label2name.items()}","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:04:38.129846Z","iopub.execute_input":"2024-04-29T04:04:38.130207Z","iopub.status.idle":"2024-04-29T04:05:46.186704Z","shell.execute_reply.started":"2024-04-29T04:04:38.130179Z","shell.execute_reply":"2024-04-29T04:05:46.185530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_augmenter():\n    augmenters = [\n        keras_cv.layers.MixUp(alpha=0.4),\n        keras_cv.layers.RandomCutout(height_factor=(1.0, 1.0),\n                                     width_factor=(0.06, 0.12)), # time-masking\n        keras_cv.layers.RandomCutout(height_factor=(0.06, 0.1),\n                                     width_factor=(1.0, 1.0)), # freq-masking\n    ]\n    \n    def augment(img, label):\n        label = tf.cast(tf.one_hot(label, 182), tf.float32)\n        data = {\"images\":img, \"labels\":label}\n        for augmenter in augmenters:\n            if tf.random.uniform([]) < 0.35:\n                data = augmenter(data, training=True)\n        return data[\"images\"], data[\"labels\"]\n    \n    return augment\n\ntrain_set_arg = train_set.map(build_augmenter(), num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:46.188763Z","iopub.execute_input":"2024-04-29T04:05:46.189088Z","iopub.status.idle":"2024-04-29T04:05:46.841173Z","shell.execute_reply.started":"2024-04-29T04:05:46.189061Z","shell.execute_reply":"2024-04-29T04:05:46.840230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot_to_decimal(one_hot_tensor):\n    return tf.argmax(one_hot_tensor, axis=1)\ntrain_set_arg = train_set_arg.map(lambda x, y: (x, one_hot_to_decimal(y)))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:46.847277Z","iopub.execute_input":"2024-04-29T04:05:46.847902Z","iopub.status.idle":"2024-04-29T04:05:46.912788Z","shell.execute_reply.started":"2024-04-29T04:05:46.847862Z","shell.execute_reply":"2024-04-29T04:05:46.911482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_some_training_data(train_set):\n    images, labels = next(iter(train_set))\n    n = 0\n    while n <= 5:\n        plt.subplot(2,3,n+1)\n        plt.imshow(images[n].numpy().astype(np.uint8))\n        #index = int(tf.argmax(labels[n].numpy().astype(np.uint8), axis=0))\n        plt.title(label2name[labels[n].numpy().astype(np.uint8)])\n        plt.axis('off')\n        n += 1\n    \n    plt.show()\n    \nplot_some_training_data(train_set_arg)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:46.914109Z","iopub.execute_input":"2024-04-29T04:05:46.914455Z","iopub.status.idle":"2024-04-29T04:05:48.551233Z","shell.execute_reply.started":"2024-04-29T04:05:46.914405Z","shell.execute_reply":"2024-04-29T04:05:48.550214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import layers\nfrom keras.applications import EfficientNetB3\n\ndef fine_tune(model, param):\n    print('model layers: ',len(model.layers))\n    if param >0:\n        for layer in model.layers[-param: ]:\n            layer.trainable = True\n    else:\n        for layer in model.layers:\n            layer.trainable = False\n\ndef build_model():\n    inputs = layers.Input(shape=(224, 224, 3))\n    base_model = EfficientNetB3(include_top=False, input_tensor=inputs, weights=\"imagenet\")\n\n    # Freeze the pretrained weights\n    base_model.trainable = False\n    #fine_tune(base_model,4)\n\n    # Rebuild top\n    x = layers.GlobalAveragePooling2D(name=\"avg_pool\")(base_model.output)\n    x = layers.BatchNormalization()(x)\n\n    top_dropout_rate = 0.2\n    x = layers.Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n    outputs = layers.Dense(182, activation=\"softmax\", name=\"pred\")(x)\n\n    # Compile\n    model = keras.Model(inputs, outputs, name=\"EfficientNet\")\n    optimizer = keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n    model.compile(\n        optimizer=optimizer, loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\"accuracy\"]\n    )\n    return model\n\nmodel = build_model()\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:48.552495Z","iopub.execute_input":"2024-04-29T04:05:48.552967Z","iopub.status.idle":"2024-04-29T04:05:51.743875Z","shell.execute_reply.started":"2024-04-29T04:05:48.552940Z","shell.execute_reply":"2024-04-29T04:05:51.742720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\ndef get_lr_callback(batch_size=32, mode='cos', epochs=100, plot=False):\n    lr_start, lr_max, lr_min = 5e-5, 8e-6 * batch_size, 1e-5\n    lr_ramp_ep, lr_sus_ep, lr_decay = 3, 0, 0.75\n\n    def lrfn(epoch):  # Learning rate update function\n        if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n        elif mode == 'exp': lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        elif mode == 'step': lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n        elif mode == 'cos':\n            decay_total_epochs, decay_epoch_index = epochs - lr_ramp_ep - lr_sus_ep + 3, epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            lr = (lr_max - lr_min) * 0.5 * (1 + math.cos(phase)) + lr_min\n        return lr\n\n    if plot:  # Plot lr curve if plot is True\n        plt.figure(figsize=(10, 5))\n        plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('lr')\n        plt.title('LR Scheduler')\n        plt.show()\n\n    return keras.callbacks.LearningRateScheduler(lrfn, verbose=False)  # Create lr callback","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:51.745101Z","iopub.execute_input":"2024-04-29T04:05:51.745446Z","iopub.status.idle":"2024-04-29T04:05:51.758155Z","shell.execute_reply.started":"2024-04-29T04:05:51.745405Z","shell.execute_reply":"2024-04-29T04:05:51.756829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_cb = get_lr_callback(BATCH_SIZE, plot=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:51.759476Z","iopub.execute_input":"2024-04-29T04:05:51.760384Z","iopub.status.idle":"2024-04-29T04:05:52.072746Z","shell.execute_reply.started":"2024-04-29T04:05:51.760344Z","shell.execute_reply":"2024-04-29T04:05:52.071802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\ncheckpoint_model_path = \"/kaggle/working/effinet.keras\"\nmetric = \"val_accuracy\"\ncheckpointer = ModelCheckpoint(\n    filepath=checkpoint_model_path,\n    monitor=metric, verbose=1, save_best_only=True)\nes_callback = EarlyStopping(monitor=metric, patience=5, verbose=0)\nmodel.compile(optimizer=tf.keras.optimizers.Adam(),\n    loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n    metrics=['accuracy'])\n\npre = model.fit(\n    train_set_arg,\n    validation_data=validate_set,\n    callbacks=[lr_cb,checkpointer,es_callback],\n    epochs=EPOCH,\n    verbose = 0)","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:05:52.073804Z","iopub.execute_input":"2024-04-29T04:05:52.074107Z","iopub.status.idle":"2024-04-29T04:06:13.198562Z","shell.execute_reply.started":"2024-04-29T04:05:52.074080Z","shell.execute_reply":"2024-04-29T04:06:13.196173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.save('BirdClef24 Efficient.keras')","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:06:13.199743Z","iopub.status.idle":"2024-04-29T04:06:13.200341Z","shell.execute_reply.started":"2024-04-29T04:06:13.200139Z","shell.execute_reply":"2024-04-29T04:06:13.200157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = pre.history['accuracy']\nval_acc = pre.history['val_accuracy']\n\nloss = pre.history['loss']\nval_loss = pre.history['val_loss']\nepoch_range = range(EPOCH)\nif es_callback.stopped_epoch >0:\n    epoch_range = range(es_callback.stopped_epoch -3)\n\nplt.figure(figsize= (8,6))\nplt.subplot(1,2,1)\nplt.plot(epoch_range,acc,label='Training Accuracy')\nplt.plot(epoch_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='center right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1,2,2)\nplt.plot(epoch_range, loss, label='Training loss')\nplt.plot(epoch_range, val_loss, label = 'Validation loss')\nplt.legend(loc='center right')\nplt.title('Training and Validation Loss')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-29T04:06:13.201262Z","iopub.status.idle":"2024-04-29T04:06:13.201644Z","shell.execute_reply.started":"2024-04-29T04:06:13.201466Z","shell.execute_reply":"2024-04-29T04:06:13.201480Z"},"trusted":true},"execution_count":null,"outputs":[]}]}