{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":1664376,"sourceType":"datasetVersion","datasetId":985270},{"sourceId":5181249,"sourceType":"datasetVersion","datasetId":3012199},{"sourceId":8721080,"sourceType":"datasetVersion","datasetId":5233176}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BirdCLEF 2024","metadata":{}},{"cell_type":"markdown","source":"# Methodology","metadata":{}},{"cell_type":"markdown","source":"I used the incredible workbook prepared by Awsaf from last year","metadata":{}},{"cell_type":"markdown","source":"Inspired by the success of previous approaches, this method builds upon the foundations laid by the Enhanced EfficientNetV2 Model for the BirdCLEF dataset. The approach combines datasets from previous years (2020-2023) with the current year's dataset (2024), employing advanced preprocessing techniques and leveraging transfer learning from EfficientNetV2 with fine-tuning.\nFor this approach, an EfficientNetV2 model is employed, building upon previous years' datasets (2020-2023) combined with the current year's dataset (2024). This model leverages transfer learning from EfficientNetV2 with fine-tuning and incorporates advanced preprocessing techniques specific to audio data.\n1.\tDataset Combination and Preprocessing:\n        Dataset: Combined BirdCLEF datasets from 2020 to 2023 and 2024.\n        Data Processing: Audio data is transformed into mel-spectrograms, resized to (128, None), and normalized for consistency.\n2.\tEfficientNetV2 Architecture:\n        Base Model: Utilizes EfficientNetV2 as the backbone model for feature extraction.\n        Transfer Learning: Pretrained weights from ImageNet are utilized to expedite model convergence and improve accuracy.\n        Fine-tuning: Specific layers of EfficientNetV2 are fine-tuned using combined datasets to adapt to the unique characteristics of BirdCLEF audio data.\n","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/awsaf49/birdclef23-pretraining-is-all-you-need-train","metadata":{}},{"cell_type":"markdown","source":"# Install Libraries ","metadata":{}},{"cell_type":"code","source":"import sys, os\nsys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')\n!pip install -q /kaggle/input/tensorflow-extra-lib-ds/tensorflow_extra-1.0.2-py3-none-any.whl --no-deps\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:21:42.159537Z","iopub.execute_input":"2024-06-18T13:21:42.160725Z","iopub.status.idle":"2024-06-18T13:21:45.519242Z","shell.execute_reply.started":"2024-06-18T13:21:42.160682Z","shell.execute_reply":"2024-06-18T13:21:45.517636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\nimport tensorflow_extra as tfe\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.layers import Dense, Activation\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:21:45.522012Z","iopub.execute_input":"2024-06-18T13:21:45.522446Z","iopub.status.idle":"2024-06-18T13:22:05.298048Z","shell.execute_reply.started":"2024-06-18T13:21:45.522407Z","shell.execute_reply":"2024-06-18T13:22:05.296693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntf.get_logger().setLevel('ERROR')\ntf.autograph.set_verbosity(0)\nimport os\nimport pandas as pd\nimport numpy as np\nimport random\nfrom glob import glob\nfrom tqdm import tqdm\ntqdm.pandas()\nimport gc\nimport librosa\nimport sklearn\nimport time\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport librosa.display as lid\nimport IPython.display as ipd\n\nimport tensorflow as tf\ntf.config.optimizer.set_jit(True) # enable xla for speed up\nimport tensorflow_io as tfio\nimport tensorflow.keras.backend as K\n\nimport efficientnet.tfkeras as efn\nimport tensorflow_extra as tfe","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:05.299715Z","iopub.execute_input":"2024-06-18T13:22:05.300876Z","iopub.status.idle":"2024-06-18T13:22:06.129748Z","shell.execute_reply.started":"2024-06-18T13:22:05.300821Z","shell.execute_reply":"2024-06-18T13:22:06.128420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('librosa:', librosa.__version__)\nprint('tf:', tf.__version__)\nprint('tfio:', tfio.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.132431Z","iopub.execute_input":"2024-06-18T13:22:06.132856Z","iopub.status.idle":"2024-06-18T13:22:06.141052Z","shell.execute_reply.started":"2024-06-18T13:22:06.132820Z","shell.execute_reply":"2024-06-18T13:22:06.139561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    debug = False\n    verbose = 0\n    \n    device = 'CPU'\n    seed = 42\n    \n    # Input image size and batch size\n    img_size = [128, 384]\n    batch_size = 16\n    infer_bs = 2\n    tta = 1\n    drop_remainder = True\n    \n    # STFT parameters\n    duration = 5 # duration for test\n    train_duration = 10\n    sample_rate = 32000\n    downsample = 1\n    audio_len = duration*sample_rate\n    nfft = 2028\n    window = 2048\n    hop_length = train_duration*32000 // (img_size[1] - 1)\n    fmin = 20\n    fmax = 16000\n    normalize = True\n\n    # Data Preprocessing Settings\n    class_names = sorted(os.listdir('/kaggle/input/birdclef-2024/train_audio/'))\n    num_classes = len(class_names)\n    class_labels = list(range(num_classes))\n    label2name = dict(zip(class_labels, class_names))\n    name2label = {v:k for k,v in label2name.items()}\n    \n    target_col = ['target']\n    tab_cols = ['filename','common_name','rate']\n    \n    target_col = ['target']\n    tab_cols = ['filename','common_name','rate']\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.142649Z","iopub.execute_input":"2024-06-18T13:22:06.143033Z","iopub.status.idle":"2024-06-18T13:22:06.186707Z","shell.execute_reply.started":"2024-06-18T13:22:06.143001Z","shell.execute_reply":"2024-06-18T13:22:06.185194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reproducibility ","metadata":{}},{"cell_type":"code","source":"tf.keras.utils.set_random_seed(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.188369Z","iopub.execute_input":"2024-06-18T13:22:06.188781Z","iopub.status.idle":"2024-06-18T13:22:06.194987Z","shell.execute_reply.started":"2024-06-18T13:22:06.188747Z","shell.execute_reply":"2024-06-18T13:22:06.193635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Up Device","metadata":{}},{"cell_type":"code","source":"def get_device():\n    \"Detect and intializes GPU/TPU automatically\"\n    # Check TPU category\n    tpu = 'local' if CFG.device=='TPU-VM' else None\n    try:\n        # Connect to TPU\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu) \n        # Set TPU strategy\n        strategy = tf.distribute.TPUStrategy(tpu)\n        print(f'> Running on {CFG.device} ', tpu.master(), end=' | ')\n        print('Num of TPUs: ', strategy.num_replicas_in_sync)\n        device=CFG.device\n    except:\n        # If TPU is not available, detect GPUs\n        gpus = tf.config.list_logical_devices('GPU')\n        ngpu = len(gpus)\n         # Check number of GPUs\n        if ngpu:\n            # Set GPU strategy\n            strategy = tf.distribute.MirroredStrategy(gpus) # single-GPU or multi-GPU\n            # Print GPU details\n            print(\"> Running on GPU\", end=' | ')\n            print(\"Num of GPUs: \", ngpu)\n            device='GPU'\n        else:\n            # If no GPUs are available, use CPU\n            print(\"> Running on CPU\")\n            strategy = tf.distribute.get_strategy()\n            device='CPU'\n    return strategy, device, tpu","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.196595Z","iopub.execute_input":"2024-06-18T13:22:06.196992Z","iopub.status.idle":"2024-06-18T13:22:06.209254Z","shell.execute_reply.started":"2024-06-18T13:22:06.196959Z","shell.execute_reply":"2024-06-18T13:22:06.208153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize GPU/TPU/TPU-VM\nstrategy, CFG.device, tpu = get_device()\nCFG.replicas = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.211342Z","iopub.execute_input":"2024-06-18T13:22:06.211776Z","iopub.status.idle":"2024-06-18T13:22:06.233515Z","shell.execute_reply.started":"2024-06-18T13:22:06.211741Z","shell.execute_reply":"2024-06-18T13:22:06.232208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Path","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/birdclef-2024'\nGCS_PATH = BASE_PATH","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.235697Z","iopub.execute_input":"2024-06-18T13:22:06.236105Z","iopub.status.idle":"2024-06-18T13:22:06.245792Z","shell.execute_reply.started":"2024-06-18T13:22:06.236070Z","shell.execute_reply":"2024-06-18T13:22:06.244444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Meta Data","metadata":{}},{"cell_type":"code","source":"test_audio_dir = '/kaggle/input/birdclef-2024/test_soundscapes/'\n\ntest_paths = [test_audio_dir+f for f in sorted(os.listdir(test_audio_dir))]\nif len(test_paths)==1:\n    test_audio_dir = '/kaggle/input/birdclef-2024/unlabeled_soundscapes/'\n\n    test_paths = [test_audio_dir+f for f in sorted(os.listdir(test_audio_dir))][:2]\n    \ntest_df = pd.DataFrame(test_paths, columns=['filepath'])\ntest_df['filename'] = test_df.filepath.map(lambda x: x.split('/')[-1].replace('.ogg',''))\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.251204Z","iopub.execute_input":"2024-06-18T13:22:06.252186Z","iopub.status.idle":"2024-06-18T13:22:06.651791Z","shell.execute_reply.started":"2024-06-18T13:22:06.252147Z","shell.execute_reply":"2024-06-18T13:22:06.650420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.io.gfile.exists(test_df.filepath.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.653413Z","iopub.execute_input":"2024-06-18T13:22:06.653788Z","iopub.status.idle":"2024-06-18T13:22:06.663666Z","shell.execute_reply.started":"2024-06-18T13:22:06.653757Z","shell.execute_reply":"2024-06-18T13:22:06.662410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader","metadata":{}},{"cell_type":"code","source":"def load_audio(filepath, sr=32000, normalize=True):\n    audio, orig_sr = librosa.load(filepath, sr=None)\n    if sr!=orig_sr:\n        audio = librosa.resample(y, orig_sr, sr)\n    audio = audio.astype('float32').ravel()\n    audio = tf.convert_to_tensor(audio)\n    return audio\n\n@tf.function(jit_compile=True)\ndef MakeFrame(audio, duration=5, sr=32000):\n    frame_length = int(duration * sr)\n    frame_step = int(duration * sr)\n    chunks = tf.signal.frame(audio, frame_length, frame_step, pad_end=True)\n    return chunks","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.665689Z","iopub.execute_input":"2024-06-18T13:22:06.666514Z","iopub.status.idle":"2024-06-18T13:22:06.676775Z","shell.execute_reply.started":"2024-06-18T13:22:06.666469Z","shell.execute_reply":"2024-06-18T13:22:06.675304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"def display_audio(row):\n    # Caption for viz\n    caption = f'Id: {row.filename}'\n    # Read audio file\n    audio = load_audio(row.filepath)\n    # Keep fixed length audio\n    audio = audio[:CFG.audio_len]\n    # Display audio\n    print(\"# Audio:\")\n    display(ipd.Audio(audio.numpy(), rate=CFG.sample_rate))\n    print('# Visualization:')\n    plt.figure(figsize=(12, 3))\n    plt.title(caption)\n    # Waveplot\n    lid.waveshow(audio.numpy(),\n                 sr=CFG.sample_rate,)\n                 \n    plt.xlabel('');\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.678422Z","iopub.execute_input":"2024-06-18T13:22:06.678829Z","iopub.status.idle":"2024-06-18T13:22:06.696675Z","shell.execute_reply.started":"2024-06-18T13:22:06.678800Z","shell.execute_reply":"2024-06-18T13:22:06.695023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_audio(test_df.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:06.699131Z","iopub.execute_input":"2024-06-18T13:22:06.699721Z","iopub.status.idle":"2024-06-18T13:22:29.244957Z","shell.execute_reply.started":"2024-06-18T13:22:06.699672Z","shell.execute_reply":"2024-06-18T13:22:29.243136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"import shutil\n# Directory of checkpoint\nCKPT_DIR = '/kaggle/input/birdclef24-pretrain'\n\n# Get file paths of all trained models in the directory\nCKPT_PATHS = sorted([x for x in glob(f'{CKPT_DIR}/fold-*keras')])\nprint(\"Checkpoints: \", CKPT_PATHS)\n\n\n# Define a writable directory\nWRITABLE_DIR = '/kaggle/working/models/'\n\n# Create the writable directory if it does not exist\nif not os.path.exists(WRITABLE_DIR):\n    os.makedirs(WRITABLE_DIR)\n\n\n# Copy the model files to the writable directory\nfor ckpt_path in CKPT_PATHS:\n    shutil.copy(ckpt_path, WRITABLE_DIR)\n\n# Update the checkpoint paths to the writable directory\nCKPT_PATHS = sorted([f'{WRITABLE_DIR}/{os.path.basename(x)}' for x in glob(f'{CKPT_DIR}/fold-*keras')])\n\n# Load all the models in memory to speed up\nCKPTS = [tf.keras.models.load_model(x, compile=False) for x in tqdm(CKPT_PATHS, desc=\"Loading ckpts \")]\n# Num of ckpt to use\nNUM_CKPTS = 1\n\n# Submit or Interactive mode\n#SUBMIT = pd.read_csv('/kaggle/input/birdclef-2024/sample_submission.csv').shape[0] != 3","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:29.246646Z","iopub.execute_input":"2024-06-18T13:22:29.247493Z","iopub.status.idle":"2024-06-18T13:22:38.120449Z","shell.execute_reply.started":"2024-06-18T13:22:29.247452Z","shell.execute_reply":"2024-06-18T13:22:38.119179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nprint(os.listdir('/kaggle/input/birdclef24-pretrain'))\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:38.122211Z","iopub.execute_input":"2024-06-18T13:22:38.122689Z","iopub.status.idle":"2024-06-18T13:22:38.129240Z","shell.execute_reply.started":"2024-06-18T13:22:38.122655Z","shell.execute_reply":"2024-06-18T13:22:38.127924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Start stopwatch\ntick = time.time()\n\n# Initialize empty list to store ids\nids = []\n# Initialize empty array to store predictions\npreds = np.empty(shape=(0, 182), dtype='float32')\n\n# Iterate over each audio file in the test dataset\nfor filepath in tqdm(test_df.filepath.tolist(), 'test '):\n    # Extract the filename without the extension\n    filename = filepath.split('/')[-1].replace('.ogg','')\n    \n    # Load audio from file and create audio frames, each recording will be a batch input\n    audio = load_audio(filepath)\n    chunks = MakeFrame(audio)\n    \n    # Predict bird species for all frames in a recording using all trained models\n    chunk_preds = np.zeros(shape=(len(chunks), 182), dtype=np.float32)\n    for model in CKPTS[:NUM_CKPTS]:\n        # Get the model's predictions for the current audio frames\n        rec_preds = model(chunks, training=False).numpy()\n        # Ensemble all prediction with average\n        chunk_preds += rec_preds/len(CKPTS)\n    \n    # Create a ID for each frame in a recording using the filename and frame number\n    rec_ids = [f'{filename}_{(frame_id+1)*5}' for frame_id in range(len(chunks))]\n    \n    # Concatenate the ids\n    ids += rec_ids\n    # Concatenate the predictions\n    preds = np.concatenate([preds, chunk_preds], axis=0)\n    \n# Stop stopwatch\ntock = time.time()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:22:38.130908Z","iopub.execute_input":"2024-06-18T13:22:38.131293Z","iopub.status.idle":"2024-06-18T13:22:47.285078Z","shell.execute_reply.started":"2024-06-18T13:22:38.131262Z","shell.execute_reply":"2024-06-18T13:22:47.283740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Summary/ Model Architecture","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\n# Load your saved model\nmodel_path = '/kaggle/input/birdclef24-pretrain/fold-0.keras'\nloaded_model = tf.keras.models.load_model(model_path)\n\n# Print model summary\nloaded_model.summary()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:24:35.834249Z","iopub.execute_input":"2024-06-18T13:24:35.834757Z","iopub.status.idle":"2024-06-18T13:24:46.521837Z","shell.execute_reply.started":"2024-06-18T13:24:35.834725Z","shell.execute_reply":"2024-06-18T13:24:46.520678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\nfrom IPython.display import Image, display\n# Generate a plot of the model architecture\nplot_model(loaded_model, to_file='model_architecture.png', show_shapes=True, show_layer_names=True)\ndisplay(Image('model_architecture.png'))\nprint(\"Model architecture plot saved as 'model_architecture.png'\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:28:47.699720Z","iopub.execute_input":"2024-06-18T13:28:47.700198Z","iopub.status.idle":"2024-06-18T13:28:48.124018Z","shell.execute_reply.started":"2024-06-18T13:28:47.700166Z","shell.execute_reply":"2024-06-18T13:28:48.122668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"preds.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:28:54.000609Z","iopub.execute_input":"2024-06-18T13:28:54.001116Z","iopub.status.idle":"2024-06-18T13:28:54.011258Z","shell.execute_reply.started":"2024-06-18T13:28:54.001070Z","shell.execute_reply":"2024-06-18T13:28:54.009164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit prediction\npred_df = pd.DataFrame(ids, columns=['row_id'])\npred_df.loc[:, CFG.class_names] = preds\npred_df.to_csv('submission.csv',index=False)\npred_df","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:28:55.765773Z","iopub.execute_input":"2024-06-18T13:28:55.766259Z","iopub.status.idle":"2024-06-18T13:28:55.981505Z","shell.execute_reply.started":"2024-06-18T13:28:55.766223Z","shell.execute_reply":"2024-06-18T13:28:55.980200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_time = (tock-tick)*550 # ~1100 recording on the test data\nsub_time = time.gmtime(sub_time)\nsub_time = time.strftime(\"%H hr: %M min : %S sec\", sub_time)\nprint(f\">> Time for submission: ~ {sub_time}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-18T13:28:58.552729Z","iopub.execute_input":"2024-06-18T13:28:58.553242Z","iopub.status.idle":"2024-06-18T13:28:58.560551Z","shell.execute_reply.started":"2024-06-18T13:28:58.553208Z","shell.execute_reply":"2024-06-18T13:28:58.559226Z"},"trusted":true},"execution_count":null,"outputs":[]}]}