{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":25954,"databundleVersionId":2091745,"sourceType":"competition"},{"sourceId":33246,"databundleVersionId":3221581,"sourceType":"competition"},{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":1487019,"sourceType":"datasetVersion","datasetId":726237},{"sourceId":1487116,"sourceType":"datasetVersion","datasetId":726312},{"sourceId":5181249,"sourceType":"datasetVersion","datasetId":3012199},{"sourceId":5195317,"sourceType":"datasetVersion","datasetId":3020983},{"sourceId":8655218,"sourceType":"datasetVersion","datasetId":5182844},{"sourceId":170457993,"sourceType":"kernelVersion"}],"dockerImageVersionId":30675,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BirdCLEF 2024 🐦","metadata":{}},{"cell_type":"code","source":"# # Tensorflow for tpu-vm\n# !pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n\n# # Tensorflow utilities\n#!pip install -q tensorflow-addons==0.19.0\n#!pip install -q tensorflow-probability==0.19.0\n#!pip install -q tensorflow-io==0.32.0\n\n# # Other utilies not available on tpu-vm\n!pip install -q opencv-python-headless\n!pip install -q librosa\n!pip install -q scikit-learn\n\n# # WandB for Experiment tracking\n!pip install -qU wandb","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For Spectrogram, SpecAug layers\n!pip install -q /kaggle/input/tensorflow-extra-lib-ds/tensorflow_extra-1.0.2-py3-none-any.whl --no-deps\n\n# efficientnet with filter stride reduction (FSR)\n!pip install -qU git+https://github.com/awsaf49/efficientnet-spec","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# os.environ['TPU_LOG_DIR'] = '/kaggle/working'\n\nimport pandas as pd\npd.options.mode.chained_assignment = None # avoids assignment warning\nimport numpy as np\nimport random\nfrom glob import glob\nfrom tqdm import tqdm\ntqdm.pandas() # enable progress bars in pandas operations\nimport gc\n\nimport librosa\nimport sklearn\nimport json\n\n# Import for visualization\nimport matplotlib as mpl\ncmap = mpl.cm.get_cmap('coolwarm')\nimport matplotlib.pyplot as plt\nimport librosa.display as lid\nimport IPython.display as ipd\n#import cv2\n\n# Import tensorflow\nimport tensorflow as tf\n# Set logging level to avoid unnecessary messages\ntf.get_logger().setLevel('ERROR')\n# Set autograph verbosity to avoid unnecessary messages\ntf.autograph.set_verbosity(0)\n# Enable xla for speed up\n# tf.config.optimizer.set_jit(False) # throws error for time-freq-mask\n\n# Import required tensorflow modules\nimport tensorflow_io as tfio\n#import tensorflow_addons as tfa\n#import tensorflow_probability as tfp\nimport tensorflow.keras.backend as K\n\n# Import KaggleDatasets for accessing Kaggle datasets\nfrom kaggle_datasets import KaggleDatasets\n\n# WandB for experiment tracking\nimport wandb","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.632068,"end_time":"2022-03-08T03:18:14.585094","exception":false,"start_time":"2022-03-08T03:18:11.953026","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('librosa:', librosa.__version__)\nprint('tf:', tf.__version__)\n\nprint('tfio:', tfio.__version__)\nprint('w&b:', wandb.__version__)","metadata":{"papermill":{"duration":0.155095,"end_time":"2022-03-08T03:18:14.939054","exception":false,"start_time":"2022-03-08T03:18:14.783959","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n # Debugging\n debug = False\n \n # Verbosity level\n verbose = 1\n \n # Plot training history\n training_plot = True\n \n # Weights and Biases logging\n wandb = True\n competition = 'birdclef-2023' \n _wandb_kernel = 'awsaf49'\n \n # Experiment name and comment\n exp_name = 'birdclef-pretrain-v3'\n comment = 'EfficientNetB1|No-FSR|t=10s|128x384|cutmix'\n # Notebook link\n notebook_link = 'https://www.kaggle.com/awsaf49/birdclef23-pretraining-is-all-you-need-train'\n \n # Device and random seed\n device = 'TPU-VM'\n seed = 42\n \n\n # Input image size and batch size\n img_size = [128, 384]\n batch_size = 32\n upsample_thr = 50 # min sample of each class (upsample)\n cv_filter = True # always keeps low sample data in train\n \n # Audio duration, sample rate, and length\n duration = 10 # second\n sample_rate = 32000\n audio_len = duration*sample_rate\n \n # STFT parameters\n nfft = 2028\n window = 2048\n hop_length = audio_len // (img_size[1] - 1)\n fmin = 20\n fmax = 16000\n normalize = True\n \n # Inference batch size, test time augmentation, and drop remainder\n infer_bs = 2\n tta = 1\n drop_remainder = True\n \n # Number of epochs, model name, and number of folds\n epochs = 25\n model_name = 'EfficientNetB1'\n fsr = False # reduce stride of stem block\n num_fold = 5\n \n # Selected folds for training and evaluation\n selected_folds = [0]\n\n # Pretraining, neck features, and final activation function\n pretrain = 'imagenet'\n neck_features = 0\n final_act = 'softmax'\n \n # Learning rate, optimizer, and scheduler\n lr = 1e-3\n scheduler = 'cos'\n optimizer = 'Adam' # AdamW, Adam\n \n # Loss function and label smoothing\n loss = 'CCE' # BCE, CCE\n label_smoothing = 0.05 # label smoothing\n \n # Data augmentation parameters\n augment=True\n \n # Time Freq masking\n freq_mask_prob=0.50\n num_freq_masks=1\n freq_mask_param=10\n time_mask_prob=0.50\n num_time_masks=2\n time_mask_param=25\n\n # Audio Augmentation Settings\n audio_augment_prob = 0.5\n \n mixup_prob = 0.65\n mixup_alpha = 0.5\n \n cutmix_prob = 0.65\n cutmix_alpha = 2.5\n \n timeshift_prob = 0.0\n \n gn_prob = 0.35\n\n # Class Labels for BirdCLEF 24\n class_names = sorted(os.listdir('/kaggle/input/allbirdclef2024/new-birdclef-2024/train_audio'))\n \n num_classes = len(class_names)\n class_labels = list(range(num_classes))\n label2name = dict(zip(class_labels, class_names))\n name2label = {v:k for k,v in label2name.items()}\n \n # Class Labels for BirdCLEF 21 & 22\n class_names2 = sorted(set(os.listdir('/kaggle/input/birdclef-2021/train_short_audio/')\n +os.listdir('/kaggle/input/birdclef-2022/train_audio/') \n +os.listdir('/kaggle/input/birdclef-2023/train_audio/')\n \n +os.listdir('/kaggle/input/birdsong-recognition/train_audio/')))\n num_classes2 = len(class_names2)\n class_labels2 = list(range(num_classes2))\n label2name2 = dict(zip(class_labels2, class_names2))\n name2label2 = {v:k for k,v in label2name2.items()}\n\n \n # Training Settings\n target_col = ['target']\n tab_cols = ['filename']\n monitor = 'auc'","metadata":{"papermill":{"duration":0.156464,"end_time":"2022-03-08T03:18:18.322809","exception":false,"start_time":"2022-03-08T03:18:18.166345","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.set_random_seed(CFG.seed)","metadata":{"papermill":{"duration":0.153451,"end_time":"2022-03-08T03:18:18.685056","exception":false,"start_time":"2022-03-08T03:18:18.531605","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import wandb library for logging and tracking experiments\nimport wandb\n\n# Try to get the API key from Kaggle secrets\ntry:\n from kaggle_secrets import UserSecretsClient\n user_secrets = UserSecretsClient()\n api_key = user_secrets.get_secret(\"WANDB\")\n # Login to wandb with the API key\n wandb.login(key=api_key)\n # Set anonymous mode to None\n anonymous = None\nexcept:\n # If Kaggle secrets are not available, set anonymous mode to 'must'\n anonymous = 'must'\n # Login to wandb anonymously and relogin if needed\n wandb.login(anonymous=anonymous, relogin=True)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"TPU\" in CFG.device:\n tpu = 'local' if CFG.device=='TPU-VM' else None\n print(\"connecting to TPU...\")\n try:\n \n resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='local')\n tf.tpu.experimental.initialize_tpu_system(resolver)\n strategy = tf.distribute.TPUStrategy(resolver)\n \n except Exception as e:\n print(e)\n CFG.device = \"GPU\"\n \nif CFG.device == \"GPU\" or CFG.device==\"CPU\":\n ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n if ngpu>1:\n print(\"Using multi GPU\")\n strategy = tf.distribute.MirroredStrategy()\n elif ngpu==1:\n print(\"Using single GPU\")\n strategy = tf.distribute.get_strategy()\n else:\n print(\"Using CPU\")\n strategy = tf.distribute.get_strategy()\n CFG.device = \"CPU\"\n\nif CFG.device == \"GPU\":\n print(\"Num GPUs Available: \", ngpu)\n \n\nAUTO = tf.data.experimental.AUTOTUNE\nCFG.replicas = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {CFG.replicas}')\nCFG.device","metadata":{"_kg_hide-input":true,"papermill":{"duration":7.941725,"end_time":"2022-03-08T03:18:26.826553","exception":false,"start_time":"2022-03-08T03:18:18.884828","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH0 = '/kaggle/input/birdsong-recognition'\nBASE_PATH1 = '/kaggle/input/birdclef-2021'\nBASE_PATH2 = '/kaggle/input/birdclef-2022'\nBASE_PATH3 = '/kaggle/input/birdclef-2023'\nBASE_PATH4 = '/kaggle/input/xeno-canto-bird-recordings-extended-a-m'\nBASE_PATH5 = '/kaggle/input/xeno-canto-bird-recordings-extended-n-z'\nBASE_PATH6 = '/kaggle/input/allbirdclef2024/new-birdclef-2024'\n\nif CFG.device==\"TPU\":\n from kaggle_datasets import KaggleDatasets\n GCS_PATH0 = KaggleDatasets().get_gcs_path(BASE_PATH0.split('/')[-1])\n GCS_PATH1 = KaggleDatasets().get_gcs_path(BASE_PATH1.split('/')[-1])\n GCS_PATH2 = KaggleDatasets().get_gcs_path(BASE_PATH2.split('/')[-1])\n GCS_PATH3 = KaggleDatasets().get_gcs_path(BASE_PATH3.split('/')[-1])\n GCS_PATH4 = KaggleDatasets().get_gcs_path(BASE_PATH4.split('/')[-1])\n GCS_PATH5 = KaggleDatasets().get_gcs_path(BASE_PATH5.split('/')[-1])\n GCS_PATH6 = KaggleDatasets().get_gcs_path(BASE_PATH6.split('/')[-1])\n\nelse:\n GCS_PATH0 = BASE_PATH0\n GCS_PATH1 = BASE_PATH1\n GCS_PATH2 = BASE_PATH2\n GCS_PATH3 = BASE_PATH3\n GCS_PATH4 = BASE_PATH4\n GCS_PATH5 = BASE_PATH5\n GCS_PATH6 = BASE_PATH6\n\n ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 24","metadata":{}},{"cell_type":"code","source":"df_24 = pd.read_csv(f'{BASE_PATH6}/train_metadata.csv', encoding='ISO-8859-1')\ndf_24['filepath'] = GCS_PATH6 + '/train_audio/' + df_24.filename\ndf_24['target'] = df_24.primary_label.map(CFG.name2label)\ndf_24['birdclef'] = '24'\n\ndf_24['filepath'] = df_24['filepath'].astype(str)\n\ndf_24['filename'] = df_24.filepath.map(lambda x: x.split('/')[-1])\ndf_24['xc_id'] = df_24.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nassert tf.io.gfile.exists(df_24.filepath.iloc[0])\n\n# Display rwos\nprint(\"# Samples in BirdCLEF 24: {:,}\".format(len(df_24)))\ndf_24.head(2).style.set_caption(\"BirdCLEF - 24\").set_table_styles([{\n 'selector': 'caption',\n 'props': [\n ('color', 'blue'),\n ('font-size', '16px')\n ]\n}])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 23","metadata":{}},{"cell_type":"code","source":"df_23 = pd.read_csv(f'{BASE_PATH3}/train_metadata.csv')\ndf_23['filepath'] = GCS_PATH3 + '/train_audio/' + df_23.filename\ndf_23['target'] = df_23.primary_label.map(CFG.name2label2)\ndf_23['birdclef'] = '23'\ndf_23['filename'] = df_23.filepath.map(lambda x: x.split('/')[-1])\ndf_23['xc_id'] = df_23.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nassert tf.io.gfile.exists(df_23.filepath.iloc[0])\n\n# Display rwos\nprint(\"# Samples in BirdCLEF 23: {:,}\".format(len(df_23)))\ndf_23.head(2).style.set_caption(\"BirdCLEF - 23\").set_table_styles([{\n 'selector': 'caption',\n 'props': [\n ('color', 'blue'),\n ('font-size', '16px')\n ]\n}])","metadata":{"papermill":{"duration":0.241649,"end_time":"2022-03-08T03:18:27.408813","exception":false,"start_time":"2022-03-08T03:18:27.167164","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 20, 21, 22 & Xeno-Canto Extend","metadata":{}},{"cell_type":"code","source":"# BirdCLEF-2020\ndf_20 = pd.read_csv(f'{BASE_PATH0}/train.csv')\ndf_20['primary_label'] = df_20['ebird_code']\ndf_20['filepath'] = GCS_PATH0 + '/train_audio/' + df_20.primary_label + '/' + df_20.filename\ndf_20['scientific_name'] = df_20['sci_name']\ndf_20['common_name'] = df_20['species']\ndf_20['target'] = df_20.primary_label.map(CFG.name2label2)\ndf_20['birdclef'] = '20'\nassert tf.io.gfile.exists(df_20.filepath.iloc[0])\n\n# Xeno-Canto Extend by @vopani\ndf_xam = pd.read_csv(f'{BASE_PATH4}/train_extended.csv')\ndf_xam['filepath'] = GCS_PATH4 + '/A-M/' + df_xam.ebird_code + '/' + df_xam.filename\ndf_xnz = pd.read_csv(f'{BASE_PATH5}/train_extended.csv')\ndf_xnz['filepath'] = GCS_PATH5 + '/N-Z/' + df_xnz.ebird_code + '/' + df_xnz.filename\ndf_xc = pd.concat([df_xam, df_xnz], axis=0, ignore_index=True)\ndf_xc['primary_label'] = df_xc['ebird_code']\ndf_xc['scientific_name'] = df_xc['sci_name']\ndf_xc['common_name'] = df_xc['species']\ndf_xc['target'] = df_xc.primary_label.map(CFG.name2label2)\ndf_xc['birdclef'] = 'xc'\nassert tf.io.gfile.exists(df_xc.filepath.iloc[0])\n\n# BirdCLEF-2021\ndf_21 = pd.read_csv(f'{BASE_PATH1}/train_metadata.csv')\ndf_21['filepath'] = GCS_PATH1 + '/train_short_audio/' + df_21.primary_label + '/' + df_21.filename\ndf_21['target'] = df_21.primary_label.map(CFG.name2label2)\ndf_21['birdclef'] = '21'\ncorrupt_paths = ['/kaggle/input/birdclef-2021/train_short_audio/houwre/XC590621.ogg',\n '/kaggle/input/birdclef-2021/train_short_audio/cogdov/XC579430.ogg']\ndf_21 = df_21[~df_21.filepath.isin(corrupt_paths)] # remove all zero audios\nassert tf.io.gfile.exists(df_21.filepath.iloc[0])\n\n# BirdCLEF-2022\ndf_22 = pd.read_csv(f'{BASE_PATH2}/train_metadata.csv')\ndf_22['filepath'] = GCS_PATH2 + '/train_audio/' + df_22.filename\ndf_22['target'] = df_22.primary_label.map(CFG.name2label2)\ndf_22['birdclef'] = '22'\nassert tf.io.gfile.exists(df_22.filepath.iloc[0])\n\n# Merge 2021 and 2022 for pretraining\ndf_pre = pd.concat([df_20, df_21, df_22, df_23, df_xc], axis=0, ignore_index=True)\ndf_pre['filename'] = df_pre.filepath.map(lambda x: x.split('/')[-1])\ndf_pre['xc_id'] = df_pre.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nnodup_idx = df_pre[['xc_id','primary_label','author']].drop_duplicates().index\ndf_pre = df_pre.loc[nodup_idx].reset_index(drop=True)\n\n# # # Remove duplicates\n# df_pre = df_pre[~df_pre.xc_id.isin(df_24.xc_id)].reset_index(drop=True)\n# corrupt_mp3s = json.load(open('/kaggle/input/birdclef-corrupt-mp3-files-ds/corrupt_mp3_files.json','r'))\n# df_pre = df_pre[~df_pre.filepath.isin(corrupt_mp3s)]\n# df_pre = df_pre[['filename','filepath','primary_label','secondary_labels',\n# 'rating','author','file_type','xc_id','scientific_name',\n# 'common_name','target','birdclef','bird_seen']]\n\n# Display rows\nprint(\"# Samples for Pre-Training: {:,}\".format(len(df_pre)))\ndf_pre.head(2).style.set_caption(\"Pre-Training Data\").set_table_styles([{\n 'selector': 'caption',\n 'props': [\n ('color', 'blue'),\n ('font-size', '16px')\n ]\n}])\n\n# Show distribution\nplt.figure(figsize=(8, 4))\ndf_pre.birdclef.value_counts().plot.bar(color=[cmap(0.0),cmap(0.25), cmap(0.65), cmap(0.9)])\nplt.xlabel(\"Dataset\")\nplt.ylabel(\"Count\")\nplt.title(\"Dataset distribution for Pre-Training\")\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA ","metadata":{}},{"cell_type":"code","source":"def load_audio(filepath):\n audio, sr = librosa.load(filepath)\n return audio, sr\n\ndef get_spectrogram(audio):\n spec = librosa.feature.melspectrogram(y=audio, \n sr=CFG.sample_rate, \n n_mels=CFG.img_size[0],\n n_fft=CFG.nfft,\n hop_length=CFG.hop_length,\n fmax=CFG.fmax,\n fmin=CFG.fmin,\n )\n spec = librosa.power_to_db(spec, ref=1.0)\n return spec\n\ndef display_audio(row):\n # Caption for viz\n caption = f'Id: {row.filename} | Name: {row.common_name} | Sci.Name: {row.scientific_name} | Rating: {row.rating}'\n # Read audio file\n audio, sr = load_audio(row.filepath)\n # Keep fixed length audio\n audio = audio[:CFG.audio_len]\n # Spectrogram from audio\n spec = get_spectrogram(audio)\n # Display audio\n print(\"# Audio:\")\n display(ipd.Audio(audio, rate=CFG.sample_rate))\n# print(\"# Image:\")\n# show_image(row.common_name)\n print('# Visualization:')\n fig, ax = plt.subplots(2, 1, figsize=(12, 2*3), sharex=True, tight_layout=True)\n fig.suptitle(caption)\n # Waveplot\n lid.waveshow(audio,\n sr=CFG.sample_rate,\n ax=ax[0],\n color= cmap(0.1))\n # Specplot\n lid.specshow(spec, \n sr = CFG.sample_rate, \n hop_length = CFG.hop_length,\n n_fft=CFG.nfft,\n fmin=CFG.fmin,\n fmax=CFG.fmax,\n x_axis = 'time', \n y_axis = 'mel',\n cmap = 'coolwarm',\n ax=ax[1])\n ax[0].set_xlabel('');\n fig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 20","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = '20'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH0, BASE_PATH0)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Xeno-Canto Extend","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = 'xc'\nprint(f\"# Xeno-Canto - Extend\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH4, BASE_PATH4).replace(GCS_PATH5, BASE_PATH5)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 21","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = '21'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH1, BASE_PATH1)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 22","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = '22'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH2, BASE_PATH2)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 23","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = '23'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_23.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH3, BASE_PATH3)\nrow = tmp.squeeze()\n\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 24","metadata":{}},{"cell_type":"code","source":"BIRDCLEF = '24'\nprint(f\"# BirdCLEF - 24{BIRDCLEF}\")\ntmp = df_24.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH6, BASE_PATH6)\nrow = tmp.squeeze()\n\n# Display audio\ndisplay_audio(row)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Split ","metadata":{"papermill":{"duration":0.09524,"end_time":"2022-03-08T03:18:34.861029","exception":false,"start_time":"2022-03-08T03:18:34.765789","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Import required packages\nfrom sklearn.model_selection import StratifiedKFold\n\ndf_24['primary_label'] = df_24['primary_label'].fillna('Unknown')\n\n# Initialize the StratifiedKFold object with 5 splits and shuffle the data\nskf1 = StratifiedKFold(n_splits=25, shuffle=True, random_state=CFG.seed)\nskf2 = StratifiedKFold(n_splits=CFG.num_fold, shuffle=True, random_state=CFG.seed)\n\n# Reset the index of the dataframe\ndf_pre = df_pre.reset_index(drop=True)\ndf_24 = df_24.reset_index(drop=True)\n\n# Create a new column in the dataframe to store the fold number for each row\ndf_pre[\"fold\"] = -1\ndf_24[\"fold\"] = -1\n\n# BirdCLEF - 21 & 22 & 23\nfor fold, (train_idx, val_idx) in enumerate(skf1.split(df_pre, df_pre['primary_label'])):\n df_pre.loc[val_idx, 'fold'] = fold\n \n# IBirdCLEF - 24\nfor fold, (train_idx, val_idx) in enumerate(skf2.split(df_24, df_24['primary_label'])):\n df_24.loc[val_idx, 'fold'] = fold","metadata":{"papermill":{"duration":0.386301,"end_time":"2022-03-08T03:18:35.325064","exception":false,"start_time":"2022-03-08T03:18:34.938763","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_data(df, thr=5):\n # Count the number of samples for each class\n counts = df.primary_label.value_counts()\n\n # Condition that selects classes with less than `thr` samples\n cond = df.primary_label.isin(counts[counts thr].index.tolist()\n\n # create an empty list to store the upsampled dataframes\n down_dfs = []\n\n # loop through the undersampled classes and upsample them\n for c in up_classes:\n # get the dataframe for the current class\n class_df = df.query(\"primary_label==@c\")\n # Remove that class data\n df = df.query(\"primary_label!=@c\")\n # upsample the dataframe\n class_df = class_df.sample(n=thr, replace=False, random_state=CFG.seed)\n # append the upsampled dataframe to the list\n down_dfs.append(class_df)\n\n # concatenate the upsampled dataframes and the original dataframe\n down_df = pd.concat([df] + down_dfs, axis=0, ignore_index=True)\n \n return down_df","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filter Data","metadata":{}},{"cell_type":"code","source":"# Filter data\nf_df = filter_data(df_pre, thr=5)\n\nplt.figure(figsize=(10, 4))\nax1 = plt.subplot(1, 2, 1)\nf_df.cv.value_counts().plot.bar(legend=True, color=cmap(0.1))\nplt.yscale(\"log\")\nplt.title(\"BirdCLEF - 20, 21,22 & 23\")\nplt.legend([\"BirdCLEF - 20, 21,22 & 23\"])\n\nf_df = filter_data(df_24, thr=5)\nax2 = plt.subplot(1, 2, 2, sharey = ax1)\nf_df.cv.value_counts().plot.bar(legend=True, color=cmap(0.9))\nplt.yscale(\"log\")\nplt.title(\"BirdCLEF - 24\")\nplt.legend([\"BirdCLEF - 24\"])\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Upsample Data","metadata":{}},{"cell_type":"code","source":"# Upsample data\nup_thr = 100\ndn_df = downsample_data(df_pre, thr=400)\nup_df = upsample_data(dn_df, thr=up_thr)\nprint(\"# Pretraing Dataset\")\nprint(f'> Original: {len(df_pre)}')\nprint(f'> After Upsample: {len(up_df)}')\nprint(f'> After Downsample: {len(dn_df)}')\n\n# Show effect of upsample\nplt.figure(figsize=(12*2, 6))\n\nax1 = plt.subplot(1, 2, 1)\ndf_pre.primary_label.value_counts()[:].plot.bar(color='blue', label='original')\nup_df.primary_label.value_counts()[:].plot.bar(color='green', label='w/ upsample')\ndn_df.primary_label.value_counts()[:].plot.bar(color='red', label='w/ dowsample')\nplt.xticks([])\nplt.axhline(y=up_thr, color='g', linestyle='--', label='up threshold')\nplt.axhline(y=400, color='r', linestyle='--', label='down threshold')\nplt.legend()\nplt.title(\"Upsample for Pre-Training\")\n# plt.show()\n\n# Upsample data\nup_thr = 50\nup_df = upsample_data(df_24, thr=up_thr)\nprint(\"\\n# BirdCLEF - 24\")\nprint(f'> Before Upsample: {len(df_24)}')\nprint(f'> After Upsample: {len(up_df)}')\n\n# Show effect of upsample\nax2 = plt.subplot(1, 2, 2, sharey=ax1)\nup_df.primary_label.value_counts()[:].plot.bar(color='green', label='w/ upsample')\ndf_24.primary_label.value_counts()[:].plot.bar(color='red', label='w/o upsample')\nplt.xticks([])\nplt.axhline(y=up_thr, color='g', linestyle='--', label='up threshold')\nplt.legend()\nplt.title(\"Upsample in BirdCLEF - 23\")\n\n# plt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation ","metadata":{"papermill":{"duration":0.151237,"end_time":"2022-03-08T03:18:47.959873","exception":false,"start_time":"2022-03-08T03:18:47.808636","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Generates random integer\ndef random_int(shape=[], minval=0, maxval=1):\n return tf.random.uniform(shape=shape, minval=minval, maxval=maxval, dtype=tf.int32)\n\n\n# Generats random float\ndef random_float(shape=[], minval=0.0, maxval=1.0):\n rnd = tf.random.uniform(shape=shape, minval=minval, maxval=maxval, dtype=tf.float32)\n return rnd","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import tensorflow as tf\n#import tensorflow_probability as tfp\n#\nclass CustomBetaDistribution:\n def __init__(self, alpha, beta):\n self.alpha = alpha\n self.beta = beta\n\n def sample(self, sample_shape):\n gamma_alpha = tf.random.gamma(shape=(sample_shape,), alpha=self.alpha, beta=1.0)\n gamma_beta = tf.random.gamma(shape=(sample_shape,), alpha=self.beta, beta=1.0)\n \n return gamma_alpha / (gamma_alpha + gamma_beta)\n \n#https://github.com/awsaf49/tensorflow_extra/blob/main/tensorflow_extra/layers.py !!!!!!!!!!!!!!!! \n@tf.keras.utils.register_keras_serializable(package=\"tensorflow_extra\")\nclass MelSpectrogram(tf.keras.layers.Layer):\n \"\"\"\n Mel Spectrogram Layer to convert audio to mel spectrogram which works with single or batched inputs.\n\n Args:\n n_fft (int): Size of the FFT window.\n hop_length (int): Number of samples between successive STFT columns.\n win_length (int): Size of the STFT window. If None, defaults to n_fft.\n window_fn (str): Name of the window function to use.\n sr (int): Sample rate of the input signal.\n n_mels (int): Number of mel bins to generate.\n fmin (float): Minimum frequency of the mel bins.\n fmax (float): Maximum frequency of the mel bins. If None, defaults to sr / 2.\n power (float): Exponent for the magnitude spectrogram.\n power_to_db (bool): Whether to convert the power spectrogram to decibels.\n top_db (float): Maximum decibel value for the output spectrogram.\n power_to_db (bool): Whether to convert spectrogram from energy to power.\n out_channels (int): Number of output channels. If None, no channel is created.\n\n Call Args:\n input (tf.Tensor): Audio signal of shape (audio_len,) or (None, audio_len)\n\n Returns:\n tf.Tensor: Mel spectrogram of shape (..., n_mels, time, out_channels)\n or (..., n_mels, time) if out_channels is None.\n\n \"\"\"\n\n def __init__(\n self,\n n_fft=2048,\n hop_length=512,\n win_length=None,\n window=\"hann_window\",\n sr=16000,\n n_mels=128,\n fmin=20.0,\n fmax=None,\n power_to_db=True,\n top_db=80.0,\n power=2.0,\n amin=1e-10,\n ref=1.0,\n out_channels=None,\n name=\"mel_spectrogram\",\n **kwargs,\n ):\n super(MelSpectrogram, self).__init__(name=name, **kwargs)\n self.n_fft = n_fft\n self.hop_length = hop_length\n self.win_length = win_length or n_fft\n self.window = window\n self.sr = sr\n self.n_mels = n_mels\n self.fmin = fmin\n self.fmax = fmax or int(sr / 2)\n self.power_to_db = power_to_db\n self.top_db = top_db\n self.power = power\n self.amin = amin\n self.ref = ref\n self.out_channels = out_channels\n\n @tf.function\n def call(self, input):\n spec = self.spectrogram(input) # audio to spectrogram with shape\n spec = self.melscale(spec) # spectrogram to mel spectrogram\n if self.power_to_db:\n spec = self.dbscale(spec) # mel spectrogram to decibel mel spectrogram\n spec = tf.linalg.matrix_transpose(\n spec\n ) # (..., time, n_mels) to (..., n_mels, time)\n if self.out_channels is not None:\n spec = self.update_channels(spec)\n return spec\n\n def spectrogram(self, input):\n spec = tf.signal.stft(\n input,\n frame_length=self.win_length,\n frame_step=self.hop_length,\n fft_length=self.n_fft,\n window_fn=getattr(tf.signal, self.window),\n pad_end=True,\n )\n spec = tf.math.pow(tf.math.abs(spec), self.power)\n return spec\n\n def melscale(self, input):\n nbin = tf.shape(input)[-1]\n matrix = tf.signal.linear_to_mel_weight_matrix(\n num_mel_bins=self.n_mels,\n num_spectrogram_bins=nbin,\n sample_rate=self.sr,\n lower_edge_hertz=self.fmin,\n upper_edge_hertz=self.fmax,\n )\n return tf.tensordot(input, matrix, axes=1)\n\n def dbscale(self, input):\n log_spec = 10.0 * (\n tf.math.log(tf.math.maximum(input, self.amin)) / tf.math.log(10.0)\n )\n if callable(self.ref):\n ref_value = self.ref(log_spec)\n else:\n ref_value = tf.math.abs(self.ref)\n log_spec -= (\n 10.0\n * tf.math.log(tf.math.maximum(ref_value, self.amin))\n / tf.math.log(10.0)\n )\n log_spec = tf.math.maximum(log_spec, tf.math.reduce_max(log_spec) - self.top_db)\n return log_spec\n\n def update_channels(self, input):\n spec = input[..., tf.newaxis]\n if self.out_channels > 1:\n multiples = tf.concat(\n [\n tf.ones(tf.rank(spec) - 1, dtype=tf.int32),\n tf.constant([self.out_channels], dtype=tf.int32),\n ],\n axis=0,\n )\n spec = tf.tile(spec, multiples)\n return spec\n\n def get_config(self):\n config = super(MelSpectrogram, self).get_config()\n config.update(\n {\n \"n_fft\": self.n_fft,\n \"hop_length\": self.hop_length,\n \"win_length\": self.win_length,\n \"window\": self.window,\n \"sr\": self.sr,\n \"n_mels\": self.n_mels,\n \"fmin\": self.fmin,\n \"fmax\": self.fmax,\n \"power_to_db\": self.power_to_db,\n \"top_db\": self.top_db,\n \"power\": self.power,\n \"amin\": self.amin,\n \"ref\": self.ref,\n \"out_channels\": self.out_channels,\n }\n )\n return config\n\n\n@tf.keras.utils.register_keras_serializable(package=\"tensorflow_extra\")\nclass MixUp(tf.keras.layers.Layer):\n \"\"\"\n MixUp Augmentation Layer to apply MixUp to one batch.\n\n Args:\n alpha (float): Alpha parameter for beta distribution.\n prob (float): Probability of applying MixUp.\n\n Call Args:\n images (tf.Tensor): Batch of images.\n labels (tf.Tensor): Batch of labels.\n\n Returns:\n tf.Tensor: Batch of image.\n tf.Tensor: Batch of labels.\n\n \"\"\"\n\n def __init__(self, alpha=0.2, prob=0.5, name=\"mix_up\", **kwargs):\n super(MixUp, self).__init__(name=name, **kwargs)\n self.alpha = alpha\n self.prob = prob\n\n @tf.function\n def call(self, images, labels=None, training=False):\n\n # Skip batch if not training or if prob is not met or if labels are not provided\n if tf.random.uniform([]) > self.prob or not training or labels is None:\n return (images, labels) if labels is not None else images\n\n # Get original shape\n spec_shape = tf.shape(images)\n label_shape = tf.shape(labels)\n\n # Select lambda from beta distribution\n\n beta = CustomBetaDistribution(self.alpha, self.alpha)#tfp.distributions.Beta(self.alpha, self.alpha)\n lam = beta.sample(1)\n\n # It's faster to roll the batch by one instead of shuffling it to create image pairs\n images = lam * images + (1 - lam) * tf.roll(images, shift=1, axis=0)\n labels = lam * labels + (1 - lam) * tf.roll(labels, shift=1, axis=0)\n\n # Ensure original shape\n images = tf.reshape(images, spec_shape)\n labels = tf.reshape(labels, label_shape)\n\n return images, labels\n\n def get_config(self):\n config = super(MixUp, self).get_config()\n config.update(\n {\n \"alpha\": self.alpha,\n \"prob\": self.prob,\n }\n )\n return config\n\n\n@tf.keras.utils.register_keras_serializable(package=\"tensorflow_extra\")\nclass CutMix(tf.keras.layers.Layer):\n \"\"\"\n Augmentation layer to apply CutMix to one batch.\n\n Args:\n alpha (float): Alpha parameter for beta distribution.\n prob (float): Probability of applying CutMix.\n full_height (bool): If True, the patch will be cut with full height of the image.\n full_width (bool): If True, the patch will be cut with full width of the image.\n\n Call Args:\n images (tf.Tensor): Batch of images.\n labels (tf.Tensor): Batch of labels.\n\n Returns:\n tf.Tensor: Batch of image.\n tf.Tensor: Batch of labels.\n \"\"\"\n\n def __init__(\n self,\n alpha=0.2,\n prob=0.5,\n full_height=False,\n full_width=False,\n name=\"cut_mix\",\n **kwargs,\n ):\n super(CutMix, self).__init__(name=name, **kwargs)\n self.alpha = alpha\n self.prob = prob\n self.full_height = full_height\n self.full_width = full_width\n\n @tf.function\n def call(self, images, labels=None, training=False):\n # Skip batch if not training or if prob is not met or if labels are not provided\n if tf.random.uniform([]) > self.prob or not training or labels is None:\n return (images, labels) if labels is not None else images\n\n # Ensure 4D input\n images, was_2d = self._ensure_4d(images)\n\n # Get original shapes\n image_shape = tf.shape(images)\n label_shape = tf.shape(labels)\n\n # Select lambda from beta distribution\n beta =CustomBetaDistribution(self.alpha, self.alpha)# tfp.distributions.Beta(self.alpha, self.alpha)\n lam = beta.sample(1)\n\n # It's faster to roll the batch by one instead of shuffling it to create image pairs\n images_rolled = tf.roll(images, shift=1, axis=0)\n labels_rolled = tf.roll(labels, shift=1, axis=0)\n\n # Find dimensions of patch\n H = tf.cast(image_shape[1], tf.int32)\n W = tf.cast(image_shape[2], tf.int32)\n r_x = (\n tf.random.uniform([], maxval=W, dtype=tf.int32)\n if not self.full_width\n else 0\n )\n r_y = (\n tf.random.uniform([], maxval=H, dtype=tf.int32)\n if not self.full_height\n else 0\n )\n r = 0.5 * tf.math.sqrt(1.0 - lam)\n r_w_p = r if not self.full_width else 1.0\n r_h_p = r if not self.full_height else 1.0\n r_w_half = tf.cast(r_w_p * tf.cast(W, tf.float32), tf.int32)\n r_h_half = tf.cast(r_h_p * tf.cast(H, tf.float32), tf.int32)\n\n # Find the coordinates of the patch\n x1 = tf.cast(tf.clip_by_value(r_x - r_w_half, 0, W), tf.int32)\n x2 = tf.cast(tf.clip_by_value(r_x + r_w_half, 1, W), tf.int32)\n y1 = tf.cast(tf.clip_by_value(r_y - r_h_half, 0, H), tf.int32)\n y2 = tf.cast(tf.clip_by_value(r_y + r_h_half, 1, H), tf.int32)\n\n # Extract outer-pad patch -> [0, 0, 1, 1, 0, 0]\n patch1 = images[:, y1:y2, x1:x2, :] # [batch, height, width, channel]\n patch1 = tf.pad(\n patch1, [[0, 0], [y1, H - y2], [x1, W - x2], [0, 0]]\n ) # outer-pad\n\n # Extract inner-pad patch -> [2, 2, 0, 0, 2, 2]\n patch2 = images_rolled[:, y1:y2, x1:x2, :]\n patch2 = tf.pad(\n patch2, [[0, 0], [y1, H - y2], [x1, W - x2], [0, 0]]\n ) # outer-pad\n patch2 = images_rolled - patch2 # inner-pad = img - outer-pad\n\n # Combine patches [0, 0, 1, 1, 0, 0] + [2, 2, 0, 0, 2, 2] -> [2, 2, 1, 1, 2, 2]\n images = patch1 + patch2\n\n # Combine labels\n lam = tf.cast((1.0 - (x2 - x1) * (y2 - y1) / (W * H)), tf.float32)\n labels = lam * labels + (1.0 - lam) * labels_rolled\n\n # Ensure original shape\n images = tf.reshape(images, image_shape)\n labels = tf.reshape(labels, label_shape)\n\n # Ensure original shape\n images = self._ensure_original_shape(images, was_2d)\n\n return images, labels\n\n def _ensure_4d(self, tensor):\n if len(tensor.shape) == 2:\n tensor = tf.expand_dims(tensor, axis=1)\n tensor = tf.expand_dims(tensor, axis=-1)\n return tensor, True\n return tensor, False\n\n def _ensure_original_shape(self, tensor, was_2d):\n if was_2d:\n tensor = tf.squeeze(tensor, axis=-1)\n tensor = tf.squeeze(tensor, axis=1)\n return tensor\n\n def get_config(self):\n config = super(CutMix, self).get_config()\n config.update(\n {\n \"alpha\": self.alpha,\n \"prob\": self.prob,\n \"full_height\": self.full_height,\n \"full_width\": self.full_width,\n }\n )\n return config\n\n\n@tf.keras.utils.register_keras_serializable(package=\"tensorflow_extra\")\nclass TimeFreqMask(tf.keras.layers.Layer):\n \"\"\"\n Applies Time Freq Mask to spectrogram input\n Ref: https://pytorch.org/audio/main/_modules/torchaudio/functional/functional.html#mask_along_axis_iid\n \"\"\"\n\n def __init__(\n self,\n freq_mask_prob=0.5,\n num_freq_masks=2,\n freq_mask_param=10,\n time_mask_prob=0.5,\n num_time_masks=2,\n time_mask_param=20,\n time_last=True,\n name=\"time_freq_mask\",\n **kwargs,\n ):\n super(TimeFreqMask, self).__init__(name=name, **kwargs)\n self.freq_mask_prob = freq_mask_prob\n self.num_freq_masks = num_freq_masks\n self.freq_mask_param = freq_mask_param\n self.time_mask_prob = time_mask_prob\n self.num_time_masks = num_time_masks\n self.time_mask_param = time_mask_param\n self.time_last = time_last\n\n @tf.function\n def call(self, inputs, training=False):\n if not training:\n return inputs\n x = inputs\n # Adjust input shape\n ndims = tf.rank(x)\n shape = tf.shape(x)\n\n # if ndims == 3:\n # x = x[tf.newaxis, ...]\n # x = tf.reshape(x, shape=(1, tf.split(shape, 3)))\n # elif ndims == 2:\n # x = x[tf.newaxis, ..., tf.newaxis]\n # x = tf.reshape(x, shape=(1, tf.split(shape, 2), 1))\n # else:\n # pass\n # elif ndims > 4 or ndims < 2:\n # raise ValueError(\"Input tensor must be 2, 3, or 4-dimensional.\")\n # Apply time mask\n for _ in tf.range(self.num_time_masks):\n x = self.mask_along_axis_iid(\n x,\n self.time_mask_param,\n 0,\n 2 + int(self.time_last),\n self.time_mask_prob,\n )\n # Apply freq mask\n for _ in tf.range(self.num_freq_masks):\n x = self.mask_along_axis_iid(\n x,\n self.freq_mask_param,\n 0,\n 2 + int(not self.time_last),\n self.freq_mask_prob,\n )\n # Re-adjust output shape\n # if ndims == 3:\n # x = x[0]\n # elif ndims == 2:\n # x = x[0, ..., 0]\n return x\n\n def mask_along_axis_iid(self, specs, mask_param, mask_value, axis, p):\n if axis not in [2, 3]:\n raise ValueError(\"Only Frequency and Time masking are supported\")\n\n if not 0.0 <= p <= 1.0:\n raise ValueError(f\"The value of p must be between 0.0 and 1.0 ({p} given).\")\n\n mask_param = mask_param # self._get_mask_param(mask_param, p, specs.shape[axis])\n if tf.random.uniform([]) > p:\n return specs\n\n specs = tf.transpose(specs, perm=[0, 3, 1, 2]) # (batch, channel, freq, time)\n\n dtype = specs.dtype\n shape = tf.shape(specs)\n\n value = tf.random.uniform(shape=shape[:2], dtype=dtype) * mask_param\n min_value = tf.random.uniform(shape=shape[:2], dtype=dtype) * (\n specs.shape[axis] - value\n )\n\n # Create broadcastable mask\n mask_start = tf.cast(min_value, tf.float32)[..., None, None]\n mask_end = (tf.cast(min_value, tf.float32) + tf.cast(value, tf.float32))[\n ..., None, None\n ]\n mask = tf.range(0, specs.shape[axis], dtype=dtype)\n\n # Per batch example masking\n specs = tf.linalg.matrix_transpose(specs) if axis == 2 else specs\n cond = (mask >= mask_start) & (mask < mask_end)\n specs = tf.where(\n cond, tf.fill(tf.shape(specs), tf.cast(mask_value, dtype=dtype)), specs\n )\n specs = tf.linalg.matrix_transpose(specs) if axis == 2 else specs\n\n specs = tf.transpose(specs, perm=[0, 2, 3, 1]) # (batch, freq, time, channel)\n\n return specs\n\n def get_config(self):\n config = super(TimeFreqMask, self).get_config()\n config.update(\n {\n \"freq_mask_prob\": self.freq_mask_prob,\n \"num_freq_masks\": self.num_freq_masks,\n \"freq_mask_param\": self.freq_mask_param,\n \"time_mask_prob\": self.time_mask_prob,\n \"num_time_masks\": self.num_time_masks,\n \"time_mask_param\": self.time_mask_param,\n \"time_last\": self.time_last,\n }\n )\n return config\n\n\n@tf.keras.utils.register_keras_serializable(package=\"tensorflow_extra\")\nclass ZScoreMinMax(tf.keras.layers.Layer):\n \"\"\"\n Applies Z-score normalization and Min-Max normalization to the input tensor.\n \"\"\"\n def __init__(self, name=\"z_score_min_max\", **kwargs):\n super(ZScoreMinMax, self).__init__(name=name, **kwargs)\n\n @tf.function\n def call(self, inputs):\n # Standardize using Z-score\n mean = tf.math.reduce_mean(inputs)\n std = tf.math.reduce_std(inputs)\n standardized = tf.where(tf.math.equal(std, 0), inputs - mean, (inputs - mean) / std)\n\n # Normalize using Min-Max\n min_val = tf.math.reduce_min(standardized)\n max_val = tf.math.reduce_max(standardized)\n normalized = tf.where(tf.math.equal(max_val - min_val, 0), standardized - min_val,\n (standardized - min_val) / (max_val - min_val))\n\n return normalized\n \n def get_config(self):\n config = super(ZScoreMinMax, self).get_config()\n return config","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Import required packages\n#import tensorflow_extra as tfe\n\n# Randomly shift audio -> any sound at time may get shifted to time\n@tf.function\ndef TimeShift(audio, prob=0.5):\n # Randomly apply time shift with probability `prob`\n if random_float() < prob:\n # Calculate random shift value\n shift = random_int(shape=[], minval=0, maxval=tf.shape(audio)[0])\n # Randomly set the shift to be negative with 50% probability\n if random_float() < 0.5:\n shift = -shift\n # Roll the audio signal by the shift value\n audio = tf.roll(audio, shift, axis=0)\n return audio\n\n# Apply random noise to audio data\n@tf.function\ndef GaussianNoise(audio, std=[0.0025, 0.025], prob=0.5):\n # Select a random value of standard deviation for Gaussian noise within the given range\n std = random_float([], std[0], std[1])\n # Randomly apply Gaussian noise with probability `prob`\n if random_float() < prob:\n # Add random Gaussian noise to the audio signal\n GN = tf.keras.layers.GaussianNoise(stddev=std)\n audio = GN(audio, training=True) # training=False don't apply noise to data\n return audio\n\n# Applies augmentation to Audio Signal\ndef AudioAug(audio):\n # Apply time shift and Gaussian noise to the audio signal\n audio = TimeShift(audio, prob=CFG.timeshift_prob)\n #audio = GaussianNoise(audio, prob=CFG.gn_prob)\n return audio\n\n# CutMix & MixUp\nmixup_layer = MixUp(alpha=CFG.mixup_alpha, prob=CFG.mixup_prob)\ncutmix_layer = CutMix(alpha=CFG.cutmix_alpha, prob=CFG.cutmix_prob)\n\ndef CutMixUp(audios, labels):\n audios, labels = mixup_layer(audios, labels, training=True)\n #audios, labels = cutmix_layer(audios, labels, training=True)\n #print(audios, labels)\n return audios, labels","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader ","metadata":{"papermill":{"duration":0.152812,"end_time":"2022-03-08T03:18:48.676686","exception":false,"start_time":"2022-03-08T03:18:48.523874","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Decodes Audio\ndef audio_decoder(with_labels=True, dim=CFG.audio_len, \n take_first=False, num_classes=264, CFG=CFG):\n def get_audio(filepath):\n ftype = filepath[1]\n filepath = filepath[0]\n file_bytes = tf.io.read_file(filepath)\n if ftype:\n audio = tfio.audio.decode_vorbis(file_bytes) # decode .ogg file\n else:\n audio = tfio.audio.decode_mp3(file_bytes) # decode .mp3 file\n audio = tf.cast(audio, tf.float32)\n if tf.shape(audio)[1]>1: # stereo -> mono\n audio = audio[...,0:1]\n audio = tf.squeeze(audio, axis=-1)\n return audio\n \n def crop_or_pad(audio, target_len, pad_mode='constant', take_first=True):\n audio_len = tf.shape(audio)[0]\n diff_len = abs(target_len - audio_len)\n if audio_len < target_len:\n pad1 = tf.random.uniform([], maxval=diff_len, dtype=tf.int32)\n pad2 = diff_len - pad1\n audio = tf.pad(audio, paddings=[[pad1, pad2]], mode=pad_mode)\n elif audio_len > target_len:\n if take_first:\n audio = audio[:target_len]\n else:\n idx = tf.random.uniform([], maxval=diff_len, dtype=tf.int32)\n audio = audio[idx: (idx + target_len)]\n return tf.reshape(audio, [target_len])\n\n def get_target(target): \n target = tf.reshape(target, [1])\n target = tf.cast(tf.one_hot(target, num_classes), tf.float32) \n target = tf.reshape(target, [num_classes])\n return target\n\n def decode(path):\n audio = get_audio(path)\n audio = crop_or_pad(audio, dim) # crop or pad audio to keep a fixed length\n audio = tf.reshape(audio, [dim])\n return audio\n \n def decode_with_labels(path, label):\n label = get_target(label)\n return decode(path), label\n \n return decode_with_labels if with_labels else decode","metadata":{"papermill":{"duration":0.251237,"end_time":"2022-03-08T03:18:49.079346","exception":false,"start_time":"2022-03-08T03:18:48.828109","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applies augmentation to audio\ndef audio_augmenter(with_labels=True, dim=CFG.audio_len, CFG=CFG):\n def augment(audio, dim=dim):\n if random_float() <= CFG.audio_augment_prob:\n audio = AudioAug(audio)\n audio = tf.reshape(audio, [dim])\n return audio\n \n def augment_with_labels(audio, label): \n return augment(audio), label\n \n return augment_with_labels if with_labels else augment","metadata":{"papermill":{"duration":0.250484,"end_time":"2022-03-08T03:18:49.79513","exception":false,"start_time":"2022-03-08T03:18:49.544646","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_dataset(paths, ftype, labels=None, batch_size=32, target_size=[128, 256], \n audio_decode_fn=None, audio_augment_fn=None,\n take_first=False, num_classes=264,\n cache=True, cache_dir=\"\",drop_remainder=False,\n augment=True, repeat=True, shuffle=1024):\n \"\"\"\n Creates a TensorFlow dataset from the given paths and labels.\n \n Args:\n paths (list): A list of file paths to the audio files.\n labels (list): A list of corresponding labels for the audio files.\n batch_size (int): Batch size for the created dataset.\n target_size (list): A list of target image size for the spectrograms.\n audio_decode_fn (function): A function to decode the audio file.\n audio_augment_fn (function): A function to augment the audio file.\n cache (bool): Whether to cache the dataset or not.\n cache_dir (str): Directory path to cache the dataset.\n drop_remainder (bool): Whether to drop the last batch if it is smaller than batch_size.\n augment (bool): Whether to augment the dataset or not.\n repeat (bool): Whether to repeat the dataset or not.\n shuffle (int): Number of elements from the dataset to buffer for shuffling.\n \n Returns:\n ds (tf.data.Dataset): A TensorFlow dataset.\n \"\"\"\n # Create cache directory if cache is enabled\n if cache_dir != \"\" and cache is True:\n os.makedirs(cache_dir, exist_ok=True)\n # Set default audio decode function if not provided\n if audio_decode_fn is None:\n audio_decode_fn = audio_decoder(labels is not None,\n dim=CFG.audio_len, \n take_first=take_first,\n num_classes=num_classes,\n CFG=CFG)\n # Set default audio augmentation function if not provided\n if audio_augment_fn is None:\n audio_augment_fn = audio_augmenter(labels is not None, \n dim=CFG.audio_len, CFG=CFG)\n \n # Set TensorFlow AUTOTUNE option\n AUTO = tf.data.experimental.AUTOTUNE\n # Create slices based on whether labels are provided\n slices = ((paths, ftype),) if labels is None else ((paths, ftype), labels)\n # Create TensorFlow dataset from slices\n ds = tf.data.Dataset.from_tensor_slices(slices)\n # Map audio decode function to dataset\n ds = ds.map(audio_decode_fn, num_parallel_calls=AUTO)\n # Cache dataset in memory if cache is enabled\n ds = ds.cache(cache_dir) if cache else ds\n # Repeat dataset indefinitely if repeat is enabled\n ds = ds.repeat() if repeat else ds\n # Create TensorFlow dataset options\n opt = tf.data.Options()\n # Shuffle dataset if shuffle is enabled\n if shuffle: \n ds = ds.shuffle(shuffle, seed=CFG.seed)\n opt.experimental_deterministic = False\n if CFG.device=='GPU':\n # If the device is a GPU, turn off auto-sharding to avoid performance issues\n opt.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n # Set the options for the dataset\n ds = ds.with_options(opt)\n # Apply audio augmentation to the dataset if augment is True\n ds = ds.map(audio_augment_fn, num_parallel_calls=AUTO) if augment else ds\n # Batch the dataset with the specified batch size\n ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n # Apply MixUp & CutMix regularization to the dataset\n if augment and labels is not None:\n ds = ds.map(CutMixUp,num_parallel_calls=AUTO)\n # Prefetch the next batch of data to improve performance\n ds = ds.prefetch(AUTO)\n return ds","metadata":{"papermill":{"duration":0.240881,"end_time":"2022-03-08T03:18:50.489717","exception":false,"start_time":"2022-03-08T03:18:50.248836","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization ","metadata":{}},{"cell_type":"code","source":"def plot_batch(batch, row=3, col=3, label2name=None,):\n \"\"\"Plot one batch data\"\"\"\n if isinstance(batch, tuple) or isinstance(batch, list):\n audios, tars = batch\n else:\n audios = batch\n tars = None\n plt.figure(figsize=(col*5, row*3))\n for idx in range(row*col):\n ax = plt.subplot(row, col, idx+1)\n plt.plot(audios[idx].numpy(), color=cmap(0.1))\n if tars is not None:\n label = tars[idx].numpy().argmax()\n name = label2name[label]\n plt.title(name)\n plt.tight_layout()\n plt.show()\n \n \ndef plot_history(history):\n \"\"\"Plot trainign history, credit: @cdeotte\"\"\"\n epochs = len(history.history['auc'])\n plt.figure(figsize=(15,5))\n plt.plot(np.arange(epochs),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n plt.plot(np.arange(epochs),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n plt.ylabel('AUC (PR)',size=14); plt.xlabel('Epoch',size=14)\n plt.legend(loc=2)\n plt2 = plt.gca().twinx()\n plt2.plot(np.arange(epochs),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n plt2.plot(np.arange(epochs),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n plt.ylabel('Loss',size=14)\n plt.title('Fold %i - Training Plot'%(fold+1),size=18)\n plt.legend(loc=3)\n plt.show() ","metadata":{"papermill":{"duration":0.328513,"end_time":"2022-03-08T03:19:59.512224","exception":false,"start_time":"2022-03-08T03:19:59.183711","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_Visualization=False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 20","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n check_df = df_pre.query(\"birdclef=='20'\").sample(50)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Xeno-Canto Extend","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n\n check_df = df_pre.query(\"birdclef=='xc'\").sample(5)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 21","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n\n check_df = df_pre.query(\"birdclef=='21'\").sample(5)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"papermill":{"duration":3.299334,"end_time":"2022-03-08T03:20:02.987986","exception":false,"start_time":"2022-03-08T03:19:59.688652","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 22","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n\n check_df = df_pre.query(\"birdclef=='22'\").sample(5)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 23","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n\n check_df = df_pre.query(\"birdclef=='23'\").sample(5)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BirdCLEF - 24","metadata":{}},{"cell_type":"code","source":"if show_Visualization:\n \n check_df = df_24.sample(50)\n check_df['target'] = check_df['target'].astype(int)\n ds = build_dataset(check_df.filepath.tolist(),\n check_df.filepath.str.contains('.ogg').tolist(),\n check_df.target.tolist(), \n num_classes=CFG.num_classes2,\n augment=True, cache=False)\n ds = ds.take(32)\n audios, labels = next(iter(ds))\n plot_batch((audios, labels), label2name=CFG.label2name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Layers ","metadata":{}},{"cell_type":"code","source":"melspec_layer = MelSpectrogram(n_fft=CFG.nfft, \n hop_length=CFG.hop_length, \n sr=CFG.sample_rate, \n ref=1.0,\n fmin=500,\n fmax=15000,\n out_channels=3)\nspecs = melspec_layer(audios)\n\nfig, ax = plt.subplots(2, 1, sharex=True, figsize=(12, 5))\n#librosa.display.waveshow(audios[0].numpy(), sr=CFG.sample_rate, ax=ax[0], axis=None)\nlibrosa.display.waveshow(audios[0].numpy(), sr=CFG.sample_rate, ax=ax[0], color=\"blue\")\n\nlibrosa.display.specshow(specs[0, ..., 0].numpy(), \n n_fft=CFG.nfft, \n hop_length=CFG.hop_length, \n sr=CFG.sample_rate,\n x_axis='time',\n y_axis='mel',\n cmap='coolwarm',\n ax=ax[1])\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfm_layer = TimeFreqMask(freq_mask_prob=0.65,\n num_freq_masks=2,\n freq_mask_param=10,\n time_mask_prob=0.65,\n num_time_masks=3,\n time_mask_param=25,\n time_last=True,)\nspecs2 = tfm_layer(specs, training=True)\n\nplt.figure(figsize=(12,3))\nlid.specshow(specs2[0, ..., 0].numpy(), \n n_fft=CFG.nfft, \n hop_length=CFG.hop_length, \n sr=CFG.sample_rate,\n x_axis='time',\n y_axis='mel',\n cmap='coolwarm')\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_layer = ZScoreMinMax()\nspecs3 = norm_layer(specs2)\n\nplt.figure(figsize=(8,3))\nplt.hist(specs2.numpy().ravel(), alpha=0.8, color=cmap(0.1))\nplt.hist(specs3.numpy().ravel(), alpha=0.8, color=cmap(0.9))\nplt.legend([\"w/o normalize\", \"w/ normalize\"])\nplt.semilogx()\nplt.title(\"Effect of Normalization\")\nplt.xlabel(\"Pixel Value\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{"papermill":{"duration":0.184301,"end_time":"2022-03-08T03:20:04.031695","exception":false,"start_time":"2022-03-08T03:20:03.847394","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sklearn.metrics\n\ndef get_metrics():\n# acc = tf.keras.metrics.BinaryAccuracy(name='acc')\n auc = tf.keras.metrics.AUC(curve='PR', name='auc', multi_label=False) # auc on prcision-recall curve\n acc = tf.keras.metrics.CategoricalAccuracy(name='acc')\n return [acc, auc]\n\ndef padded_cmap(y_true, y_pred, padding_factor=5):\n num_classes = y_true.shape[1]\n pad_rows = np.array([[1]*num_classes]*padding_factor)\n y_true = np.concatenate([y_true, pad_rows])\n y_pred = np.concatenate([y_pred, pad_rows])\n score = sklearn.metrics.average_precision_score(y_true, y_pred, average='macro',)\n return score\n\ndef get_loss():\n if CFG.loss==\"CCE\":\n loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=CFG.label_smoothing)\n elif CFG.loss==\"BCE\":\n loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=CFG.label_smoothing)\n else:\n raise ValueError(\"Loss not found\")\n return loss\n \ndef get_optimizer():\n if CFG.optimizer == \"Adam\":\n opt = tf.keras.optimizers.Adam(learning_rate=CFG.lr)\n else:\n raise ValueError(\"Optmizer not found\")\n return opt","metadata":{"papermill":{"duration":0.28125,"end_time":"2022-03-08T03:20:04.498883","exception":false,"start_time":"2022-03-08T03:20:04.217633","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling ","metadata":{"papermill":{"duration":0.182769,"end_time":"2022-03-08T03:20:04.861966","exception":false,"start_time":"2022-03-08T03:20:04.679197","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\n# Will download and load pretrained imagenet weights.\ndef build_model(CFG, model_name=None, num_classes=264, compile_model=True):\n \"\"\"\n Builds and returns a model based on the specified configuration.\n \"\"\"\n # Create an input layer for the model\n inp = tf.keras.layers.Input(shape=(None,))\n # Spectrogram\n out = MelSpectrogram(n_mels=CFG.img_size[0],\n n_fft=CFG.nfft,\n hop_length=CFG.hop_length, \n sr=CFG.sample_rate,\n ref=1.0,\n out_channels=3)(inp)\n # Normalize\n out = ZScoreMinMax()(out)\n # TimeFreqMask\n out = TimeFreqMask(freq_mask_prob=0.5,\n num_freq_masks=1,\n freq_mask_param=10,\n time_mask_prob=0.5,\n num_time_masks=2,\n time_mask_param=25,\n time_last=False,)(out)\n # Load backbone model\n base = getattr(efn, model_name)(input_shape=(None, None, 3),\n include_top=0,\n weights=CFG.pretrain,\n fsr=CFG.fsr)\n # Pass the input through the base model\n out = base(out)\n out = tf.keras.layers.GlobalAveragePooling2D()(out)\n out = tf.keras.layers.Dense(num_classes, activation='softmax')(out)\n model = tf.keras.models.Model(inputs=inp, outputs=out)\n if compile_model:\n # Set the optimizer\n opt = get_optimizer()\n # Set the loss function\n loss = get_loss()\n # Set the evaluation metrics\n metrics = get_metrics()\n # Compile the model with the specified optimizer, loss function, and metrics\n model.compile(optimizer=opt, loss=loss, metrics=metrics)\n return model","metadata":{"papermill":{"duration":1.239321,"end_time":"2022-03-08T03:20:06.281118","exception":false,"start_time":"2022-03-08T03:20:05.041797","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(CFG, model_name=CFG.model_name)\nmodel.summary()","metadata":{"papermill":{"duration":37.756883,"end_time":"2022-03-08T03:20:44.226871","exception":false,"start_time":"2022-03-08T03:20:06.469988","status":"completed"},"tags":[],"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audios = tf.random.uniform((1, CFG.audio_len))\nwith strategy.scope():\n out = model(audios, training=False)\nprint(out.shape)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LR ","metadata":{}},{"cell_type":"code","source":"import math\n\ndef get_lr_callback(batch_size=8, mode='cos', epochs=CFG.epochs, plot=False):\n \"\"\"\n Returns a learning rate scheduler callback for a given batch size, mode, and number of epochs.\n \"\"\"\n # Define the learning rate schedule.\n lr_start = 0.000005\n lr_max = 0.00000140 * batch_size\n lr_min = 0.000001\n lr_ramp_ep = 5\n lr_sus_ep = 0\n lr_decay = 0.8\n \n # Function to update the lr\n def lrfn(epoch):\n if epoch < lr_ramp_ep:\n lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n elif epoch < lr_ramp_ep + lr_sus_ep:\n lr = lr_max\n\n elif CFG.scheduler == 'exp':\n lr = (lr_max - lr_min) * lr_decay**(epoch - \\\n lr_ramp_ep - lr_sus_ep) + lr_min\n\n elif CFG.scheduler == 'step':\n lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n\n elif CFG.scheduler == 'cos':\n decay_total_epochs = epochs - lr_ramp_ep - lr_sus_ep + 3\n decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n phase = math.pi * decay_epoch_index / decay_total_epochs\n cosine_decay = 0.5 * (1 + math.cos(phase))\n lr = (lr_max - lr_min) * cosine_decay + lr_min\n return lr\n \n # Plot the lr curve\n if plot:\n plt.figure(figsize=(10,5))\n plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n plt.xlabel('epoch'); plt.ylabel('learnig rate')\n plt.title('Learning Rate Scheduler')\n plt.show()\n \n # Crate lr-callback to update lr during training\n lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n return lr_callback","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.510014,"end_time":"2022-03-08T03:20:45.290695","exception":false,"start_time":"2022-03-08T03:20:44.780681","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_=get_lr_callback(CFG.batch_size*CFG.replicas, plot=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\n\ndef wandb_init(fold):\n \"\"\"\n Initializes the W&B run by creating a config file and initializing a W&B run.\n \"\"\"\n # Create a dictionary of configuration parameters\n config = {k:v for k,v in dict(vars(CFG)).items() if '__' not in k}\n config.update({\"fold\":int(fold)}) # int is to convert numpy.int -> int\n # Dump the configuration dictionary to a YAML file\n yaml.dump(config, open(f'/kaggle/working/config fold-{fold}.yaml', 'w'),)\n # Load the configuration dictionary from the YAML file\n config = yaml.load(open(f'/kaggle/working/config fold-{fold}.yaml', 'r'), Loader=yaml.FullLoader)\n # Initialize a W&B run with the given configuration parameters\n run = wandb.init(project=\"birdclef-2023-public\",\n name=f\"fold-{fold}|dim-{CFG.img_size[1]}x{CFG.img_size[0]}|model-{CFG.model_name}\",\n config=config,\n group=CFG.comment,\n save_code=True,)\n return run\n\n \ndef log_wandb(valid_df):\n \"\"\"Log and save validation results with missclassified examples as audio in W&B\"\"\"\n # Query only the rows with miss predictions\n save_df = valid_df.query(\"miss==True\")\n # Map the predicted and target labels to their corresponding names\n save_df.loc[:, 'pred_name'] = save_df.pred.map(CFG.label2name)\n save_df.loc[:, 'target_name'] = save_df.target.map(CFG.label2name)\n # Trim the dataframe for debugging purposes\n if CFG.debug:\n save_df = save_df.iloc[:CFG.replicas*CFG.batch_size*CFG.infer_bs]\n # Get the columns to be included in the wandb table\n noimg_cols = [*CFG.tab_cols, 'target', 'pred', 'target_name','pred_name']\n # Retain only the necessary columns\n save_df = save_df.loc[:, noimg_cols]\n\n data = []\n # Load audio files for each miss prediction\n for idx, row in tqdm(save_df.iterrows(), total=len(save_df), desc='wandb ', position=0, leave=True):\n filepath = '/kaggle/input/birdclef-2023/train_audio/'+CFG.label2name[row.target]+'/'+row.filename\n audio, sr = librosa.load(filepath, sr=None)\n # Add the audio file to the data list along with the other relevant information\n data+=[[*row.tolist(), wandb.Audio(audio, caption=row.filename, sample_rate=sr)]]\n # Create a wandb table with the audio files and other relevant information\n wandb_table = wandb.Table(data=data, columns=[*noimg_cols, 'audio'])\n # Manually unpack dict values\n scores_wb = {f'best.{k}': v for k,v in scores.items()}\n # Log the scores and wandb table to wandb\n wandb.log({**scores_wb,\n 'table': wandb_table,\n })\n \n# get wandb callbacks\ndef get_wb_callbacks(fold):\n wb_ckpt = wandb.keras.WandbModelCheckpoint(filepath='fold-%i.h5'%fold, \n monitor='val_auc',\n verbose=CFG.verbose,\n save_best_only=True,\n save_weights_only=False,\n mode='max',)\n wb_metr = wandb.keras.WandbMetricsLogger()\n return [wb_ckpt, wb_metr]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.288344,"end_time":"2022-03-08T03:20:47.977099","exception":false,"start_time":"2022-03-08T03:20:47.688755","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-Training ","metadata":{"papermill":{"duration":0.182448,"end_time":"2022-03-08T03:20:48.340679","exception":false,"start_time":"2022-03-08T03:20:48.158231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# print(f'valid_paths shape: {len(valid_paths)}')\n# print(f'valid_ftype shape: {len(valid_ftype)}')\n# print(f'valid_labels shape: {len(valid_labels)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configurations\nnum_classes = CFG.num_classes2\ndf = df_pre.copy()\nfold = 0\n\n# Compute batch size and number of samples to drop\ninfer_bs = (CFG.batch_size*CFG.infer_bs)\ndrop_remainder = CFG.drop_remainder\n\n# Split dataset with cv filter\nif CFG.cv_filter:\n df = filter_data(df, thr=5)\n train_df = df.query(\"fold!=@fold | ~cv\").reset_index(drop=True)\n valid_df = df.query(\"fold==@fold & cv\").reset_index(drop=True)\nelse:\n train_df = df.query(\"fold!=@fold\").reset_index(drop=True)\n valid_df = df.query(\"fold==@fold\").reset_index(drop=True)\n\n# Upsample train data\ntrain_df = upsample_data(train_df, thr=50)\ntrain_df = downsample_data(train_df, thr=500)\n\n# Get file paths and labels\ntrain_paths = train_df.filepath.values; train_labels = train_df.target.values\nvalid_paths = valid_df.filepath.values; valid_labels = valid_df.target.values\n\n# Shuffle the file paths and labels\nindex = np.arange(len(train_paths))\nnp.random.shuffle(index)\ntrain_paths = train_paths[index]\ntrain_labels = train_labels[index]\n\n# For debugging\nif CFG.debug:\n min_samples = CFG.batch_size*CFG.replicas*2\n train_paths = train_paths[:min_samples]; train_labels = train_labels[:min_samples]\n valid_paths = valid_paths[:min_samples]; valid_labels = valid_labels[:min_samples]\n \n# Ogg or Mp3\n# train_ftype = list(map(lambda x: '.ogg' in x, train_paths))\n# valid_ftype = list(map(lambda x: '.ogg' in x, valid_paths))\ntrain_ftype = list(map(lambda x: '.ogg' in x or '.mp3' in x, train_paths))\nvalid_ftype = list(map(lambda x: '.ogg' in x or '.mp3' in x, valid_paths))\n\n# Compute the number of training and validation samples\nnum_train = len(train_paths); num_valid = len(valid_paths)\n\n# Build the training and validation datasets\ncache=False\ntrain_ds = build_dataset(train_paths, train_ftype, train_labels, \n batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=True,\n drop_remainder=drop_remainder, num_classes=num_classes)\nvalid_ds = build_dataset(valid_paths, valid_ftype, valid_labels,\n batch_size=CFG.batch_size*CFG.replicas, cache=True, shuffle=False,\n augment=False, repeat=False, drop_remainder=drop_remainder,\n take_first=True, num_classes=num_classes)\n\n# Print information about the fold and training\nprint('#'*25); print('#### Pre-Training')\nprint('#### Image Size: (%i, %i) | Model: %s | Batch Size: %i | Scheduler: %s'%\n (*CFG.img_size, CFG.model_name, CFG.batch_size*CFG.replicas, CFG.scheduler))\nprint('#### Num Train: {:,} | Num Valid: {:,}'.format(len(train_paths), len(valid_paths)))\n\n# Clear the session and build the model\nK.clear_session()\nwith strategy.scope():\n model = build_model(CFG, model_name=CFG.model_name, num_classes=num_classes)\n #model.load_weights(\"/kaggle/input/birdclef24-pretraining-is-all-you-need-train/birdclef_pretrained_ckpt.keras\", skip_mismatch=True)\n print('#'*25) \n\n # Checkpoint Callback\n ckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n 'birdclef_pretrained_ckpt.keras', monitor='val_auc', verbose=0, save_best_only=True,\n save_weights_only=False, mode='max', save_freq='epoch')\n\n\n # LR Scheduler Callback\n lr_cb = get_lr_callback(CFG.batch_size*CFG.replicas)\n callbacks = [ckpt_cb, lr_cb]\n\n # Training\n history = model.fit(\n train_ds, \n epochs=2 if CFG.debug else CFG.epochs, \n callbacks=callbacks, \n steps_per_epoch=len(train_paths)//CFG.batch_size//CFG.replicas,\n validation_data=valid_ds, \n verbose=CFG.verbose,\n )\n\n# Show training plot\nif CFG.training_plot:\n plot_history(history)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"papermill":{"duration":3436.548849,"end_time":"2022-03-08T04:18:05.540592","exception":true,"start_time":"2022-03-08T03:20:48.991743","status":"failed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"oof_pred = []; oof_true = []; oof_val = []; oof_ids = []; oof_folds = [] \n\nnum_classes = CFG.num_classes\ndf = df_24.copy()\nfor fold in range(CFG.num_fold):\n # Check if the fold is selected\n if fold not in CFG.selected_folds:\n continue\n \n # Initialize Weights and Biases\n if CFG.wandb:\n run = wandb_init(fold)\n \n # Compute batch size and number of samples to drop\n infer_bs = (CFG.batch_size*CFG.infer_bs)\n drop_remainder = CFG.drop_remainder\n \n # Split dataset with cv filter\n if CFG.cv_filter:\n df = filter_data(df, thr=5)\n train_df = df.query(\"fold!=@fold | ~cv\").reset_index(drop=True)\n valid_df = df.query(\"fold==@fold & cv\").reset_index(drop=True)\n else:\n train_df = df.query(\"fold!=@fold\").reset_index(drop=True)\n valid_df = df.query(\"fold==@fold\").reset_index(drop=True)\n \n # Upsample train data\n train_df = upsample_data(train_df, thr=50)\n# train_df = downsample_data(train_df, thr=500)\n\n # Get file paths and labels\n train_paths = train_df.filepath.values; train_labels = train_df.target.values\n valid_paths = valid_df.filepath.values; valid_labels = valid_df.target.values\n\n # Shuffle the file paths and labels\n index = np.arange(len(train_paths))\n np.random.shuffle(index)\n train_paths = train_paths[index]\n train_labels = train_labels[index]\n \n # For debugging\n if CFG.debug:\n min_samples = CFG.batch_size*CFG.replicas*2\n train_paths = train_paths[:min_samples]; train_labels = train_labels[:min_samples]\n valid_paths = valid_paths[:min_samples]; valid_labels = valid_labels[:min_samples]\n\n # Ogg or Mp3\n train_ftype = list(map(lambda x: '.ogg' in x, train_paths))\n valid_ftype = list(map(lambda x: '.ogg' in x, valid_paths))\n\n # Compute the number of training and validation samples\n num_train = len(train_paths); num_valid = len(valid_paths)\n \n # Log the number of training and validation samples if Weights and Biases is being used\n if CFG.wandb:\n wandb.log({'num_train':num_train,\n 'num_valid':num_valid})\n \n # Build the training and validation datasets\n cache=True\n train_ds = build_dataset(train_paths, train_ftype, train_labels, \n batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=True,\n drop_remainder=drop_remainder, num_classes=num_classes)\n valid_ds = build_dataset(valid_paths, valid_ftype, valid_labels,\n batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=False,\n augment=False, repeat=False, drop_remainder=drop_remainder,\n take_first=True, num_classes=num_classes)\n \n # Print information about the fold and training\n print('#'*25); print('#### Training')\n print('#### Fold: %i | Image Size: (%i, %i) | Model: %s | Batch Size: %i | Scheduler: %s'%\n (fold+1, *CFG.img_size, CFG.model_name, CFG.batch_size*CFG.replicas, CFG.scheduler))\n print('#### Num Train: {:,} | Num Valid: {:,}'.format(len(train_paths), len(valid_paths)))\n \n # Clear the session and build the model\n K.clear_session()\n with strategy.scope():\n model = build_model(CFG, model_name=CFG.model_name, num_classes=num_classes)\n # Load birdclef pretrained weights\n model.load_weights(\"birdclef_pretrained_ckpt.keras\", skip_mismatch=True)\n\n print('#'*25) \n\n # Callbacks\n ckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n 'fold-%i.keras'%fold, monitor='val_auc', verbose=0, save_best_only=True,\n save_weights_only=False, mode='max', save_freq='epoch')\n # LR Scheduler Callback\n lr_cb = get_lr_callback(CFG.batch_size*CFG.replicas)\n callbacks = [ckpt_cb, lr_cb]\n # WandB Callbacks\n #if CFG.wandb:\n # wb_cb = get_wb_callbacks(fold)\n # callbacks+=[wb_cb]\n\n # Training\n history = model.fit(\n train_ds, \n epochs=2 if CFG.debug else CFG.epochs, \n callbacks=callbacks, \n steps_per_epoch=len(train_paths)//CFG.batch_size//CFG.replicas,\n validation_data=valid_ds, \n verbose=CFG.verbose,\n )\n \"\"\"\n # Load best checkpoint\n print('# Loading best model')\n model.load_weights('fold-%i.keras'%fold)\n\n # Predict on the validation data for oof result\n print('# Infering OOF')\n valid_ds = build_dataset(valid_paths, valid_ftype, labels=None, augment=CFG.tta>1, repeat=True, cache=False, \n shuffle=False, batch_size=infer_bs*CFG.replicas, \n drop_remainder=drop_remainder, take_first=True, num_classes=num_classes)\n ct_valid = len(valid_paths); STEPS = CFG.tta * ct_valid//infer_bs//CFG.replicas\n pred = model.predict(valid_ds,steps=STEPS,verbose=CFG.verbose)[:CFG.tta*ct_valid,] \n pred = np.mean(pred.reshape((CFG.tta,ct_valid,-1)),axis=0)\n oof_pred.append(pred) \n\n # Get ids and targets\n oof_true.append(valid_df[CFG.target_col].values[:ct_valid])\n oof_folds.append(np.ones_like(oof_true[-1],dtype='int8')*fold )\n oof_ids.append(valid_paths)\n\n # Save valid data prediction\n y_true = oof_true[-1].reshape(-1).astype('float32')\n y_pred = oof_pred[-1].argmax(axis=-1)\n valid_df.loc[:num_valid - 1, 'pred'] = y_pred\n valid_df.loc[:num_valid - 1, 'miss'] = y_true != y_pred\n valid_df.loc[:num_valid - 1, CFG.class_names] = oof_pred[-1].tolist()\n\n # Log the metrics\n scores = {}\n cmAP = padded_cmap(tf.keras.utils.to_categorical(y_true), oof_pred[-1])\n best_epoch = np.argmax(history.history['val_'+CFG.monitor], axis=-1) + 1\n best_score = history.history['val_'+CFG.monitor][best_epoch - 1]\n scores.update({'auc': best_score,\n 'epoch': best_epoch,\n 'cmAP': cmAP,})\n oof_val.append(best_score)\n print('\\n>>> FOLD %i OOF AUC = %.3f | Padded_cmAP = %.3f' % (fold+1, oof_val[-1], cmAP))\n\n # Show training plot\n if CFG.training_plot:\n plot_history(history)\n\n # Log metrics, media to wandb\n #if CFG.wandb:\n # print('# WandB')\n # log_wandb(valid_df)\n # wandb.run.finish()\n # display(ipd.IFrame(run.url, width=1080, height=720))\n \"\"\"","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def get_id(row):\n# row['filename'] = row['filepath'].split('/',5)[-1]\n# return row","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# OOF Data\ny_pred = np.concatenate(oof_pred); y_true = np.concatenate(oof_true);\nids = np.concatenate(oof_ids); folds = np.concatenate(oof_folds)\n\n# Overall cmAP\ncmAP = padded_cmap(tf.keras.utils.to_categorical(y_true), y_pred)\n\n# Overall AUC in PR curve\nm = tf.keras.metrics.AUC(curve='PR')\nm.update_state(tf.keras.utils.to_categorical(y_true), y_pred)\nauc = m.result().numpy()\n\nprint('>>> Overall cmAP: ', cmAP)\nprint('>>> Overall AUC(PR): ', auc)\"\"\"","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# Save OOF data to disk\ncolumns = ['filepath', 'fold', 'true', 'pred', *CFG.class_names]\ndf_oof = pd.DataFrame(np.concatenate([ids[:,None], folds, y_true,\n np.argmax(y_pred,axis=1)[:,None], y_pred], axis=1), columns=columns)\ndf_oof['class_name'] = df_oof.true.map(CFG.label2name)\ndf_oof['miss'] = df_oof.true!=df_oof.pred\ntqdm.pandas(desc='id ')\ndf_oof = df_oof.progress_apply(get_id,axis=1)\ndf_oof.to_csv('oof.csv',index=False)\ndisplay(df_oof.head(2))\"\"\"","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"print('Miss Total:')\ndisplay(df_oof.query(\"miss==True\").shape[0])\n\nprint()\nprint('Miss Distribution Top10:')\ndisplay(df_oof.query(\"miss==True\").class_name.value_counts()[:10])\"\"\"","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference ✍️\n* [Fake Speech Detection: Conformer [TF]](https://www.kaggle.com/code/awsaf49/fake-speech-detection-conformer-tf/) by @awsaf49\n* [RANZCR: EfficientNet TPU Training](https://www.kaggle.com/code/xhlulu/ranzcr-efficientnet-tpu-training) by @xhlulu\n* [Triple Stratified KFold with TFRecords](https://www.kaggle.com/code/cdeotte/triple-stratified-kfold-with-tfrecords) by @cdeotte","metadata":{}},{"cell_type":"markdown","source":"# Remove Files ✂️","metadata":{}},{"cell_type":"code","source":"\"\"\"import shutil\ntry:\n !rm -rf ./wandb\nexcept:\n pass\"\"\"","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}