{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # Tensorflow for tpu-vm\n# !pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n\n# # Tensorflow utilities\n!pip install -q tensorflow-addons==0.19.0\n!pip install -q tensorflow-probability==0.19.0\n!pip install -q tensorflow-io==0.32.0\n\n# # Other utilies not available on tpu-vm\n!pip install -q opencv-python-headless\n!pip install -q librosa\n!pip install -q scikit-learn\n\n# # WandB for Experiment tracking\n!pip install -qU wandb","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-26T19:39:28.459213Z","iopub.execute_input":"2023-04-26T19:39:28.460120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For Spectrogram, SpecAug layers\n!pip install -qU tensorflow_extra --no-deps\n\n# efficientnet with filter stride reduction (FSR)\n!pip install -qU git+https://github.com/awsaf49/efficientnet-spec","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# os.environ['TPU_LOG_DIR'] = '/kaggle/working'\n\nimport pandas as pd\npd.options.mode.chained_assignment = None # avoids assignment warning\nimport numpy as np\nimport random\nfrom glob import glob\nfrom tqdm import tqdm\ntqdm.pandas()  # enable progress bars in pandas operations\nimport gc\n\nimport librosa\nimport sklearn\nimport json\n\n# Import for visualization\nimport matplotlib as mpl\ncmap = mpl.cm.get_cmap('coolwarm')\nimport matplotlib.pyplot as plt\nimport librosa.display as lid\nimport IPython.display as ipd\nimport cv2\n\n# Import tensorflow\nimport tensorflow as tf\n# Set logging level to avoid unnecessary messages\ntf.get_logger().setLevel('ERROR')\n# Set autograph verbosity to avoid unnecessary messages\ntf.autograph.set_verbosity(0)\n# Enable xla for speed up\n# tf.config.optimizer.set_jit(False) #  throws error for time-freq-mask\n\n# Import required tensorflow modules\nimport tensorflow_io as tfio\nimport tensorflow_addons as tfa\nimport tensorflow_probability as tfp\nimport tensorflow.keras.backend as K\n\n# Import KaggleDatasets for accessing Kaggle datasets\nfrom kaggle_datasets import KaggleDatasets\n\n# WandB for experiment tracking\nimport wandb","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.632068,"end_time":"2022-03-08T03:18:14.585094","exception":false,"start_time":"2022-03-08T03:18:11.953026","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('librosa:', librosa.__version__)\nprint('tf:', tf.__version__)\nprint('tfp:', tfp.__version__)\nprint('tfa:', tfa.__version__)\nprint('tfio:', tfio.__version__)\nprint('w&b:', wandb.__version__)","metadata":{"papermill":{"duration":0.155095,"end_time":"2022-03-08T03:18:14.939054","exception":false,"start_time":"2022-03-08T03:18:14.783959","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    # Debugging\n    debug = False\n    \n    # Verbosity level\n    verbose = 0\n    \n    # Plot training history\n    training_plot = True\n    \n    # Weights and Biases logging\n    wandb = True\n    competition   = 'birdclef-2023' \n    _wandb_kernel = 'awsaf49'\n    \n    # Experiment name and comment\n    exp_name = 'birdclef-pretrain-v3'\n    comment = 'EfficientNetB1|No-FSR|t=10s|128x384|cutmix'\n    # Notebook link\n    notebook_link = 'https://www.kaggle.com/awsaf49/birdclef23-pretraining-is-all-you-need-train'\n    \n    # Device and random seed\n    device = 'TPU-VM'\n    seed = 42\n    \n\n    # Input image size and batch size\n    img_size = [128, 384]\n    batch_size = 32\n    upsample_thr = 50 # min sample of each class (upsample)\n    cv_filter = True # always keeps low sample data in train\n    \n    # Audio duration, sample rate, and length\n    duration = 10 # second\n    sample_rate = 32000\n    audio_len = duration*sample_rate\n    \n    # STFT parameters\n    nfft = 2028\n    window = 2048\n    hop_length = audio_len // (img_size[1] - 1)\n    fmin = 20\n    fmax = 16000\n    normalize = True\n    \n    # Inference batch size, test time augmentation, and drop remainder\n    infer_bs = 2\n    tta = 1\n    drop_remainder = True\n    \n    # Number of epochs, model name, and number of folds\n    epochs = 25\n    model_name = 'EfficientNetB1'\n    fsr = False # reduce stride of stem block\n    num_fold = 5\n    \n    # Selected folds for training and evaluation\n    selected_folds = [0]\n\n    # Pretraining, neck features, and final activation function\n    pretrain = 'imagenet'\n    neck_features = 0\n    final_act = 'softmax'\n    \n    # Learning rate, optimizer, and scheduler\n    lr = 1e-3\n    scheduler = 'cos'\n    optimizer = 'Adam' # AdamW, Adam\n    \n    # Loss function and label smoothing\n    loss = 'CCE' # BCE, CCE\n    label_smoothing = 0.05 # label smoothing\n    \n    # Data augmentation parameters\n    augment=True\n    \n    # Time Freq masking\n    freq_mask_prob=0.50\n    num_freq_masks=1\n    freq_mask_param=10\n    time_mask_prob=0.50\n    num_time_masks=2\n    time_mask_param=25\n\n    # Audio Augmentation Settings\n    audio_augment_prob = 0.5\n    \n    mixup_prob = 0.65\n    mixup_alpha = 0.5\n    \n    cutmix_prob = 0.65\n    cutmix_alpha = 2.5\n    \n    timeshift_prob = 0.0\n    \n    gn_prob = 0.35\n\n    # Class Labels for BirdCLEF 23\n    class_names = sorted(os.listdir('/kaggle/input/birdclef-2023/train_audio/'))\n    num_classes = len(class_names)\n    class_labels = list(range(num_classes))\n    label2name = dict(zip(class_labels, class_names))\n    name2label = {v:k for k,v in label2name.items()}\n    \n    # Class Labels for BirdCLEF 21 & 22\n    class_names2 = sorted(set(os.listdir('/kaggle/input/birdclef-2021/train_short_audio/')\n                       +os.listdir('/kaggle/input/birdclef-2022/train_audio/')\n                       +os.listdir('/kaggle/input/birdsong-recognition/train_audio/')))\n    num_classes2 = len(class_names2)\n    class_labels2 = list(range(num_classes2))\n    label2name2 = dict(zip(class_labels2, class_names2))\n    name2label2 = {v:k for k,v in label2name2.items()}\n\n    # Training Settings\n    target_col = ['target']\n    tab_cols = ['filename']\n    monitor = 'auc'","metadata":{"papermill":{"duration":0.156464,"end_time":"2022-03-08T03:18:18.322809","exception":false,"start_time":"2022-03-08T03:18:18.166345","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.set_random_seed(CFG.seed)","metadata":{"papermill":{"duration":0.153451,"end_time":"2022-03-08T03:18:18.685056","exception":false,"start_time":"2022-03-08T03:18:18.531605","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import wandb library for logging and tracking experiments\nimport wandb\n\n# Try to get the API key from Kaggle secrets\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"WANDB\")\n    # Login to wandb with the API key\n    wandb.login(key=api_key)\n    # Set anonymous mode to None\n    anonymous = None\nexcept:\n    # If Kaggle secrets are not available, set anonymous mode to 'must'\n    anonymous = 'must'\n    # Login to wandb anonymously and relogin if needed\n    wandb.login(anonymous=anonymous, relogin=True)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_device():\n    \"Detect and intializes GPU/TPU automatically\"\n    # Check TPU category\n    tpu = 'local' if CFG.device=='TPU-VM' else None\n    try:\n        # Connect to TPU\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu) \n        # Set TPU strategy\n        strategy = tf.distribute.TPUStrategy(tpu)\n        print(f'> Running on {CFG.device}', tpu.master(), end=' | ')\n        print('Num of TPUs: ', strategy.num_replicas_in_sync)\n        device=CFG.device\n    except:\n        # If TPU is not available, detect GPUs\n        gpus = tf.config.list_logical_devices('GPU')\n        ngpu = len(gpus)\n         # Check number of GPUs\n        if ngpu:\n            # Set GPU strategy\n            strategy = tf.distribute.MirroredStrategy(gpus) # single-GPU or multi-GPU\n            # Print GPU details\n            print(\"> Running on GPU\", end=' | ')\n            print(\"Num of GPUs: \", ngpu)\n            device='GPU'\n        else:\n            # If no GPUs are available, use CPU\n            print(\"> Running on CPU\")\n            strategy = tf.distribute.get_strategy()\n            device='CPU'\n    return strategy, device, tpu","metadata":{"_kg_hide-input":true,"papermill":{"duration":7.941725,"end_time":"2022-03-08T03:18:26.826553","exception":false,"start_time":"2022-03-08T03:18:18.884828","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize GPU/TPU/TPU-VM\nstrategy, CFG.device, tpu = get_device()\nCFG.replicas = strategy.num_replicas_in_sync","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH0 = '/kaggle/input/birdsong-recognition'\nBASE_PATH1 = '/kaggle/input/birdclef-2021'\nBASE_PATH2 = '/kaggle/input/birdclef-2022'\nBASE_PATH3 = '/kaggle/input/birdclef-2023'\nBASE_PATH4 = '/kaggle/input/xeno-canto-bird-recordings-extended-a-m'\nBASE_PATH5 = '/kaggle/input/xeno-canto-bird-recordings-extended-n-z'\n\nif CFG.device==\"TPU\":\n    from kaggle_datasets import KaggleDatasets\n    GCS_PATH0 = KaggleDatasets().get_gcs_path(BASE_PATH0.split('/')[-1])\n    GCS_PATH1 = KaggleDatasets().get_gcs_path(BASE_PATH1.split('/')[-1])\n    GCS_PATH2 = KaggleDatasets().get_gcs_path(BASE_PATH2.split('/')[-1])\n    GCS_PATH3 = KaggleDatasets().get_gcs_path(BASE_PATH3.split('/')[-1])\n    GCS_PATH4 = KaggleDatasets().get_gcs_path(BASE_PATH4.split('/')[-1])\n    GCS_PATH5 = KaggleDatasets().get_gcs_path(BASE_PATH5.split('/')[-1])\nelse:\n    GCS_PATH0 = BASE_PATH0\n    GCS_PATH1 = BASE_PATH1\n    GCS_PATH2 = BASE_PATH2\n    GCS_PATH3 = BASE_PATH3\n    GCS_PATH4 = BASE_PATH4\n    GCS_PATH5 = BASE_PATH5","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_23 = pd.read_csv(f'{BASE_PATH3}/train_metadata.csv')\ndf_23['filepath'] = GCS_PATH3 + '/train_audio/' + df_23.filename\ndf_23['target'] = df_23.primary_label.map(CFG.name2label)\ndf_23['birdclef'] = '23'\ndf_23['filename'] = df_23.filepath.map(lambda x: x.split('/')[-1])\ndf_23['xc_id'] = df_23.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nassert tf.io.gfile.exists(df_23.filepath.iloc[0])\n\n# Display rwos\nprint(\"# Samples in BirdCLEF 23: {:,}\".format(len(df_23)))\ndf_23.head(2).style.set_caption(\"BirdCLEF - 23\").set_table_styles([{\n    'selector': 'caption',\n    'props': [\n        ('color', 'blue'),\n        ('font-size', '16px')\n    ]\n}])","metadata":{"papermill":{"duration":0.241649,"end_time":"2022-03-08T03:18:27.408813","exception":false,"start_time":"2022-03-08T03:18:27.167164","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BirdCLEF-2020\ndf_20 = pd.read_csv(f'{BASE_PATH0}/train.csv')\ndf_20['primary_label'] = df_20['ebird_code']\ndf_20['filepath'] = GCS_PATH0 + '/train_audio/' + df_20.primary_label + '/' + df_20.filename\ndf_20['scientific_name'] = df_20['sci_name']\ndf_20['common_name'] = df_20['species']\ndf_20['target'] = df_20.primary_label.map(CFG.name2label2)\ndf_20['birdclef'] = '20'\nassert tf.io.gfile.exists(df_20.filepath.iloc[0])\n\n# Xeno-Canto Extend by @vopani\ndf_xam = pd.read_csv(f'{BASE_PATH4}/train_extended.csv')\ndf_xam['filepath'] = GCS_PATH4 + '/A-M/' + df_xam.ebird_code + '/' + df_xam.filename\ndf_xnz = pd.read_csv(f'{BASE_PATH5}/train_extended.csv')\ndf_xnz['filepath'] = GCS_PATH5 + '/N-Z/' + df_xnz.ebird_code + '/' + df_xnz.filename\ndf_xc = pd.concat([df_xam, df_xnz], axis=0, ignore_index=True)\ndf_xc['primary_label'] = df_xc['ebird_code']\ndf_xc['scientific_name'] = df_xc['sci_name']\ndf_xc['common_name'] = df_xc['species']\ndf_xc['target'] = df_xc.primary_label.map(CFG.name2label2)\ndf_xc['birdclef'] = 'xc'\nassert tf.io.gfile.exists(df_xc.filepath.iloc[0])\n\n# BirdCLEF-2021\ndf_21 = pd.read_csv(f'{BASE_PATH1}/train_metadata.csv')\ndf_21['filepath'] = GCS_PATH1 + '/train_short_audio/' + df_21.primary_label + '/' + df_21.filename\ndf_21['target'] = df_21.primary_label.map(CFG.name2label2)\ndf_21['birdclef'] = '21'\ncorrupt_paths = ['/kaggle/input/birdclef-2021/train_short_audio/houwre/XC590621.ogg',\n                 '/kaggle/input/birdclef-2021/train_short_audio/cogdov/XC579430.ogg']\ndf_21 = df_21[~df_21.filepath.isin(corrupt_paths)] # remove all zero audios\nassert tf.io.gfile.exists(df_21.filepath.iloc[0])\n\n# BirdCLEF-2022\ndf_22 = pd.read_csv(f'{BASE_PATH2}/train_metadata.csv')\ndf_22['filepath'] = GCS_PATH2 + '/train_audio/' + df_22.filename\ndf_22['target'] = df_22.primary_label.map(CFG.name2label2)\ndf_22['birdclef'] = '22'\nassert tf.io.gfile.exists(df_22.filepath.iloc[0])\n\n# Merge 2021 and 2022 for pretraining\ndf_pre = pd.concat([df_20, df_21, df_22, df_xc], axis=0, ignore_index=True)\ndf_pre['filename'] = df_pre.filepath.map(lambda x: x.split('/')[-1])\ndf_pre['xc_id'] = df_pre.filepath.map(lambda x: x.split('/')[-1].split('.')[0])\nnodup_idx = df_pre[['xc_id','primary_label','author']].drop_duplicates().index\ndf_pre = df_pre.loc[nodup_idx].reset_index(drop=True)\n\n# # Remove duplicates\ndf_pre = df_pre[~df_pre.xc_id.isin(df_23.xc_id)].reset_index(drop=True)\ncorrupt_mp3s = json.load(open('/kaggle/input/birdclef-corrupt-mp3-files-ds/corrupt_mp3_files.json','r'))\ndf_pre = df_pre[~df_pre.filepath.isin(corrupt_mp3s)]\ndf_pre = df_pre[['filename','filepath','primary_label','secondary_labels',\n                 'rating','author','file_type','xc_id','scientific_name',\n                'common_name','target','birdclef','bird_seen']]\n# Display rows\nprint(\"# Samples for Pre-Training: {:,}\".format(len(df_pre)))\ndf_pre.head(2).style.set_caption(\"Pre-Training Data\").set_table_styles([{\n    'selector': 'caption',\n    'props': [\n        ('color', 'blue'),\n        ('font-size', '16px')\n    ]\n}])\n\n# Show distribution\nplt.figure(figsize=(8, 4))\ndf_pre.birdclef.value_counts().plot.bar(color=[cmap(0.0),cmap(0.25), cmap(0.65), cmap(0.9)])\nplt.xlabel(\"Dataset\")\nplt.ylabel(\"Count\")\nplt.title(\"Dataset distribution for Pre-Training\")\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_audio(filepath):\n    audio, sr = librosa.load(filepath)\n    return audio, sr\n\ndef get_spectrogram(audio):\n    spec = librosa.feature.melspectrogram(y=audio, \n                                   sr=CFG.sample_rate, \n                                   n_mels=CFG.img_size[0],\n                                   n_fft=CFG.nfft,\n                                   hop_length=CFG.hop_length,\n                                   fmax=CFG.fmax,\n                                   fmin=CFG.fmin,\n                                   )\n    spec = librosa.power_to_db(spec, ref=1.0)\n    return spec\n\ndef display_audio(row):\n    # Caption for viz\n    caption = f'Id: {row.filename} | Name: {row.common_name} | Sci.Name: {row.scientific_name} | Rating: {row.rating}'\n    # Read audio file\n    audio, sr = load_audio(row.filepath)\n    # Keep fixed length audio\n    audio = audio[:CFG.audio_len]\n    # Spectrogram from audio\n    spec = get_spectrogram(audio)\n    # Display audio\n    print(\"# Audio:\")\n    display(ipd.Audio(audio, rate=CFG.sample_rate))\n#     print(\"# Image:\")\n#     show_image(row.common_name)\n    print('# Visualization:')\n    fig, ax = plt.subplots(2, 1, figsize=(12, 2*3), sharex=True, tight_layout=True)\n    fig.suptitle(caption)\n    # Waveplot\n    lid.waveshow(audio,\n                 sr=CFG.sample_rate,\n                 ax=ax[0],\n                color= cmap(0.1))\n    # Specplot\n    lid.specshow(spec, \n                 sr = CFG.sample_rate, \n                 hop_length = CFG.hop_length,\n                 n_fft=CFG.nfft,\n                 fmin=CFG.fmin,\n                 fmax=CFG.fmax,\n                 x_axis = 'time', \n                 y_axis = 'mel',\n                 cmap = 'coolwarm',\n                 ax=ax[1])\n    ax[0].set_xlabel('');\n    fig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIRDCLEF = '20'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH0, BASE_PATH0)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIRDCLEF = 'xc'\nprint(f\"# Xeno-Canto - Extend\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH4, BASE_PATH4).replace(GCS_PATH5, BASE_PATH5)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIRDCLEF = '21'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH1, BASE_PATH1)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIRDCLEF = '22'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_pre.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH2, BASE_PATH2)\nrow = tmp.squeeze()\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIRDCLEF = '23'\nprint(f\"# BirdCLEF - 20{BIRDCLEF}\")\ntmp = df_23.query(\"birdclef==@BIRDCLEF\").sample(1)\ntmp.loc[:, 'filepath'] = tmp.filepath.str.replace(GCS_PATH3, BASE_PATH3)\nrow = tmp.squeeze()\n\n# Display audio\ndisplay_audio(row)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import required packages\nfrom sklearn.model_selection import StratifiedKFold\n\n# Initialize the StratifiedKFold object with 5 splits and shuffle the data\nskf1 = StratifiedKFold(n_splits=25, shuffle=True, random_state=CFG.seed)\nskf2 = StratifiedKFold(n_splits=CFG.num_fold, shuffle=True, random_state=CFG.seed)\n\n# Reset the index of the dataframe\ndf_pre = df_pre.reset_index(drop=True)\ndf_23 = df_23.reset_index(drop=True)\n\n# Create a new column in the dataframe to store the fold number for each row\ndf_pre[\"fold\"] = -1\ndf_23[\"fold\"] = -1\n\n# BirdCLEF - 21 & 22\nfor fold, (train_idx, val_idx) in enumerate(skf1.split(df_pre, df_pre['primary_label'])):\n    df_pre.loc[val_idx, 'fold'] = fold\n    \n# IBirdCLEF - 23\nfor fold, (train_idx, val_idx) in enumerate(skf2.split(df_23, df_23['primary_label'])):\n    df_23.loc[val_idx, 'fold'] = fold","metadata":{"papermill":{"duration":0.386301,"end_time":"2022-03-08T03:18:35.325064","exception":false,"start_time":"2022-03-08T03:18:34.938763","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_data(df, thr=5):\n    # Count the number of samples for each class\n    counts = df.primary_label.value_counts()\n\n    # Condition that selects classes with less than `thr` samples\n    cond = df.primary_label.isin(counts[counts<thr].index.tolist())\n\n    # Add a new column to select samples for cross validation\n    df['cv'] = True\n\n    # Set cv = False for those class where there is samples less than thr\n    df.loc[cond, 'cv'] = False\n\n    # Return the filtered dataframe\n    return df\n    \ndef upsample_data(df, thr=20):\n    # get the class distribution\n    class_dist = df['primary_label'].value_counts()\n\n    # identify the classes that have less than the threshold number of samples\n    down_classes = class_dist[class_dist < thr].index.tolist()\n\n    # create an empty list to store the upsampled dataframes\n    up_dfs = []\n\n    # loop through the undersampled classes and upsample them\n    for c in down_classes:\n        # get the dataframe for the current class\n        class_df = df.query(\"primary_label==@c\")\n        # find number of samples to add\n        num_up = thr - class_df.shape[0]\n        # upsample the dataframe\n        class_df = class_df.sample(n=num_up, replace=True, random_state=CFG.seed)\n        # append the upsampled dataframe to the list\n        up_dfs.append(class_df)\n\n    # concatenate the upsampled dataframes and the original dataframe\n    up_df = pd.concat([df] + up_dfs, axis=0, ignore_index=True)\n    \n    return up_df\n\ndef downsample_data(df, thr=500):\n    # get the class distribution\n    class_dist = df['primary_label'].value_counts()\n    \n    # identify the classes that have less than the threshold number of samples\n    up_classes = class_dist[class_dist > thr].index.tolist()\n\n    # create an empty list to store the upsampled dataframes\n    down_dfs = []\n\n    # loop through the undersampled classes and upsample them\n    for c in up_classes:\n        # get the dataframe for the current class\n        class_df = df.query(\"primary_label==@c\")\n        # Remove that class data\n        df = df.query(\"primary_label!=@c\")\n        # upsample the dataframe\n        class_df = class_df.sample(n=thr, replace=False, random_state=CFG.seed)\n        # append the upsampled dataframe to the list\n        down_dfs.append(class_df)\n\n    # concatenate the upsampled dataframes and the original dataframe\n    down_df = pd.concat([df] + down_dfs, axis=0, ignore_index=True)\n    \n    return down_df","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter data\nf_df = filter_data(df_pre, thr=5)\n\nplt.figure(figsize=(10, 4))\nax1 = plt.subplot(1, 2, 1)\nf_df.cv.value_counts().plot.bar(legend=True, color=cmap(0.1))\nplt.yscale(\"log\")\nplt.title(\"BirdCLEF - 20, 21 & 22\")\nplt.legend([\"BirdCLEF - 20, 21 & 22\"])\n\nf_df = filter_data(df_23, thr=5)\nax2 = plt.subplot(1, 2, 2, sharey = ax1)\nf_df.cv.value_counts().plot.bar(legend=True, color=cmap(0.9))\nplt.yscale(\"log\")\nplt.title(\"BirdCLEF - 23\")\nplt.legend([\"BirdCLEF - 23\"])\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Upsample data\nup_thr = 100\ndn_df = downsample_data(df_pre, thr=400)\nup_df = upsample_data(dn_df, thr=up_thr)\nprint(\"# Pretraing Dataset\")\nprint(f'> Original: {len(df_pre)}')\nprint(f'> After Upsample: {len(up_df)}')\nprint(f'> After Downsample: {len(dn_df)}')\n\n# Show effect of upsample\nplt.figure(figsize=(12*2, 6))\n\nax1 = plt.subplot(1, 2, 1)\ndf_pre.primary_label.value_counts()[:].plot.bar(color='blue', label='original')\nup_df.primary_label.value_counts()[:].plot.bar(color='green', label='w/ upsample')\ndn_df.primary_label.value_counts()[:].plot.bar(color='red', label='w/ dowsample')\nplt.xticks([])\nplt.axhline(y=up_thr, color='g', linestyle='--', label='up threshold')\nplt.axhline(y=400, color='r', linestyle='--', label='down threshold')\nplt.legend()\nplt.title(\"Upsample for Pre-Training\")\n# plt.show()\n\n# Upsample data\nup_thr = 50\nup_df = upsample_data(df_23, thr=up_thr)\nprint(\"\\n# BirdCLEF - 23\")\nprint(f'> Before Upsample: {len(df_23)}')\nprint(f'> After Upsample: {len(up_df)}')\n\n# Show effect of upsample\nax2 = plt.subplot(1, 2, 2, sharey=ax1)\nup_df.primary_label.value_counts()[:].plot.bar(color='green', label='w/ upsample')\ndf_23.primary_label.value_counts()[:].plot.bar(color='red', label='w/o upsample')\nplt.xticks([])\nplt.axhline(y=up_thr, color='g', linestyle='--', label='up threshold')\nplt.legend()\nplt.title(\"Upsample in BirdCLEF - 23\")\n\n# plt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generates random integer\ndef random_int(shape=[], minval=0, maxval=1):\n    return tf.random.uniform(shape=shape, minval=minval, maxval=maxval, dtype=tf.int32)\n\n\n# Generats random float\ndef random_float(shape=[], minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(shape=shape, minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import required packages\nimport tensorflow_extra as tfe\n\n# Randomly shift audio -> any sound at <t> time may get shifted to <t+shift> time\n@tf.function\ndef TimeShift(audio, prob=0.5):\n    # Randomly apply time shift with probability `prob`\n    if random_float() < prob:\n        # Calculate random shift value\n        shift = random_int(shape=[], minval=0, maxval=tf.shape(audio)[0])\n        # Randomly set the shift to be negative with 50% probability\n        if random_float() < 0.5:\n            shift = -shift\n        # Roll the audio signal by the shift value\n        audio = tf.roll(audio, shift, axis=0)\n    return audio\n\n# Apply random noise to audio data\n@tf.function\ndef GaussianNoise(audio, std=[0.0025, 0.025], prob=0.5):\n    # Select a random value of standard deviation for Gaussian noise within the given range\n    std = random_float([], std[0], std[1])\n    # Randomly apply Gaussian noise with probability `prob`\n    if random_float() < prob:\n        # Add random Gaussian noise to the audio signal\n        GN = tf.keras.layers.GaussianNoise(stddev=std)\n        audio = GN(audio, training=True) # training=False don't apply noise to data\n    return audio\n\n# Applies augmentation to Audio Signal\ndef AudioAug(audio):\n    # Apply time shift and Gaussian noise to the audio signal\n    audio = TimeShift(audio, prob=CFG.timeshift_prob)\n    audio = GaussianNoise(audio, prob=CFG.gn_prob)\n    return audio\n\n# CutMix & MixUp\nmixup_layer = tfe.layers.MixUp(alpha=CFG.mixup_alpha, prob=CFG.mixup_prob)\ncutmix_layer = tfe.layers.CutMix(alpha=CFG.cutmix_alpha, prob=CFG.cutmix_prob)\n\ndef CutMixUp(audios, labels):\n    audios, labels = mixup_layer(audios, labels, training=True)\n    audios, labels = cutmix_layer(audios, labels, training=True)\n    return audios, labels","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decodes Audio\ndef audio_decoder(with_labels=True, dim=CFG.audio_len, \n                  take_first=False, num_classes=264, CFG=CFG):\n    def get_audio(filepath):\n        ftype = filepath[1]\n        filepath = filepath[0]\n        file_bytes = tf.io.read_file(filepath)\n        if ftype:\n            audio = tfio.audio.decode_vorbis(file_bytes) # decode .ogg file\n        else:\n            audio = tfio.audio.decode_mp3(file_bytes) # decode .mp3 file\n        audio = tf.cast(audio, tf.float32)\n        if tf.shape(audio)[1]>1: # stereo -> mono\n            audio = audio[...,0:1]\n        audio = tf.squeeze(audio, axis=-1)\n        return audio\n    \n    def crop_or_pad(audio, target_len, pad_mode='constant', take_first=True):\n        audio_len = tf.shape(audio)[0]\n        diff_len = abs(target_len - audio_len)\n        if audio_len < target_len:\n            pad1 = tf.random.uniform([], maxval=diff_len, dtype=tf.int32)\n            pad2 = diff_len - pad1\n            audio = tf.pad(audio, paddings=[[pad1, pad2]], mode=pad_mode)\n        elif audio_len > target_len:\n            if take_first:\n                audio = audio[:target_len]\n            else:\n                idx = tf.random.uniform([], maxval=diff_len, dtype=tf.int32)\n                audio = audio[idx: (idx + target_len)]\n        return tf.reshape(audio, [target_len])\n\n    def get_target(target):          \n        target = tf.reshape(target, [1])\n        target = tf.cast(tf.one_hot(target, num_classes), tf.float32) \n        target = tf.reshape(target, [num_classes])\n        return target\n\n    def decode(path):\n        audio = get_audio(path)\n        audio = crop_or_pad(audio, dim) # crop or pad audio to keep a fixed length\n        audio = tf.reshape(audio, [dim])\n        return audio\n    \n    def decode_with_labels(path, label):\n        label = get_target(label)\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode","metadata":{"papermill":{"duration":0.251237,"end_time":"2022-03-08T03:18:49.079346","exception":false,"start_time":"2022-03-08T03:18:48.828109","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Applies augmentation to audio\ndef audio_augmenter(with_labels=True, dim=CFG.audio_len, CFG=CFG):\n    def augment(audio, dim=dim):\n        if random_float() <= CFG.audio_augment_prob:\n            audio = AudioAug(audio)\n        audio = tf.reshape(audio, [dim])\n        return audio\n    \n    def augment_with_labels(audio, label):    \n        return augment(audio), label\n    \n    return augment_with_labels if with_labels else augment","metadata":{"papermill":{"duration":0.250484,"end_time":"2022-03-08T03:18:49.79513","exception":false,"start_time":"2022-03-08T03:18:49.544646","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_dataset(paths, ftype, labels=None, batch_size=32, target_size=[128, 256], \n                  audio_decode_fn=None, audio_augment_fn=None,\n                  take_first=False, num_classes=264,\n                  cache=True, cache_dir=\"\",drop_remainder=False,\n                  augment=True, repeat=True, shuffle=1024):\n    \"\"\"\n    Creates a TensorFlow dataset from the given paths and labels.\n    \n    Args:\n        paths (list): A list of file paths to the audio files.\n        labels (list): A list of corresponding labels for the audio files.\n        batch_size (int): Batch size for the created dataset.\n        target_size (list): A list of target image size for the spectrograms.\n        audio_decode_fn (function): A function to decode the audio file.\n        audio_augment_fn (function): A function to augment the audio file.\n        cache (bool): Whether to cache the dataset or not.\n        cache_dir (str): Directory path to cache the dataset.\n        drop_remainder (bool): Whether to drop the last batch if it is smaller than batch_size.\n        augment (bool): Whether to augment the dataset or not.\n        repeat (bool): Whether to repeat the dataset or not.\n        shuffle (int): Number of elements from the dataset to buffer for shuffling.\n        \n    Returns:\n        ds (tf.data.Dataset): A TensorFlow dataset.\n    \"\"\"\n    # Create cache directory if cache is enabled\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    # Set default audio decode function if not provided\n    if audio_decode_fn is None:\n        audio_decode_fn = audio_decoder(labels is not None,\n                                        dim=CFG.audio_len, \n                                        take_first=take_first,\n                                        num_classes=num_classes,\n                                        CFG=CFG)\n    # Set default audio augmentation function if not provided\n    if audio_augment_fn is None:\n        audio_augment_fn = audio_augmenter(labels is not None, \n                                           dim=CFG.audio_len, CFG=CFG)\n        \n    # Set TensorFlow AUTOTUNE option\n    AUTO = tf.data.experimental.AUTOTUNE\n    # Create slices based on whether labels are provided\n    slices = ((paths, ftype),) if labels is None else ((paths, ftype), labels)\n    # Create TensorFlow dataset from slices\n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    # Map audio decode function to dataset\n    ds = ds.map(audio_decode_fn, num_parallel_calls=AUTO)\n    # Cache dataset in memory if cache is enabled\n    ds = ds.cache(cache_dir) if cache else ds\n    # Repeat dataset indefinitely if repeat is enabled\n    ds = ds.repeat() if repeat else ds\n    # Create TensorFlow dataset options\n    opt = tf.data.Options()\n    # Shuffle dataset if shuffle is enabled\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt.experimental_deterministic = False\n    if CFG.device=='GPU':\n        # If the device is a GPU, turn off auto-sharding to avoid performance issues\n        opt.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n    # Set the options for the dataset\n    ds = ds.with_options(opt)\n    # Apply audio augmentation to the dataset if augment is True\n    ds = ds.map(audio_augment_fn, num_parallel_calls=AUTO) if augment else ds\n    # Batch the dataset with the specified batch size\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    # Apply MixUp & CutMix regularization to the dataset\n    if augment and labels is not None:\n        ds = ds.map(CutMixUp,num_parallel_calls=AUTO)\n    # Prefetch the next batch of data to improve performance\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"papermill":{"duration":0.240881,"end_time":"2022-03-08T03:18:50.489717","exception":false,"start_time":"2022-03-08T03:18:50.248836","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_batch(batch, row=3, col=3, label2name=None,):\n    \"\"\"Plot one batch data\"\"\"\n    if isinstance(batch, tuple) or isinstance(batch, list):\n        audios, tars = batch\n    else:\n        audios = batch\n        tars = None\n    plt.figure(figsize=(col*5, row*3))\n    for idx in range(row*col):\n        ax = plt.subplot(row, col, idx+1)\n        plt.plot(audios[idx].numpy(), color=cmap(0.1))\n        if tars is not None:\n            label = tars[idx].numpy().argmax()\n            name = label2name[label]\n            plt.title(name)\n    plt.tight_layout()\n    plt.show()\n    \n    \ndef plot_history(history):\n    \"\"\"Plot trainign history, credit: @cdeotte\"\"\"\n    epochs = len(history.history['auc'])\n    plt.figure(figsize=(15,5))\n    plt.plot(np.arange(epochs),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n    plt.plot(np.arange(epochs),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n    x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n    plt.ylabel('AUC (PR)',size=14); plt.xlabel('Epoch',size=14)\n    plt.legend(loc=2)\n    plt2 = plt.gca().twinx()\n    plt2.plot(np.arange(epochs),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n    plt2.plot(np.arange(epochs),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n    x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n    plt.ylabel('Loss',size=14)\n    plt.title('Fold %i - Training Plot'%(fold+1),size=18)\n    plt.legend(loc=3)\n    plt.show()  ","metadata":{"papermill":{"duration":0.328513,"end_time":"2022-03-08T03:19:59.512224","exception":false,"start_time":"2022-03-08T03:19:59.183711","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = df_pre.query(\"birdclef=='20'\").sample(50)\nds = build_dataset(check_df.filepath.tolist(),\n                   check_df.filepath.str.contains('.ogg').tolist(),\n                   check_df.target.tolist(), \n                   num_classes=CFG.num_classes2,\n                   augment=True, cache=False)\nds = ds.take(32)\naudios, labels = next(iter(ds))\nplot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = df_pre.query(\"birdclef=='xc'\").sample(50)\nds = build_dataset(check_df.filepath.tolist(),\n                   check_df.filepath.str.contains('.ogg').tolist(),\n                   check_df.target.tolist(), \n                   num_classes=CFG.num_classes2,\n                   augment=True, cache=False)\nds = ds.take(32)\naudios, labels = next(iter(ds))\nplot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = df_pre.query(\"birdclef=='21'\").sample(50)\nds = build_dataset(check_df.filepath.tolist(),\n                   check_df.filepath.str.contains('.ogg').tolist(),\n                   check_df.target.tolist(), \n                   num_classes=CFG.num_classes2,\n                   augment=True, cache=False)\nds = ds.take(32)\naudios, labels = next(iter(ds))\nplot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"papermill":{"duration":3.299334,"end_time":"2022-03-08T03:20:02.987986","exception":false,"start_time":"2022-03-08T03:19:59.688652","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = df_pre.query(\"birdclef=='22'\").sample(50)\nds = build_dataset(check_df.filepath.tolist(),\n                   check_df.filepath.str.contains('.ogg').tolist(),\n                   check_df.target.tolist(), \n                   num_classes=CFG.num_classes2,\n                   augment=True, cache=False)\nds = ds.take(32)\naudios, labels = next(iter(ds))\nplot_batch((audios, labels), label2name=CFG.label2name2)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_df = df_23.sample(50)\nds = build_dataset(check_df.filepath.tolist(),\n                   check_df.filepath.str.contains('.ogg').tolist(),\n                   check_df.target.tolist(), \n                   num_classes=CFG.num_classes2,\n                   augment=True, cache=False)\nds = ds.take(32)\naudios, labels = next(iter(ds))\nplot_batch((audios, labels), label2name=CFG.label2name)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"melspec_layer = tfe.layers.MelSpectrogram(n_fft=CFG.nfft, \n                                          hop_length=CFG.hop_length, \n                                          sr=CFG.sample_rate, \n                                          ref=1.0,\n                                          fmin=500,\n                                          fmax=15000,\n                                          out_channels=3)\nspecs = melspec_layer(audios)\n\nfig, ax = plt.subplots(2, 1, sharex=True, figsize=(12, 5))\nlid.waveshow(audios[0].numpy(), sr=CFG.sample_rate, ax=ax[0], axis=None)\n\nlid.specshow(specs[0, ..., 0].numpy(), \n             n_fft=CFG.nfft, \n             hop_length=CFG.hop_length, \n             sr=CFG.sample_rate,\n             x_axis='time',\n             y_axis='mel',\n             cmap='coolwarm',\n              ax=ax[1])\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfm_layer = tfe.layers.TimeFreqMask(freq_mask_prob=0.65,\n                                  num_freq_masks=2,\n                                  freq_mask_param=10,\n                                  time_mask_prob=0.65,\n                                  num_time_masks=3,\n                                  time_mask_param=25,\n                                  time_last=True,)\nspecs2 = tfm_layer(specs, training=True)\n\nplt.figure(figsize=(12,3))\nlid.specshow(specs2[0, ..., 0].numpy(), \n             n_fft=CFG.nfft, \n             hop_length=CFG.hop_length, \n             sr=CFG.sample_rate,\n            x_axis='time',\n            y_axis='mel',\n            cmap='coolwarm')\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_layer = tfe.layers.ZScoreMinMax()\nspecs3 = norm_layer(specs2)\n\nplt.figure(figsize=(8,3))\nplt.hist(specs2.numpy().ravel(), alpha=0.8, color=cmap(0.1))\nplt.hist(specs3.numpy().ravel(), alpha=0.8, color=cmap(0.9))\nplt.legend([\"w/o normalize\", \"w/ normalize\"])\nplt.semilogx()\nplt.title(\"Effect of Normalization\")\nplt.xlabel(\"Pixel Value\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn.metrics\n\ndef get_metrics():\n#     acc = tf.keras.metrics.BinaryAccuracy(name='acc')\n    auc = tf.keras.metrics.AUC(curve='PR', name='auc', multi_label=False) # auc on prcision-recall curve\n    acc = tf.keras.metrics.CategoricalAccuracy(name='acc')\n    return [acc, auc]\n\ndef padded_cmap(y_true, y_pred, padding_factor=5):\n    num_classes = y_true.shape[1]\n    pad_rows = np.array([[1]*num_classes]*padding_factor)\n    y_true = np.concatenate([y_true, pad_rows])\n    y_pred = np.concatenate([y_pred, pad_rows])\n    score = sklearn.metrics.average_precision_score(y_true, y_pred, average='macro',)\n    return score\n\ndef get_loss():\n    if CFG.loss==\"CCE\":\n        loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=CFG.label_smoothing)\n    elif CFG.loss==\"BCE\":\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=CFG.label_smoothing)\n    else:\n        raise ValueError(\"Loss not found\")\n    return loss\n    \ndef get_optimizer():\n    if CFG.optimizer == \"Adam\":\n        opt = tf.keras.optimizers.Adam(learning_rate=CFG.lr)\n    else:\n        raise ValueError(\"Optmizer not found\")\n    return opt","metadata":{"papermill":{"duration":0.28125,"end_time":"2022-03-08T03:20:04.498883","exception":false,"start_time":"2022-03-08T03:20:04.217633","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\nimport tensorflow_extra as tfe\n\n# Will download and load pretrained imagenet weights.\ndef build_model(CFG, model_name=None, num_classes=264, compile_model=True):\n    \"\"\"\n    Builds and returns a model based on the specified configuration.\n    \"\"\"\n    # Create an input layer for the model\n    inp = tf.keras.layers.Input(shape=(None,))\n    # Spectrogram\n    out = tfe.layers.MelSpectrogram(n_mels=CFG.img_size[0],\n                                    n_fft=CFG.nfft,\n                                    hop_length=CFG.hop_length, \n                                    sr=CFG.sample_rate,\n                                    ref=1.0,\n                                    out_channels=3)(inp)\n    # Normalize\n    out = tfe.layers.ZScoreMinMax()(out)\n    # TimeFreqMask\n    out = tfe.layers.TimeFreqMask(freq_mask_prob=0.5,\n                                  num_freq_masks=1,\n                                  freq_mask_param=10,\n                                  time_mask_prob=0.5,\n                                  num_time_masks=2,\n                                  time_mask_param=25,\n                                  time_last=False,)(out)\n    # Load backbone model\n    base = getattr(efn, model_name)(input_shape=(None, None, 3),\n                                            include_top=0,\n                                            weights=CFG.pretrain,\n                                           fsr=CFG.fsr)\n    # Pass the input through the base model\n    out = base(out)\n    out = tf.keras.layers.GlobalAveragePooling2D()(out)\n    out = tf.keras.layers.Dense(num_classes, activation='softmax')(out)\n    model = tf.keras.models.Model(inputs=inp, outputs=out)\n    if compile_model:\n        # Set the optimizer\n        opt = get_optimizer()\n        # Set the loss function\n        loss = get_loss()\n        # Set the evaluation metrics\n        metrics = get_metrics()\n        # Compile the model with the specified optimizer, loss function, and metrics\n        model.compile(optimizer=opt, loss=loss, metrics=metrics)\n    return model","metadata":{"papermill":{"duration":1.239321,"end_time":"2022-03-08T03:20:06.281118","exception":false,"start_time":"2022-03-08T03:20:05.041797","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(CFG, model_name=CFG.model_name)\nmodel.summary()","metadata":{"papermill":{"duration":37.756883,"end_time":"2022-03-08T03:20:44.226871","exception":false,"start_time":"2022-03-08T03:20:06.469988","status":"completed"},"tags":[],"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audios = tf.random.uniform((1, CFG.audio_len))\nwith strategy.scope():\n    out = model(audios, training=False)\nprint(out.shape)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\ndef get_lr_callback(batch_size=8, mode='cos', epochs=CFG.epochs, plot=False):\n    \"\"\"\n    Returns a learning rate scheduler callback for a given batch size, mode, and number of epochs.\n    \"\"\"\n    # Define the learning rate schedule.\n    lr_start   = 0.000005\n    lr_max     = 0.00000140 * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    # Function to update the lr\n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n\n        elif CFG.scheduler == 'exp':\n            lr = (lr_max - lr_min) * lr_decay**(epoch - \\\n                  lr_ramp_ep - lr_sus_ep) + lr_min\n\n        elif CFG.scheduler == 'step':\n            lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n\n        elif CFG.scheduler == 'cos':\n            decay_total_epochs = epochs - lr_ramp_ep - lr_sus_ep + 3\n            decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            cosine_decay = 0.5 * (1 + math.cos(phase))\n            lr = (lr_max - lr_min) * cosine_decay + lr_min\n        return lr\n    \n    # Plot the lr curve\n    if plot:\n        plt.figure(figsize=(10,5))\n        plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('learnig rate')\n        plt.title('Learning Rate Scheduler')\n        plt.show()\n        \n    # Crate lr-callback to update lr during training\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.510014,"end_time":"2022-03-08T03:20:45.290695","exception":false,"start_time":"2022-03-08T03:20:44.780681","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_=get_lr_callback(CFG.batch_size*CFG.replicas, plot=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\n\ndef wandb_init(fold):\n    \"\"\"\n    Initializes the W&B run by creating a config file and initializing a W&B run.\n    \"\"\"\n    # Create a dictionary of configuration parameters\n    config = {k:v for k,v in dict(vars(CFG)).items() if '__' not in k}\n    config.update({\"fold\":int(fold)}) # int is to convert numpy.int -> int\n    # Dump the configuration dictionary to a YAML file\n    yaml.dump(config, open(f'/kaggle/working/config fold-{fold}.yaml', 'w'),)\n    # Load the configuration dictionary from the YAML file\n    config = yaml.load(open(f'/kaggle/working/config fold-{fold}.yaml', 'r'), Loader=yaml.FullLoader)\n    # Initialize a W&B run with the given configuration parameters\n    run = wandb.init(project=\"birdclef-2023-public\",\n                     name=f\"fold-{fold}|dim-{CFG.img_size[1]}x{CFG.img_size[0]}|model-{CFG.model_name}\",\n                     config=config,\n                     group=CFG.comment,\n                     save_code=True,)\n    return run\n\n    \ndef log_wandb(valid_df):\n    \"\"\"Log and save validation results with missclassified examples as audio in W&B\"\"\"\n    # Query only the rows with miss predictions\n    save_df = valid_df.query(\"miss==True\")\n    # Map the predicted and target labels to their corresponding names\n    save_df.loc[:, 'pred_name'] = save_df.pred.map(CFG.label2name)\n    save_df.loc[:, 'target_name'] = save_df.target.map(CFG.label2name)\n    # Trim the dataframe for debugging purposes\n    if CFG.debug:\n        save_df = save_df.iloc[:CFG.replicas*CFG.batch_size*CFG.infer_bs]\n    # Get the columns to be included in the wandb table\n    noimg_cols = [*CFG.tab_cols, 'target', 'pred', 'target_name','pred_name']\n    # Retain only the necessary columns\n    save_df = save_df.loc[:, noimg_cols]\n\n    data = []\n    # Load audio files for each miss prediction\n    for idx, row in tqdm(save_df.iterrows(), total=len(save_df), desc='wandb ', position=0, leave=True):\n        filepath = '/kaggle/input/birdclef-2023/train_audio/'+CFG.label2name[row.target]+'/'+row.filename\n        audio, sr = librosa.load(filepath, sr=None)\n        # Add the audio file to the data list along with the other relevant information\n        data+=[[*row.tolist(), wandb.Audio(audio, caption=row.filename, sample_rate=sr)]]\n    # Create a wandb table with the audio files and other relevant information\n    wandb_table = wandb.Table(data=data, columns=[*noimg_cols, 'audio'])\n    # Manually unpack dict values\n    scores_wb = {f'best.{k}': v for k,v in scores.items()}\n    # Log the scores and wandb table to wandb\n    wandb.log({**scores_wb,\n               'table': wandb_table,\n               })\n    \n# get wandb callbacks\ndef get_wb_callbacks(fold):\n    wb_ckpt = wandb.keras.WandbModelCheckpoint(filepath='fold-%i.h5'%fold, \n                                               monitor='val_auc',\n                                               verbose=CFG.verbose,\n                                               save_best_only=True,\n                                               save_weights_only=False,\n                                               mode='max',)\n    wb_metr = wandb.keras.WandbMetricsLogger()\n    return [wb_ckpt, wb_metr]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.288344,"end_time":"2022-03-08T03:20:47.977099","exception":false,"start_time":"2022-03-08T03:20:47.688755","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configurations\nnum_classes = CFG.num_classes2\ndf = df_pre.copy()\nfold = 0\n\n# Compute batch size and number of samples to drop\ninfer_bs = (CFG.batch_size*CFG.infer_bs)\ndrop_remainder = CFG.drop_remainder\n\n# Split dataset with cv filter\nif CFG.cv_filter:\n    df = filter_data(df, thr=5)\n    train_df = df.query(\"fold!=@fold | ~cv\").reset_index(drop=True)\n    valid_df = df.query(\"fold==@fold & cv\").reset_index(drop=True)\nelse:\n    train_df = df.query(\"fold!=@fold\").reset_index(drop=True)\n    valid_df = df.query(\"fold==@fold\").reset_index(drop=True)\n\n# Upsample train data\ntrain_df = upsample_data(train_df, thr=50)\ntrain_df = downsample_data(train_df, thr=500)\n\n# Get file paths and labels\ntrain_paths = train_df.filepath.values; train_labels = train_df.target.values\nvalid_paths = valid_df.filepath.values; valid_labels = valid_df.target.values\n\n# Shuffle the file paths and labels\nindex = np.arange(len(train_paths))\nnp.random.shuffle(index)\ntrain_paths  = train_paths[index]\ntrain_labels = train_labels[index]\n\n# For debugging\nif CFG.debug:\n    min_samples = CFG.batch_size*CFG.replicas*2\n    train_paths = train_paths[:min_samples]; train_labels = train_labels[:min_samples]\n    valid_paths = valid_paths[:min_samples]; valid_labels = valid_labels[:min_samples]\n    \n# Ogg or Mp3\ntrain_ftype = list(map(lambda x: '.ogg' in x, train_paths))\nvalid_ftype = list(map(lambda x: '.ogg' in x, valid_paths))\n\n# Compute the number of training and validation samples\nnum_train = len(train_paths); num_valid = len(valid_paths)\n\n# Build the training and validation datasets\ncache=False\ntrain_ds = build_dataset(train_paths, train_ftype, train_labels, \n                         batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=True,\n                        drop_remainder=drop_remainder, num_classes=num_classes)\nvalid_ds = build_dataset(valid_paths, valid_ftype, valid_labels,\n                         batch_size=CFG.batch_size*CFG.replicas, cache=True, shuffle=False,\n                         augment=False, repeat=False, drop_remainder=drop_remainder,\n                         take_first=True, num_classes=num_classes)\n\n# Print information about the fold and training\nprint('#'*25); print('#### Pre-Training')\nprint('#### Image Size: (%i, %i) | Model: %s | Batch Size: %i | Scheduler: %s'%\n      (*CFG.img_size, CFG.model_name, CFG.batch_size*CFG.replicas, CFG.scheduler))\nprint('#### Num Train: {:,} | Num Valid: {:,}'.format(len(train_paths), len(valid_paths)))\n\n# Clear the session and build the model\nK.clear_session()\nwith strategy.scope():\n    model = build_model(CFG, model_name=CFG.model_name, num_classes=num_classes)\n\nprint('#'*25) \n\n# Checkpoint Callback\nckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n    'birdclef_pretrained_ckpt.h5', monitor='val_auc', verbose=0, save_best_only=True,\n    save_weights_only=False, mode='max', save_freq='epoch')\n# LR Scheduler Callback\nlr_cb = get_lr_callback(CFG.batch_size*CFG.replicas)\ncallbacks = [ckpt_cb, lr_cb]\n\n# Training\nhistory = model.fit(\n    train_ds, \n    epochs=2 if CFG.debug else CFG.epochs, \n    callbacks=callbacks, \n    steps_per_epoch=len(train_paths)/CFG.batch_size//CFG.replicas,\n    validation_data=valid_ds, \n    verbose=CFG.verbose,\n)\n\n# Show training plot\nif CFG.training_plot:\n    plot_history(history)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"papermill":{"duration":3436.548849,"end_time":"2022-03-08T04:18:05.540592","exception":true,"start_time":"2022-03-08T03:20:48.991743","status":"failed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_pred = []; oof_true = []; oof_val = []; oof_ids = []; oof_folds = [] \n\nnum_classes = CFG.num_classes\ndf = df_23.copy()\nfor fold in range(CFG.num_fold):\n    # Check if the fold is selected\n    if fold not in CFG.selected_folds:\n        continue\n    \n    # Initialize Weights and Biases\n    if CFG.wandb:\n        run = wandb_init(fold)\n    \n    # Compute batch size and number of samples to drop\n    infer_bs = (CFG.batch_size*CFG.infer_bs)\n    drop_remainder = CFG.drop_remainder\n    \n    # Split dataset with cv filter\n    if CFG.cv_filter:\n        df = filter_data(df, thr=5)\n        train_df = df.query(\"fold!=@fold | ~cv\").reset_index(drop=True)\n        valid_df = df.query(\"fold==@fold & cv\").reset_index(drop=True)\n    else:\n        train_df = df.query(\"fold!=@fold\").reset_index(drop=True)\n        valid_df = df.query(\"fold==@fold\").reset_index(drop=True)\n    \n    # Upsample train data\n    train_df = upsample_data(train_df, thr=50)\n#     train_df = downsample_data(train_df, thr=500)\n\n    # Get file paths and labels\n    train_paths = train_df.filepath.values; train_labels = train_df.target.values\n    valid_paths = valid_df.filepath.values; valid_labels = valid_df.target.values\n\n    # Shuffle the file paths and labels\n    index = np.arange(len(train_paths))\n    np.random.shuffle(index)\n    train_paths  = train_paths[index]\n    train_labels = train_labels[index]\n    \n    # For debugging\n    if CFG.debug:\n        min_samples = CFG.batch_size*CFG.replicas*2\n        train_paths = train_paths[:min_samples]; train_labels = train_labels[:min_samples]\n        valid_paths = valid_paths[:min_samples]; valid_labels = valid_labels[:min_samples]\n\n    # Ogg or Mp3\n    train_ftype = list(map(lambda x: '.ogg' in x, train_paths))\n    valid_ftype = list(map(lambda x: '.ogg' in x, valid_paths))\n\n    # Compute the number of training and validation samples\n    num_train = len(train_paths); num_valid = len(valid_paths)\n        \n    # Log the number of training and validation samples if Weights and Biases is being used\n    if CFG.wandb:\n        wandb.log({'num_train':num_train,\n                   'num_valid':num_valid})\n        \n    # Build the training and validation datasets\n    cache=True\n    train_ds = build_dataset(train_paths, train_ftype, train_labels, \n                             batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=True,\n                            drop_remainder=drop_remainder, num_classes=num_classes)\n    valid_ds = build_dataset(valid_paths, valid_ftype, valid_labels,\n                             batch_size=CFG.batch_size*CFG.replicas, cache=cache, shuffle=False,\n                             augment=False, repeat=False, drop_remainder=drop_remainder,\n                             take_first=True, num_classes=num_classes)\n    \n    # Print information about the fold and training\n    print('#'*25); print('#### Training')\n    print('#### Fold: %i | Image Size: (%i, %i) | Model: %s | Batch Size: %i | Scheduler: %s'%\n          (fold+1, *CFG.img_size, CFG.model_name, CFG.batch_size*CFG.replicas, CFG.scheduler))\n    print('#### Num Train: {:,} | Num Valid: {:,}'.format(len(train_paths), len(valid_paths)))\n    \n    # Clear the session and build the model\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(CFG, model_name=CFG.model_name, num_classes=num_classes)\n    # Load birdclef pretrained weights\n    model.load_weights(\"birdclef_pretrained_ckpt.h5\", by_name=True, skip_mismatch=True)\n        \n    print('#'*25) \n    \n    # Callbacks\n    ckpt_cb = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_auc', verbose=0, save_best_only=True,\n        save_weights_only=False, mode='max', save_freq='epoch')\n    # LR Scheduler Callback\n    lr_cb = get_lr_callback(CFG.batch_size*CFG.replicas)\n    callbacks = [ckpt_cb, lr_cb]\n    # WandB Callbacks\n    if CFG.wandb:\n        wb_cb = get_wb_callbacks(fold)\n        callbacks+=[wb_cb]\n\n    # Training\n    history = model.fit(\n        train_ds, \n        epochs=2 if CFG.debug else CFG.epochs, \n        callbacks=callbacks, \n        steps_per_epoch=len(train_paths)/CFG.batch_size//CFG.replicas,\n        validation_data=valid_ds, \n        verbose=CFG.verbose,\n    )\n    \n    # Load best checkpoint\n    print('# Loading best model')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # Predict on the validation data for oof result\n    print('# Infering OOF')\n    valid_ds = build_dataset(valid_paths, valid_ftype, labels=None, augment=CFG.tta>1, repeat=True, cache=False, \n                             shuffle=False, batch_size=infer_bs*CFG.replicas, \n                             drop_remainder=drop_remainder, take_first=True, num_classes=num_classes)\n    ct_valid = len(valid_paths); STEPS = CFG.tta * ct_valid/infer_bs/CFG.replicas\n    pred = model.predict(valid_ds,steps=STEPS,verbose=CFG.verbose)[:CFG.tta*ct_valid,] \n    pred = np.mean(pred.reshape((CFG.tta,ct_valid,-1)),axis=0)\n    oof_pred.append(pred)               \n\n    # Get ids and targets\n    oof_true.append(valid_df[CFG.target_col].values[:ct_valid])\n    oof_folds.append(np.ones_like(oof_true[-1],dtype='int8')*fold )\n    oof_ids.append(valid_paths)\n   \n    # Save valid data prediction\n    y_true = oof_true[-1].reshape(-1).astype('float32')\n    y_pred = oof_pred[-1].argmax(axis=-1)\n    valid_df.loc[:num_valid - 1, 'pred'] = y_pred\n    valid_df.loc[:num_valid - 1, 'miss'] = y_true != y_pred\n    valid_df.loc[:num_valid - 1, CFG.class_names] = oof_pred[-1].tolist()\n    \n    # Log the metrics\n    scores = {}\n    cmAP = padded_cmap(tf.keras.utils.to_categorical(y_true), oof_pred[-1])\n    best_epoch = np.argmax(history.history['val_'+CFG.monitor], axis=-1) + 1\n    best_score = history.history['val_'+CFG.monitor][best_epoch - 1]\n    scores.update({'auc': best_score,\n                   'epoch': best_epoch,\n                   'cmAP': cmAP,})\n    oof_val.append(best_score)\n    print('\\n>>> FOLD %i OOF AUC = %.3f | Padded_cmAP = %.3f' % (fold+1, oof_val[-1], cmAP))\n    \n    # Show training plot\n    if CFG.training_plot:\n        plot_history(history)\n        \n    # Log metrics, media to wandb\n    if CFG.wandb:\n        print('# WandB')\n        log_wandb(valid_df)\n        wandb.run.finish()\n        display(ipd.IFrame(run.url, width=1080, height=720))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_id(row):\n    row['filename'] = row['filepath'].split('/',5)[-1]\n    return row","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# OOF Data\ny_pred = np.concatenate(oof_pred); y_true = np.concatenate(oof_true);\nids = np.concatenate(oof_ids); folds = np.concatenate(oof_folds)\n\n# Overall cmAP\ncmAP = padded_cmap(tf.keras.utils.to_categorical(y_true), y_pred)\n\n# Overall AUC in PR curve\nm = tf.keras.metrics.AUC(curve='PR')\nm.update_state(tf.keras.utils.to_categorical(y_true), y_pred)\nauc = m.result().numpy()\n\nprint('>>> Overall cmAP: ', cmAP)\nprint('>>> Overall AUC(PR): ', auc)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save OOF data to disk\ncolumns = ['filepath', 'fold', 'true', 'pred', *CFG.class_names]\ndf_oof = pd.DataFrame(np.concatenate([ids[:,None], folds, y_true,\n                                      np.argmax(y_pred,axis=1)[:,None], y_pred], axis=1), columns=columns)\ndf_oof['class_name'] = df_oof.true.map(CFG.label2name)\ndf_oof['miss'] = df_oof.true!=df_oof.pred\ntqdm.pandas(desc='id ')\ndf_oof = df_oof.progress_apply(get_id,axis=1)\ndf_oof.to_csv('oof.csv',index=False)\ndisplay(df_oof.head(2))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Miss Total:')\ndisplay(df_oof.query(\"miss==True\").shape[0])\n\nprint()\nprint('Miss Distribution Top10:')\ndisplay(df_oof.query(\"miss==True\").class_name.value_counts()[:10])","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\ntry:\n    !rm -rf ./wandb\nexcept:\n    pass","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}