{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nimport shutil\nwarnings.filterwarnings(action='ignore')\n\nimport math\nimport pandas as pd\nimport librosa\nimport librosa.display\nimport numpy as np\nimport seaborn as sns; sns.set(style='whitegrid')\nimport matplotlib.pyplot as plt\n\nfrom sklearn.utils import shuffle\nfrom PIL import Image\nfrom tqdm import tqdm,tnrange,tqdm_notebook\nimport tensorflow as tf\nfrom tqdm.keras import TqdmCallback\nfrom keras.callbacks import ReduceLROnPlateau,EarlyStopping,ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator \nfrom tensorflow.keras import applications as app\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten,AveragePooling2D\nfrom tensorflow.keras.layers import Dense,BatchNormalization,Dropout \nfrom tensorflow.keras.models import Sequential \nfrom tensorflow.keras.applications import EfficientNetB4, ResNet50,ResNet101, VGG16, MobileNet, InceptionV3","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global Coefficients that can be modified\nclass coefs:\n    \n    # Generate Subset\n    rat_id = 4 # rating subset limiter \n    recs = 200 # each specie must have X recodings\n    max_files = 1500 # general last limit for rows\n    thresh = 0.25 # label probability selection threshold\n    submission = True # For Submission Only (Less Inference Output)\n    \n    # Global vars\n    seed = 1337\n    sr = 32000        # librosa sample rate input\n    sl = 5 # seconds   \n    sshape = (48,128) # height x width\n    fmin = 500      # spectrum min frequency\n    fmax = 12500    # spectrum max frequency\n    n_epoch = 100   # training epochs\n    cutoff = 15     # 3 sample spectogram (training) overwritten for inference\n\npath_switch = False\n\n# Helper Functions Stored Below","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Keras Training History\ndef HistPlot():\n\n    fig,ax = plt.subplots(1,2,figsize=(12,4))\n    sns.despine(top=True,left=True,bottom=True)\n\n    ax[0].plot(history.history['accuracy'])\n    ax[0].plot(history.history['val_accuracy'])\n    ax[0].set_title('model accuracy')\n    ax[0].set_ylabel('accuracy')\n    ax[0].set_xlabel('epoch')\n    ax[0].grid(True,linestyle='--',alpha=0.5)\n    \n    ax[1].plot(history.history['loss'])\n    ax[1].plot(history.history['val_loss'])\n    ax[1].set_title('model loss')\n    ax[1].set_ylabel('loss')\n    ax[1].set_xlabel('epoch')\n    ax[1].legend(['train', 'test'], loc='upper left')\n    ax[1].grid(True,linestyle='--',alpha=0.5)\n    plt.show()\n\n# Split the Input signal into segments\ndef split_signal(sig):\n    sig_splits = []\n    for i in range(0, len(sig), int(coefs.sl * coefs.sr)):\n        split = sig[i:i + int(coefs.sl * coefs.sr)]\n        if len(split) < int(coefs.sl * coefs.sr):\n            break\n        sig_splits.append(split)\n    \n    return sig_splits\n\n# extracts spectrograms and saves them in a working directory\ndef get_spectrograms(filepath, primary_label, output_dir):\n\n    # duration is set from global variable\n    sig, rate = librosa.load(filepath, sr=coefs.sr, offset=None, duration=coefs.cutoff)\n    sig_splits = split_signal(sig) # split the signal into parts\n    \n    # Extract mel spectrograms for each audio chunk\n    s_cnt = 0\n    saved_samples = []\n    for chunk in sig_splits:\n        \n        hop_length = int(coefs.sl * coefs.sr / (coefs.sshape[1] - 1))\n        mel_spec = librosa.feature.melspectrogram(y=chunk, \n                                                  sr=coefs.sr, \n                                                  n_fft=1024, \n                                                  hop_length=hop_length, \n                                                  n_mels=coefs.sshape[0], \n                                                  fmin=coefs.fmin, \n                                                  fmax=coefs.fmax)\n    \n        mel_spec = librosa.power_to_db(mel_spec**2, ref=np.max) \n        \n        # Normalize\n        mel_spec -= mel_spec.min()\n        mel_spec /= mel_spec.max()\n        \n        # Save as image file\n        save_dir = os.path.join(output_dir, primary_label)\n        if not os.path.exists(save_dir):\n            os.makedirs(save_dir)\n        save_path = os.path.join(save_dir, filepath.rsplit(os.sep, 1)[-1].rsplit('.', 1)[0] + \n                                 '_' + str(s_cnt) + '.png')\n        im = Image.fromarray(mel_spec * 255.0).convert(\"L\")\n        im.save(save_path)\n        \n        saved_samples.append(save_path)\n        s_cnt += 1\n        \n    return saved_samples\n\n# https://stackoverflow.com/questions/1524126/how-to-print-a-list-more-nicely\ndef list_columns(obj, cols=4, columnwise=True, gap=4):\n    sobj = [str(item) for item in obj]\n    if cols > len(sobj): cols = len(sobj)\n    max_len = max([len(item) for item in sobj])\n    if columnwise: cols = int(math.ceil(float(len(sobj)) / float(cols)))\n    plist = [sobj[i: i+cols] for i in range(0, len(sobj), cols)]\n    if columnwise:\n        if not len(plist[-1]) == cols:\n            plist[-1].extend(['']*(len(sobj) - len(plist[-1])))\n        plist = zip(*plist)\n    printer = '\\n'.join([\n        ''.join([c.ljust(max_len + gap) for c in p])\n        for p in plist])\n    print (printer)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **KERAS BASED MODEL GENERATION**\n\n<b>Notebook Aim & Modifications</b>\n\n- The aim of this notebook is to slightly expand on the already very useful notebook posted by the host; [notebook](https://www.kaggle.com/stefankahl/birdclef2021-model-training).\n- That notebook contains a <b>single layer model approach</b>, which as it turns out cannot be used with more <b>sophisticated pretrained models</b>, I regrouped a few things and overall there doesn't seem to be a big difference between the two approaches when it comes to training.\n\n<b>Dataloaders & Augmentation</b>\n\n- <b>Dataloaders</b> are used here (as opposed to the additional step of reloading the into a numpy array) in order for one to use <b>image augmentations</b>, which help improve the model during training. The winning entry of the previous competition hosted by the same lab used <b>noise</b>, as an example.\n- An example notebook which shows the <b>benefit of image augmentation</b> can be seen here; [Hummingbird Classification with CNN](https://www.kaggle.com/shtrausslearning/hummingbird-classification-with-cnn). If you are interested in <b>birds & their classification</b>, which I assumed a lot of you are, consider taking out the [Hummingbird Dataset](https://www.kaggle.com/akimball002/hummingbirds-at-my-feeders) dataset for a spin by [Amanda K Kimball](https://www.kaggle.com/akimball002/cnn-hummingbird-speciesgender-image-classification) and liking her work. Accurate bird classification most definitely requires the addition of video for accurate classification and not just sound, which is why I brought the above example up.\n\n<b>By Not Means Complete</b>\n\n- The notebook, like the one posted by the host, is by no means complete, <b>subsets are created via parameter selection</b> (rating filter,recording number per specie,general limiter)\n- They barely are even able to correctly select the correct species present in the <b>training soundscapes</b> to begin with (as you will see in the soundscape), not even having done any training, \n- It's likey this is a critical step, not just this competition, but for bird classification in general. Some ideas have already been put forward in this notebook, [At the right place in the right time?](https://www.kaggle.com/aramacus/at-the-right-place-in-the-right-time)\n\n# <sub>1.</sub> <span style='color:#F7765E'><sub>SUBSET GENERATION</sub></span>\n- As per host's notebook, a simple subset selection of potetial birds that will be present in the <b>soundscape</b> are chosen;\n    - <b>rating limitation</b>; only high quality recordings (as per Xeno Laws) are used.\n    - <b>recording per specie limitation</b>; recordings with appropriate ammount of recordings in the dataset.\n    - <b>overall limiter of rows</b>; general final limiter.\n    \n    \n- Having limited the dataset, <b>spectograms</b> are generated, these arrays are exported via images and reinported via dataloader during training.\n- <b>subset</b> is used to define which folders dataloader is the <b>training data</b> & the <b>validation data</b> using the <b>flow_from_directory</b> function/method.\n- One slight concern I have with this approach of splitting the general non (train/vald) sorted folder data via <b>identical seed</b> & <b>validation_split</b> specification in the <b>ImageDataGenerator</b> input is the potenial occurence of image leakage.","metadata":{}},{"cell_type":"code","source":"''' CREATE A SUBSET OF THE DATA '''\nprint('STEP 1) CREATING A SUBSET OF DATASET:\\n')\n\nif(path_switch):\n    lpath = '.\\\\train_metadata.csv'\nelse:\n    lpath = '../input/birdclef-2021/train_metadata.csv'\ntrain = pd.read_csv(lpath)\nprint(f\"[DATASET]: {train.values.shape} : LABELS {len(train.primary_label.value_counts())}\")\n\n# subset filter 1 (rating)\ntemp_str = 'rating>='+str(coefs.rat_id)\ntrain = train.query(temp_str)\nprint('\\nRATING LIMITER APPLIED:')\nprint(f'[SUBSET]: {train.values.shape} : LABELS {len(train.primary_label.value_counts())}')\n\n# subset filter 2 (number of recordings per specie)\nbirds_count = {};\na = train.primary_label.unique() \na_val = train.groupby('primary_label')['primary_label'].count().values\nfor bird_species, count in zip(a,a_val):\n    birds_count[bird_species] = count\nto_model_spec = [key for key,value in birds_count.items() if value >= coefs.recs] \n\nprint(f'\\n {coefs.recs}+ RECORDINGS ONLY BIRDS LIMITED:')\nTRAIN = train.query('primary_label in @to_model_spec')\nLABELS = sorted(TRAIN.primary_label.unique())\nprint(f'[SUBSET]: {TRAIN.values.shape} : LABELS {len(LABELS)}')\n\nprint('\\n BIRD LABELS AVAILABLE AFTER FILTER:')\nlist_columns(to_model_spec, cols=4, columnwise=True, gap=4)\n\n# subset filter 3 (max audio files)\n\n# Shuffle the training data and limit the number of audio files to max_files\nprint('\\nLIMITING AUDIO FILES ...')\nTRAIN = shuffle(TRAIN, random_state=coefs.seed)[:coefs.max_files]\nLABELS = sorted(TRAIN.primary_label.unique())\nprint(f'[SUBSET]: {TRAIN.values.shape} : LABELS {len(LABELS)}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' CREATE & OUTPUT SPECTOGRAMS FOR TRAINING'''\n# we will use CNN approach \n\n# Parse audio files and extract training samples\nif(path_switch):\n    input_dir = '.\\\\train_short_audio\\\\'\n    output_dir = '.\\\\working\\\\melspectrogram_dataset\\\\'\nelse:\n    input_dir = '../input/birdclef-2021/train_short_audio/'\n    output_dir = './/working/melspectrogram_dataset/'\n\nsamples = []\nwith tqdm_notebook(total=len(TRAIN)) as pbar:\n    for idx, row in TRAIN.iterrows():\n        pbar.update(1)\n        \n        if row.primary_label in to_model_spec:\n            audio_file_path = os.path.join(input_dir, row.primary_label, row.filename)\n            samples += get_spectrograms(audio_file_path, row.primary_label, output_dir)\n            \nTRAIN_SPECS = shuffle(samples, random_state=coefs.seed)\nprint('SUCCESSFULLY EXTRACTED {} SPECTROGRAMS'.format(len(TRAIN_SPECS)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' DATALOADERS '''\n# Create Data Generators/Loader for Keras, images not to be deleted\n\ntrain_folder = './working/melspectrogram_dataset/'\nvalid_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.2)\ntrain_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.2,\\\n                                   shear_range=10,fill_mode='nearest')\n\ntrain_generator = train_datagen.flow_from_directory(train_folder, \n                        target_size=(coefs.sshape[0],coefs.sshape[1]),  # target size\n                        batch_size=32, \n                        seed=42,\n                        subset = \"training\",\n                        class_mode='categorical')    # batch size\nvalidation_generator = valid_datagen.flow_from_directory(train_folder, \n                        target_size=(coefs.sshape[0],coefs.sshape[1]),  # target size\n                        batch_size=32, \n                        seed=42,\n                        subset = \"validation\",\n                        class_mode='categorical')    # batch size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <sub>2.</sub> <span style='color:#F7765E'><sub>MODEL GENERATION</sub></span>\n\n<b>Base Model</b>\n\n- The same model is used from [notebook](https://www.kaggle.com/stefankahl/birdclef2021-model-training), with the exception of a three layer <b>input shape (X,X,3)</b>\n\n<b>Pretrained Models</b>\n\n- Pretrained Models all require 3 layer inputs, in the input shape.\n- Pretrained Models are also provided in the function, <code>pretrained_model</code>, which requires one to specify which <b>head model</b> is chosen. \n- The <b>tail end</b> Dense Layer is also fixed, by no means optimal and adjusted to be used for classification in this problem.\n- <b>head weight coefficients</b> are often fixed to prevent overfitting, the same is done here.\n\n<b>The Rest</b>\n\n- Compilation settings, <b>optimiser</b>, <b>loss function</b> & <b>evaluation metric</b> are all identical to the previous notebook.\n- <b>Callbacks</b> are all quite standard, <b>TqdmCallback</b> is used to reduce keras training output.\n- <b>Train & Validation Generators</b> are used for training and evaluation during training, defined earlier.\n- <b>Results</b> of the <b>evaluation metric (accuracy)</b> & <b>model loss</b> are plotted for each iteration of image dataset passes (epoch).","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(coefs.seed)\nmodel = tf.keras.Sequential([\n    \n    # First conv block\n    tf.keras.layers.Conv2D(16, (3, 3), activation='relu', \n                           input_shape=(coefs.sshape[0], coefs.sshape[1],3)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    \n    # Second conv block\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)), \n    \n    # Third conv block\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)), \n    \n    # Fourth conv block\n    tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    \n    # Global pooling instead of flatten()\n    tf.keras.layers.GlobalAveragePooling2D(), \n    \n    # Dense block\n    tf.keras.layers.Dense(256, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),  \n    tf.keras.layers.Dense(256, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),\n    \n    # Classification layer\n    tf.keras.layers.Dense(len(LABELS), activation='softmax')\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(coefs.seed)\ndef pretrained_model(head_id):\n\n    # Define model with different applications\n    model = Sequential()\n\n    ''' Define Head Pretrained Models '''\n\n    if(head_id is 'vgg'):\n        model.add(VGG16(input_shape=(coefs.sshape[0],coefs.sshape[1],3),\n                            pooling='avg',\n                            classes=1000,\n                            include_top=False,\n                            weights='imagenet'))\n\n    elif(head_id is 'resnet'):\n        model.add(ResNet101(include_top=False,\n                               input_tensor=None,\n                               input_shape=(coefs.sshape[0],coefs.sshape[1],3),\n                               pooling='avg',\n                               classes=100,\n                               weights='imagenet'))\n\n    elif(head_id is 'mobilenet'):\n        model.add(MobileNet(alpha=1.0,\n                               depth_multiplier=1,\n                               dropout=0.001,\n                               include_top=False,\n                               weights=\"imagenet\",\n                               input_tensor=None,\n                               input_shape = (coefs.sshape[0],coefs.sshape[1],3),\n                               pooling=None,\n                               classes=1000))\n\n    elif(head_id is 'inception'):\n        # 75x75\n        model.add(InceptionV3(input_shape = (coefs.sshape[0],coefs.sshape[1],3), \n                                                    include_top = False, \n                                                    weights = 'imagenet'))\n\n    elif(head_id is 'efficientnet'):\n        model.add(EfficientNetB4(input_shape = (coefs.sshape[0],coefs.sshape[1],3), \n                                    include_top = False, \n                                    weights = 'imagenet'))\n\n    ''' Tail Model Part '''\n    model.add(Flatten())\n    model.add(Dense(1024,activation='relu'))\n    model.add(Dropout(0.01))\n    model.add(Dense(len(LABELS),activation='softmax'))\n\n    # # freeze main model coefficients\n    model.layers[0].trainable = False\n    model.summary()\n\n    return model\n\n# Select & Comment out above cell if used\n# model = pretrained_model('resnet') # define the model\n# model = tf.keras.models.load_model('../input/birdclef-resnet101-1/best_model.h5') # Reload your model ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model and specify optimizer, loss and metric\nmodel.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n              loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.01),\n              metrics=['accuracy'])\n\n# Add callbacks to reduce the learning rate if needed, early stopping, and checkpoint saving\ncallbacks = [ReduceLROnPlateau(monitor='val_loss',patience=50,verbose=1,factor=0.5),\n             EarlyStopping(monitor='val_loss',verbose=1,patience=5),\n             ModelCheckpoint(filepath='best_model.h5',monitor='val_loss',verbose=0,save_best_only=True),\n             TqdmCallback(verbose=0)\n            ]\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n                    validation_data = validation_generator,\n                    verbose = 0,\n                    callbacks=callbacks,\n                    epochs=coefs.n_epoch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Training \nHistPlot() # plot accuracy metric & loss function","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' I. HELPER FUNCTIONS'''\n# Define Path for Training & Test Soundscape Data\n# If competition reruns notebook, test data folder will be loaded\n\n# Print Available Soundscape Files\ndef print_available(verbose = False):\n    \n    def list_files(path):\n        return [os.path.join(path, f) for f in os.listdir(path) if f.rsplit('.', 1)[-1] in ['ogg']]\n    test_audio = list_files('../input/birdclef-2021/test_soundscapes')\n    if len(test_audio) == 0:\n        test_audio = list_files('../input/birdclef-2021/train_soundscapes')\n\n    if(verbose):\n        print('AVAILABLE SOUNDSCAPES:')\n        print('{} FILES IN TEST SET.'.format(len(test_audio)))\n        print('')\n    \n        ii=-1\n        for i in test_audio:\n            ii+=1\n            print(ii,i)\n        \n    return test_audio\n\n# Get the labels that will be used to train the model\ndef print_label_species():\n    list_columns(to_model_spec, cols=4, columnwise=True, gap=2)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <sub>3.</sub> <span style='color:#F7765E'><sub>SOUNDSCAPE INFERENCE (TRAIN/TEST)</sub></span>\n- <b>Function <code>soundscape_records</code> requires:</b>\n    - pathway to <b>soundscape</b> file.\n    - model to be used for <b>evaluation (predict)</b>\n    - submission option allows one to quickly switch from <b>soundscape investigations/confirmations</b> to <b>submission format</b>\n    \n    \n- <b>The function does the following:</b>\n    - Firstly, clears common/temporary <code>mel_soundscape</code> folder used for <b>spectogram</b> export for each individual soundscape file.\n    - Reads soundscape audio using librosa & temporary store 600 segment spectrums in folder (each image is a 5 second segment); <code>mel_soundscape</code>.\n    - Keras <b>Dataloader/Data Generator is created</b> for the \"test\" set of 600 spectrum segments.\n    - <b>Inference is conducted</b>, using <b>imported model</b> using \"test\" data generator (w/ standard augmentation), <b>results (probability)</b> are stored in a common array and extracted individually. \n    - For each segment, if a probability exceeds <b>a threshold</b>, the label corresponding to that probability is stored in dictionary, <b>data</b>. If none of the probabilites exceeds this threshold, a no bird call result is stored <b>nocall</b>.\n    - Individual soundscape results are stored in local DataFrame and passed via return, the results for all possible soundscape files are then stored in a unified DataFrame <b>(df_allres)</b>, which is used for submission.\n   \n   \n- <b>Unified submission option (submission=False)</b>:\n    - <b>False</b> is used for different training soundscape investigations, eg. comparison of birds correctly predicted / all available birds ..., \n    - <b>True</b> option is used for creating a submission...","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\n# Function to evaluate inference on given recording\ndef soundscape_records(path,model,submission=False):\n    \n    # before prediction clear read folder if it exists\n    if(os.path.exists('.//working/mel_soundscape/')):\n        shutil.rmtree('.//working/mel_soundscape/')\n#         os.listdir('.//working//')\n\n    # General Output / Submission DataFrame Structure  \n    if(submission is False):\n        data = {'row_id': [], 'prediction': [], 'score': []}\n    else:\n        data = {'row_id': [], 'birds': []}\n    \n    print('*** READING NEW FILE & STARTING PREDICTION... ***')\n    print(f'Reading File: {path}')\n    \n    ''' 1. CREATE SOUNDSCAPE FILES AND SAVE THEM '''\n    # for each spectogram input, call get_spectogram, which cuts the entire\n    # soundscape into chunks of 5 seconds\n    \n    coefs.cutoff = 600 # change to get all 5s segments in soundscape; should make 120 files\n    get_spectrograms(path,'soundscape','.//working/mel_soundscape/')\n#     print(len(os.listdir('.//working/mel_soundscape/soundscape'))) # should be 120\n    \n    ''' 2. LOAD IMAGE FILES & DATALOADER '''\n    # soundscape recording folder\n    soundscape_folder = './/working/mel_soundscape/'   \n    # image augmentation & generate dataloader\n    gen_datagen = ImageDataGenerator(rescale=1./255)\n    gen_test = gen_datagen.flow_from_directory(soundscape_folder,\n                        target_size=(coefs.sshape[0],coefs.sshape[1]),\n                        batch_size=32,\n                        class_mode='categorical')\n    \n    ''' 3. MAKE MODEL PREDICTION '''\n    # for each class predict probability for all images simulaneously\n    scores = model.predict(gen_test, verbose=1)\n    \n    # For each soundcape -> create X chunks + predict each \n    \n    time_id=0\n    for i in range(len(scores)):\n    \n        time_id+=5 # update segment time interval \n        idx = scores[i].argmax()      # possibly not best choice\n        species = LABELS[idx]\n        score = scores[i][idx]\n        \n        data['row_id'].append(path.split(os.sep)[-1].rsplit('_', 1)[0] + '_' + str(time_id))\n        \n        ''' *DECIDE IF PREDICTION PROBABILITY SHOULD EXCEED THRESHOLD '''\n        if score > coefs.thresh:\n            if(submission is False):\n                data['prediction'].append(species)\n            else:\n                data['birds'].append(species)\n        else:\n            if(submission is False):\n                data['prediction'].append('nocall')\n            else:\n                data['birds'].append('nocall')\n        \n         # store score\n        if(submission is False):\n            data['score'].append(score) # Add the confidence score as well\n        \n    # COMBINE & SHOW RESULTS\n    if(submission is False):\n        results = pd.DataFrame(data, columns = ['row_id', 'prediction', 'score'])\n    else:\n        results = pd.DataFrame(data, columns = ['row_id', 'birds'])\n    \n    if(submission is False):\n        gt = pd.read_csv('../input/birdclef-2021/train_soundscape_labels.csv')\n        results = pd.merge(gt, results, on='row_id') # merge only at available rows\n        results['outcome'] = results.birds == results.prediction\n        intersection_set = list(set(LABELS) & set(results.birds.to_list()))\n\n        print('1A. Before Prediction:')\n        list_columns(LABELS, cols=8, columnwise=True, gap=4)\n        print('1B. Birds Present')\n        list_columns(results.birds.unique())\n        print(f'bird overlap: {len(intersection_set)}/{len(results.birds.unique())} are even present')\n\n        print('\\n 2. All Predictions:')\n        print(results.outcome.value_counts())\n        print('')\n\n        print('3. Bird Predictions Only:')\n        df_bird = results[results.birds!='nocall']\n        print(df_bird.outcome.value_counts())\n        print('\\n\\n')\n        return 0\n    else:\n        return results # return one soundscape inference\n    \n''' MAIN INFERENCE OPTIONS '''\n# Use model to evaluate on soundscape segments in one or many files\n    \n# A. Get Pathways to Soundscapes \ntest_audio = print_available()\n    \n# B. Inference on all Soundscape, tlist stores all soundscape individual results\nii=-1;tlist = []\nfor i in test_audio:\n#     model = tf.keras.models.load_model('best_model.h5') # load external model\n    ii+=1;df_infer = soundscape_records(test_audio[ii],model,coefs.submission)\n    if(coefs.submission):\n        tlist.append(df_infer)\n\n# combine all soundscape inference results\nif(coefs.submission):    \n    df_allres = pd.concat(tlist)\n    df_allres.to_csv(\"submission.csv\", index=False)\n    \n# C. Inference for one soundscape\n# model = tf.keras.models.load_model('best_model.h5')\n# soundscape_records(test_audio[0],model,coefs.submission)\n\n# Remove Training Spectrums to not show them in output\nshutil.rmtree('.//working/melspectrogram_dataset/')\nos.listdir('.//working//')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}