{"cells":[{"metadata":{},"cell_type":"markdown","source":"WARNING: Kernel fails to automatically score if more than one file is saved to disk. You can still download and manually submit prediction. To allow model/spectrograms saving, change setting below."},{"metadata":{"trusted":true},"cell_type":"code","source":"save_to_disk = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generating Mel spectrograms for training from true positive data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import csv\nimport librosa\nimport numpy as np\nfrom skimage.transform import resize\nfrom PIL import Image\n\nfft = 2048\nhop = 512\n# Less rounding errors this way\nsr = 48000\nlength = 10 * sr\n\nwith open('/kaggle/input/rfcx-species-audio-detection/train_tp.csv') as f:\n    reader = csv.reader(f)\n    data = list(reader)\n\n# Check minimum/maximum frequencies for bird calls\n# Not neccesary, but there are usually plenty of noise in low frequencies, and removing it helps\nfmin = 24000\nfmax = 0\n\n# Skip header row (recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max) and start from 1 instead of 0\nfor i in range(1, len(data)):\n    if fmin > float(data[i][4]):\n        fmin = float(data[i][4])\n    if fmax < float(data[i][6]):\n        fmax = float(data[i][6])\n# Get some safety margin\nfmin = int(fmin * 0.9)\nfmax = int(fmax * 1.1)\nprint('Minimum frequency: ' + str(fmin) + ', maximum frequency: ' + str(fmax))\n\n\nprint('Starting spectrogram generation')\nfor i in range(1, len(data)):\n    # All sound files are 48000 bitrate, no need to slowly resample\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/train/' + data[i][0] + '.flac', sr=None)\n    \n    t_min = float(data[i][3]) * sr\n    t_max = float(data[i][5]) * sr\n    \n    # Positioning sound slice\n    center = np.round((t_min + t_max) / 2)\n    beginning = center - length / 2\n    if beginning < 0:\n        beginning = 0\n    \n    ending = beginning + length\n    if ending > len(wav):\n        ending = len(wav)\n        beginning = ending - length\n        \n    slice = wav[int(beginning):int(ending)]\n    \n    # Mel spectrogram generation\n    # Default settings were bad, parameters are adjusted to generate somewhat reasonable quality images\n    # The better your images are, the better your neural net would perform\n    # You can also use librosa.stft + librosa.amplitude_to_db instead\n    mel_spec = librosa.feature.melspectrogram(slice, n_fft=fft, hop_length=hop, sr=sr, fmin=fmin, fmax=fmax, power=1.5)\n    mel_spec = resize(mel_spec, (224, 400))\n    \n    # Normalize to 0...1 - this is what goes into neural net\n    mel_spec = mel_spec - np.min(mel_spec)\n    mel_spec = mel_spec / np.max(mel_spec)\n\n    # And this 0...255 is for the saving in bmp format\n    mel_spec = mel_spec * 255\n    mel_spec = np.round(mel_spec)    \n    mel_spec = mel_spec.astype('uint8')\n    mel_spec = np.asarray(mel_spec)\n    \n    bmp = Image.fromarray(mel_spec, 'L')\n    bmp.save('/kaggle/working/' + data[i][0] + '_' + data[i][1] + '_' + str(center) + '.bmp')\n    \n    if i % 100 == 0:\n        print('Processed ' + str(i) + ' train examples from ' + str(len(data)))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Settings and random seeds initialization for reproducible results"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport random\n\nnum_birds = 24\n# 6GB GPU-friendly (~4 GB used by model)\n# Increase if neccesary\nbatch_size = 16\n\n# This is enough to exactly reproduce results on local machine (Windows / Turing GPU)\n# Kaggle GPU kernels (Linux / Pascal GPU) are not deterministic even with random seeds set\n# Your score might vary a lot (~up to 0.05) on a different runs due to picking different epochs to submit\nrng_seed = 1234\nrandom.seed(rng_seed)\nnp.random.seed(rng_seed)\nos.environ['PYTHONHASHSEED'] = str(rng_seed)\ntorch.manual_seed(rng_seed)\ntorch.cuda.manual_seed(rng_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model dataset class"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.utils.data as torchdata\n\nclass RainforestDataset(torchdata.Dataset):\n    def __init__(self, filelist):\n        self.specs = []\n        self.labels = []\n        for f in filelist:\n            # Easier to pass species in filename at the start; worth changing later to more capable method\n            label = int(str.split(f, '_')[1])\n            label_array = np.zeros(num_birds, dtype=np.single)\n            label_array[label] = 1.\n            self.labels.append(label_array)\n            \n            # Open and save spectrogram to memory\n            \n            # If you use more spectrograms (add train_fp, for example), then they would not all fit to memory\n            # In this case you should load them on the fly in __getitem__\n            img = Image.open('/kaggle/working/' + f)\n            mel_spec = np.array(img)\n            img.close()\n            \n            # Transforming spectrogram from bmp to 0..1 array\n            mel_spec = mel_spec / 255\n            # Stacking for 3-channel image for resnet\n            mel_spec = np.stack((mel_spec, mel_spec, mel_spec))\n            \n            self.specs.append(mel_spec)\n    \n    def __len__(self):\n        return len(self.specs)\n    \n    def __getitem__(self, item):\n        # Augment here if you want\n        return self.specs[item], self.labels[item]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split training set on training and validation  \n  \nWhat StratifiedKFold does:  \n![StratifiedKFold](https://scikit-learn.org/stable/_images/sphx_glr_plot_cv_indices_003.png)"},{"metadata":{"trusted":true},"cell_type":"code","source":"file_list = []\nlabel_list = []\n\nfor f in os.listdir('/kaggle/working/'):\n    if '.bmp' in f:\n        file_list.append(f)\n        label = str.split(f, '_')[1]\n        label_list.append(label)\n\n\nfrom sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=rng_seed)\n\ntrain_files = []\nval_files = []\n\nfor fold_id, (train_index, val_index) in enumerate(skf.split(file_list, label_list)):\n    # Picking only first fold to train/val on\n    # This means loss of 20% training data\n    # To avoid this, you can train 5 different models on 5 folds and average predictions\n    if fold_id == 0:\n        train_files = np.take(file_list, train_index)\n        val_files = np.take(file_list, val_index)\n\nprint('Training on ' + str(len(train_files)) + ' examples')\nprint('Validating on ' + str(len(val_files)) + ' examples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Preparing everything for training"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install resnest > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nfrom resnest.torch import resnest50\n\ntrain_dataset = RainforestDataset(train_files)\nval_dataset = RainforestDataset(val_files)\n\ntrain_loader = torchdata.DataLoader(train_dataset, batch_size=batch_size, sampler=torchdata.RandomSampler(train_dataset))\nval_loader = torchdata.DataLoader(val_dataset, batch_size=batch_size, sampler=torchdata.RandomSampler(val_dataset))\n\n# ResNeSt: Split-Attention Networks\n# https://arxiv.org/abs/2004.08955\n# Significantly outperforms standard Resnet\nmodel = resnest50(pretrained=True)\n\nmodel.fc = nn.Sequential(\n    nn.Linear(2048, 1024),\n    nn.ReLU(),\n    nn.Dropout(p=0.2),\n    nn.Linear(1024, 1024),\n    nn.ReLU(),\n    nn.Dropout(p=0.2),\n    nn.Linear(1024, num_birds)\n)\n\n# Picked for this notebook; pick new ones after major changes (such as adding train_fp to train data)\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01, weight_decay=0.0001, momentum=0.9)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.4)\n\n# This loss function is not exactly suited for competition metric, which only cares about ranking of predictions\n# Exploring different loss fuctions would be a good idea\npos_weights = torch.ones(num_birds)\npos_weights = pos_weights * num_birds\nloss_function = nn.BCEWithLogitsLoss(pos_weight=pos_weights)\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    loss_function = loss_function.cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training model on saved spectrograms"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_corrects = 0\n\n# Train loop\nprint('Starting training loop')\nfor e in range(0, 32):\n    # Stats\n    train_loss = []\n    train_corr = []\n    \n    # Single epoch - train\n    model.train()\n    for batch, (data, target) in enumerate(train_loader):\n        data = data.float()\n        if torch.cuda.is_available():\n            data, target = data.cuda(), target.cuda()\n            \n        optimizer.zero_grad()\n        \n        output = model(data)\n        loss = loss_function(output, target)\n        \n        loss.backward()\n        optimizer.step()\n        \n        # Stats\n        vals, answers = torch.max(output, 1)\n        vals, targets = torch.max(target, 1)\n        corrects = 0\n        for i in range(0, len(answers)):\n            if answers[i] == targets[i]:\n                corrects = corrects + 1\n        train_corr.append(corrects)\n        \n        train_loss.append(loss.item())\n    \n    # Stats\n    for g in optimizer.param_groups:\n        lr = g['lr']\n    print('Epoch ' + str(e) + ' training end. LR: ' + str(lr) + ', Loss: ' + str(sum(train_loss) / len(train_loss)) +\n          ', Correct answers: ' + str(sum(train_corr)) + '/' + str(train_dataset.__len__()))\n    \n    # Single epoch - validation\n    with torch.no_grad():\n        # Stats\n        val_loss = []\n        val_corr = []\n        \n        model.eval()\n        for batch, (data, target) in enumerate(val_loader):\n            data = data.float()\n            if torch.cuda.is_available():\n                data, target = data.cuda(), target.cuda()\n            \n            output = model(data)\n            loss = loss_function(output, target)\n            \n            # Stats\n            vals, answers = torch.max(output, 1)\n            vals, targets = torch.max(target, 1)\n            corrects = 0\n            for i in range(0, len(answers)):\n                if answers[i] == targets[i]:\n                    corrects = corrects + 1\n            val_corr.append(corrects)\n        \n            val_loss.append(loss.item())\n    \n    # Stats\n    print('Epoch ' + str(e) + ' validation end. LR: ' + str(lr) + ', Loss: ' + str(sum(val_loss) / len(val_loss)) +\n          ', Correct answers: ' + str(sum(val_corr)) + '/' + str(val_dataset.__len__()))\n    \n    # If this epoch is better than previous on validation, save model\n    # Validation loss is the more common metric, but in this case our loss is misaligned with competition metric, making accuracy a better metric\n    if sum(val_corr) > best_corrects:\n        print('Saving new best model at epoch ' + str(e) + ' (' + str(sum(val_corr)) + '/' + str(val_dataset.__len__()) + ')')\n        torch.save(model, 'best_model.pt')\n        best_corrects = sum(val_corr)\n        \n    # Call every epoch\n    scheduler.step()\n\n# Free memory\ndel model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Function to split and load one test file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Already defined above; for reference\n\n# fft = 2048\n# hop = 512\n# sr = 48000\n# length = 10 * sr\n\ndef load_test_file(f):\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/test/' + f, sr=None)\n\n    # Split for enough segments to not miss anything\n    segments = len(wav) / length\n    segments = int(np.ceil(segments))\n    \n    mel_array = []\n    \n    for i in range(0, segments):\n        # Last segment going from the end\n        if (i + 1) * length > len(wav):\n            slice = wav[len(wav) - length:len(wav)]\n        else:\n            slice = wav[i * length:(i + 1) * length]\n        \n        # Same mel spectrogram as before\n        mel_spec = librosa.feature.melspectrogram(slice, n_fft=fft, hop_length=hop, sr=sr, fmin=fmin, fmax=fmax, power=1.5)\n        mel_spec = resize(mel_spec, (224, 400))\n    \n        mel_spec = mel_spec - np.min(mel_spec)\n        mel_spec = mel_spec / np.max(mel_spec)\n        \n        mel_spec = np.stack((mel_spec, mel_spec, mel_spec))\n\n        mel_array.append(mel_spec)\n    \n    return mel_array","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submitting predictions with best model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading model back\nmodel = resnest50(pretrained=True)\n\nmodel.fc = nn.Sequential(\n    nn.Linear(2048, 1024),\n    nn.ReLU(),\n    nn.Dropout(p=0.2),\n    nn.Linear(1024, 1024),\n    nn.ReLU(),\n    nn.Dropout(p=0.2),\n    nn.Linear(1024, num_birds)\n)\n\nmodel = torch.load('/kaggle/working/best_model.pt')\nmodel.eval()\n\n# Scoring does not like many files:(\nif save_to_disk == 0:\n    for f in os.listdir('/kaggle/working/'):\n        os.remove('/kaggle/working/' + f)\n\nif torch.cuda.is_available():\n    model.cuda()\n    \n# Prediction loop\nprint('Starting prediction loop')\nwith open('submission.csv', 'w', newline='') as csvfile:\n    submission_writer = csv.writer(csvfile, delimiter=',')\n    submission_writer.writerow(['recording_id','s0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11',\n                               's12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23'])\n    \n    test_files = os.listdir('/kaggle/input/rfcx-species-audio-detection/test/')\n    print(len(test_files))\n    \n    # Every test file is split on several chunks and prediction is made for each chunk\n    for i in range(0, len(test_files)):\n        data = load_test_file(test_files[i])\n        data = torch.tensor(data)\n        data = data.float()\n        if torch.cuda.is_available():\n            data = data.cuda()\n\n        output = model(data)\n\n        # Taking max prediction from all slices per bird species\n        # Usually you want Sigmoid layer here to convert output to probabilities\n        # In this competition only relative ranking matters, and not the exact value of prediction, so we can use it directly\n        maxed_output = torch.max(output, dim=0)[0]\n        maxed_output = maxed_output.cpu().detach()\n        \n        file_id = str.split(test_files[i], '.')[0]\n        write_array = [file_id]\n        \n        for out in maxed_output:\n            write_array.append(out.item())\n    \n        submission_writer.writerow(write_array)\n        \n        if i % 100 == 0 and i > 0:\n            print('Predicted for ' + str(i) + ' of ' + str(len(test_files) + 1) + ' files')\n\nprint('Submission generated')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}