{"cells":[{"metadata":{},"cell_type":"markdown","source":"* Hi to all! as a basis I took this notebook :https://www.kaggle.com/fffrrt/all-in-one-rfcx-baseline-for-beginners. \n* In this notebook, I implemented a K-fold CV and tuned the base model.\n* Good luck to all and special thanks to the author of the basic notebook."},{"metadata":{},"cell_type":"markdown","source":"WARNING: Kernel fails to automatically score if more than one file is saved to disk. You can still download and manually submit prediction. To allow model/spectrograms saving, change setting below."},{"metadata":{"trusted":true},"cell_type":"code","source":"save_to_disk = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generating Mel spectrograms for training from true positive data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import csv\nimport librosa\nimport numpy as np\nfrom skimage.transform import resize\nfrom PIL import Image\n\nfft = 2048\nhop = 512\n# Less rounding errors this way\nsr = 48000\nlength = 10 * sr\n\nwith open('/kaggle/input/rfcx-species-audio-detection/train_tp.csv') as f:\n    reader = csv.reader(f)\n    data = list(reader)\n\n# Check minimum/maximum frequencies for bird calls\n# Not neccesary, but there are usually plenty of noise in low frequencies, and removing it helps\nfmin = 24000\nfmax = 0\n\n# Skip header row (recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max) and start from 1 instead of 0\nfor i in range(1, len(data)):\n    if fmin > float(data[i][4]):\n        fmin = float(data[i][4])\n    if fmax < float(data[i][6]):\n        fmax = float(data[i][6])\n# Get some safety margin\nfmin = int(fmin * 0.9)\nfmax = int(fmax * 1.1)\nprint('Minimum frequency: ' + str(fmin) + ', maximum frequency: ' + str(fmax))\n\n\nprint('Starting spectrogram generation')\nfor i in range(1, len(data)):\n    # All sound files are 48000 bitrate, no need to slowly resample\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/train/' + data[i][0] + '.flac', sr=None)\n    \n    t_min = float(data[i][3]) * sr\n    t_max = float(data[i][5]) * sr\n    \n    # Positioning sound slice\n    center = np.round((t_min + t_max) / 2)\n    beginning = center - length / 2\n    if beginning < 0:\n        beginning = 0\n    \n    ending = beginning + length\n    if ending > len(wav):\n        ending = len(wav)\n        beginning = ending - length\n        \n    slice = wav[int(beginning):int(ending)]\n    \n    # Mel spectrogram generation\n    # Default settings were bad, parameters are adjusted to generate somewhat reasonable quality images\n    # The better your images are, the better your neural net would perform\n    # You can also use librosa.stft + librosa.amplitude_to_db instead\n    mel_spec = librosa.feature.melspectrogram(slice, n_fft=fft, hop_length=hop, sr=sr, fmin=fmin, fmax=fmax, power=1.5)\n    mel_spec = resize(mel_spec, (224, 400))\n    \n    # Normalize to 0...1 - this is what goes into neural net\n    mel_spec = mel_spec - np.min(mel_spec)\n    mel_spec = mel_spec / np.max(mel_spec)\n\n    # And this 0...255 is for the saving in bmp format\n    mel_spec = mel_spec * 255\n    mel_spec = np.round(mel_spec)    \n    mel_spec = mel_spec.astype('uint8')\n    mel_spec = np.asarray(mel_spec)\n    \n    bmp = Image.fromarray(mel_spec, 'L')\n    bmp.save('/kaggle/working/' + data[i][0] + '_' + data[i][1] + '_' + str(center) + '.bmp')\n    \n    if i % 100 == 0:\n        print('Processed ' + str(i) + ' train examples from ' + str(len(data)))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Settings and random seeds initialization for reproducible results"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport random\n\nnum_birds = 24\n# 6GB GPU-friendly (~4 GB used by model)\n# Increase if neccesary\nbatch_size = 16\n\n# This is enough to exactly reproduce results on local machine (Windows / Turing GPU)\n# Kaggle GPU kernels (Linux / Pascal GPU) are not deterministic even with random seeds set\n# Your score might vary a lot (~up to 0.05) on a different runs due to picking different epochs to submit\nrng_seed = 1234\nrandom.seed(rng_seed)\nnp.random.seed(rng_seed)\nos.environ['PYTHONHASHSEED'] = str(rng_seed)\ntorch.manual_seed(rng_seed)\ntorch.cuda.manual_seed(rng_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model dataset class"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.utils.data as torchdata\n\nclass RainforestDataset(torchdata.Dataset):\n    def __init__(self, filelist):\n        self.specs = []\n        self.labels = []\n        for f in filelist:\n            # Easier to pass species in filename at the start; worth changing later to more capable method\n            label = int(str.split(f, '_')[1])\n            label_array = np.zeros(num_birds, dtype=np.single)\n            label_array[label] = 1.\n            self.labels.append(label_array)\n            \n            # Open and save spectrogram to memory\n            \n            # If you use more spectrograms (add train_fp, for example), then they would not all fit to memory\n            # In this case you should load them on the fly in __getitem__\n            img = Image.open('/kaggle/working/' + f)\n            mel_spec = np.array(img)\n            img.close()\n            \n            # Transforming spectrogram from bmp to 0..1 array\n            mel_spec = mel_spec / 255\n            # Stacking for 3-channel image for resnet\n            mel_spec = np.stack((mel_spec, mel_spec, mel_spec))\n            \n            self.specs.append(mel_spec)\n    \n    def __len__(self):\n        return len(self.specs)\n    \n    def __getitem__(self, item):\n        # Augment here if you want\n        return self.specs[item], self.labels[item]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split training set on training and validation  \n  \nWhat StratifiedKFold does:  \n![StratifiedKFold](https://scikit-learn.org/stable/_images/sphx_glr_plot_cv_indices_003.png)"},{"metadata":{},"cell_type":"markdown","source":"Preparing everything for training"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install resnest > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nfrom resnest.torch import resnest50\n\n\n# ResNeSt: Split-Attention Networks\n# https://arxiv.org/abs/2004.08955\n# Significantly outperforms standard Resnet\ndef get_model():\n    model = resnest50(pretrained=True)\n\n    model.fc = nn.Sequential(\n        nn.Linear(2048, 1024),\n        nn.ReLU(),\n        nn.Dropout(p=0.4),\n        nn.Linear(1024, 1024),\n        nn.ReLU(),\n        nn.Dropout(p=0.4),\n        nn.Linear(1024, num_birds)\n    )\n\n    # Picked for this notebook; pick new ones after major changes (such as adding train_fp to train data)\n    optimizer = torch.optim.SGD(model.parameters(), lr=0.01, weight_decay=0.0001, momentum=0.9)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.4)\n\n    # This loss function is not exactly suited for competition metric, which only cares about ranking of predictions\n    # Exploring different loss fuctions would be a good idea\n    pos_weights = torch.ones(num_birds)\n    pos_weights = pos_weights * num_birds\n    loss_function = nn.BCEWithLogitsLoss(pos_weight=pos_weights)\n\n    if torch.cuda.is_available():\n        model = model.cuda()\n        loss_function = loss_function.cuda()\n    return model,optimizer,scheduler,loss_function","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training model on saved spectrograms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, n_epochs, train_loader, val_loader, fold_id, optimizer,scheduler,loss_function):\n    best_corrects = 0\n\n    # Train loop\n    print('Starting training loop')\n    for e in range(0, n_epochs):\n        # Stats\n        train_loss = []\n        train_corr = []\n\n        # Single epoch - train\n        model.train()\n        for batch, (data, target) in enumerate(train_loader):\n            data = data.float()\n            if torch.cuda.is_available():\n                data, target = data.cuda(), target.cuda()\n\n            optimizer.zero_grad()\n\n            output = model(data)\n            loss = loss_function(output, target)\n\n            loss.backward()\n            optimizer.step()\n\n            # Stats\n            vals, answers = torch.max(output, 1)\n            vals, targets = torch.max(target, 1)\n            corrects = 0\n            for i in range(0, len(answers)):\n                if answers[i] == targets[i]:\n                    corrects = corrects + 1\n            train_corr.append(corrects)\n\n            train_loss.append(loss.item())\n\n        # Stats\n        for g in optimizer.param_groups:\n            lr = g['lr']\n        print('Epoch ' + str(e) + ' training end. LR: ' + str(lr) + ', Loss: ' + str(sum(train_loss) / len(train_loss)) +\n              ', Correct answers: ' + str(sum(train_corr)) + '/' + str(train_dataset.__len__()))\n\n        # Single epoch - validation\n        with torch.no_grad():\n            # Stats\n            val_loss = []\n            val_corr = []\n\n            model.eval()\n            for batch, (data, target) in enumerate(val_loader):\n                data = data.float()\n                if torch.cuda.is_available():\n                    data, target = data.cuda(), target.cuda()\n\n                output = model(data)\n                loss = loss_function(output, target)\n\n                # Stats\n                vals, answers = torch.max(output, 1)\n                vals, targets = torch.max(target, 1)\n                corrects = 0\n                for i in range(0, len(answers)):\n                    if answers[i] == targets[i]:\n                        corrects = corrects + 1\n                val_corr.append(corrects)\n\n                val_loss.append(loss.item())\n\n        # Stats\n        print('Epoch ' + str(e) + ' validation end. LR: ' + str(lr) + ', Loss: ' + str(sum(val_loss) / len(val_loss)) +\n              ', Correct answers: ' + str(sum(val_corr)) + '/' + str(val_dataset.__len__()))\n\n        # If this epoch is better than previous on validation, save model\n        # Validation loss is the more common metric, but in this case our loss is misaligned with competition metric, making accuracy a better metric\n        if sum(val_corr) > best_corrects:\n            print('Saving new best model at epoch ' + str(e) + ' (' + str(sum(val_corr)) + '/' + str(val_dataset.__len__()) + ')')\n            torch.save(model, 'best_model_[0].pt'.format(fold_id))\n            best_corrects = sum(val_corr)\n\n        # Call every epoch\n        scheduler.step()\n\n    # Free memory\n    del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_epochs=35\nn_splits=5\nfile_list = []\nlabel_list = []\n\nfor f in os.listdir('/kaggle/working/'):\n    if '.bmp' in f:\n        file_list.append(f)\n        label = str.split(f, '_')[1]\n        label_list.append(label)\n\n\nfrom sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=rng_seed)\n\ntrain_files = []\nval_files = []\n\nfor fold_id, (train_index, val_index) in enumerate(skf.split(file_list, label_list)):\n    print ('fold_id:',fold_id)\n    # Picking only first fold to train/val on\n    # This means loss of 20% training data\n    # To avoid this, you can train 5 different models on 5 folds and average predictions\n    train_files = np.take(file_list, train_index)\n    val_files = np.take(file_list, val_index)\n    train_dataset = RainforestDataset(train_files)\n    val_dataset = RainforestDataset(val_files)\n\n    train_loader = torchdata.DataLoader(train_dataset, batch_size=batch_size, sampler=torchdata.RandomSampler(train_dataset))\n    val_loader = torchdata.DataLoader(val_dataset, batch_size=batch_size, sampler=torchdata.RandomSampler(val_dataset))\n    model,optimizer,scheduler,loss_function=get_model()\n    train_model(model, n_epochs, train_loader, val_loader, fold_id,optimizer,scheduler,loss_function)\n\n    print('Training on ' + str(len(train_files)) + ' examples')\n    print('Validating on ' + str(len(val_files)) + ' examples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Function to split and load one test file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Already defined above; for reference\n\n# fft = 2048\n# hop = 512\n# sr = 48000\n# length = 10 * sr\n\ndef load_test_file(f):\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/test/' + f, sr=None)\n\n    # Split for enough segments to not miss anything\n    segments = len(wav) / length\n    segments = int(np.ceil(segments))\n    \n    mel_array = []\n    \n    for i in range(0, segments):\n        # Last segment going from the end\n        if (i + 1) * length > len(wav):\n            slice = wav[len(wav) - length:len(wav)]\n        else:\n            slice = wav[i * length:(i + 1) * length]\n        \n        # Same mel spectrogram as before\n        mel_spec = librosa.feature.melspectrogram(slice, n_fft=fft, hop_length=hop, sr=sr, fmin=fmin, fmax=fmax, power=1.5)\n        mel_spec = resize(mel_spec, (224, 400))\n    \n        mel_spec = mel_spec - np.min(mel_spec)\n        mel_spec = mel_spec / np.max(mel_spec)\n        \n        mel_spec = np.stack((mel_spec, mel_spec, mel_spec))\n\n        mel_array.append(mel_spec)\n    \n    return mel_array","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submitting predictions with best model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading model back\nmodels=[]\nfor model_id in range (n_splits):\n    \n    model = torch.load('/kaggle/working/best_model_[0].pt'.format(model_id))\n    model.eval()\n    if torch.cuda.is_available():\n        model.cuda()\n    models.append(model)\n\n# Scoring does not like many files:(\nif save_to_disk == 0:\n    for f in os.listdir('/kaggle/working/'):\n        os.remove('/kaggle/working/' + f)\n\n\n#   \n## Prediction loop\nprint('Starting prediction loop')\nwith open('submission.csv', 'w', newline='') as csvfile:\n    submission_writer = csv.writer(csvfile, delimiter=',')\n    submission_writer.writerow(['recording_id','s0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11',\n                               's12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23'])\n    \n    test_files = os.listdir('/kaggle/input/rfcx-species-audio-detection/test/')\n    print(len(test_files))\n    \n    # Every test file is split on several chunks and prediction is made for each chunk\n    for i in range(0, len(test_files)):\n        data = load_test_file(test_files[i])\n        data = torch.tensor(data)\n        data = data.float()\n        output=torch.zeros([6, 24]) \n        if torch.cuda.is_available():\n            output=torch.zeros([6, 24], device='cuda') \n            data = data.cuda()\n              \n        for model_id in range (n_splits):\n            output+= models[model_id](data)/n_splits\n            #print (output.shape)\n            \n        \n        # Taking max prediction from all slices per bird species\n        # Usually you want Sigmoid layer here to convert output to probabilities\n        # In this competition only relative ranking matters, and not the exact value of prediction, so we can use it directly\n        maxed_output = torch.max(output, dim=0)[0]\n        \n        maxed_output = maxed_output.cpu().detach()\n        \n        \n        file_id = str.split(test_files[i], '.')[0]\n        write_array = [file_id]\n        \n        for out in maxed_output:\n            write_array.append(out.item())\n    \n        submission_writer.writerow(write_array)\n        \n        if i % 100 == 0 and i > 0:\n            print('Predicted for ' + str(i) + ' of ' + str(len(test_files) + 1) + ' files')\n\nprint('Submission generated')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}