{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"save_to_disk = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage.transform import resize\nfrom skimage.filters import gaussian\nfrom skimage.color import rgb2gray\nfrom skimage import exposure, util\nimport cv2\nimport numpy as np\nimport random\n\ndef horizontal_flip(img):\n    horizontal_flip_img = img[:, ::-1]\n    return addChannels(horizontal_flip_img)\n\ndef vertical_flip(img):\n    vertical_flip_img = img[::-1, :]\n    return addChannels(vertical_flip_img)\n\ndef addNoisy(img):\n    noise_img = util.random_noise(img)\n    return addChannels(noise_img)\n\ndef contrast_stretching(img):\n    contrast_img = exposure.rescale_intensity(img)\n    return addChannels(contrast_img)\n\ndef randomGaussian(img):\n    gaussian_img = gaussian(img)\n    return addChannels(gaussian_img)\n\ndef grayScale(img):\n    gray_img = rgb2gray(img)\n    return addChannels(gray_img)\n\ndef randomGamma(img):\n    img_gamma = exposure.adjust_gamma(img)\n    return addChannels(img_gamma)\n\ndef addChannels(img):\n    return np.stack((img, img, img))\n\ndef spec_to_image(spec):    \n    spec = resize(spec, (224, 400))\n    eps=1e-6\n    mean = spec.mean()\n    std = spec.std()\n    spec_norm = (spec - mean) / (std + eps)\n    spec_min, spec_max = spec_norm.min(), spec_norm.max()\n    spec_scaled = 255 * (spec_norm - spec_min) / (spec_max - spec_min)\n    spec_scaled = spec_scaled.astype(np.uint8)\n    spec_scaled = np.asarray(spec_scaled)\n    return spec_scaled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa\nfrom torch.utils.data import Dataset, DataLoader\n\nclass AudioData(Dataset):\n    def __init__(self, X, y, data_type):\n        self.data = []\n        self.labels = []\n        for i in range(0, len(X)):\n            recording_id = X[i]\n            label = int(y[i])\n            mel_spec = audio_data[recording_id]['original']\n            self.data.append(mel_spec)\n            self.labels.append(label)\n            \n            if data_type == \"train\":\n                for mel_spec in audio_data[recording_id]['augmentation']:\n                    self.data.append(mel_spec)\n                    self.labels.append(label)\n                \n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        return self.data[idx], self.labels[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models import resnet50\nimport torch\nimport torch.nn as nn\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim import Optimizer\n\nclass Adas(Optimizer):\n    r\"\"\"\n    Introduction:\n        For the mathematical part see https://github.com/YanaiEliyahu/AdasOptimizer,\n        the `Theory` section contains the major innovation,\n        and then `How ADAS works` contains more low level details that are still somewhat related to the theory.\n    Arguments:\n        params (iterable): iterable of parameters to optimize or dicts defining parameter groups\n        lr: float > 0. Initial learning rate that is per feature/input (e.g. dense layer with N inputs and M outputs, will have N learning rates).\n        lr2: float >= 0.  lr's Initial learning rate. (just ~1-2 per layer, additonal one because of bias)\n        lr3: float >= 0. lr2's fixed learning rate. (global)\n        beta_1: 0 < float < 1. Preferably close to 1. Second moments decay factor to update lr and lr2 weights.\n        beta_2: 0 < float < 1. Preferably close to 1. 1/(1 - beta_2) steps back in time that `lr`s will be optimized for, larger dataset might require more nines.\n        beta_3: 0 < float < 1. Preferably close to 1. Same as beta_2, but for `lr2`s.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n    \"\"\"\n\n    def __init__(self, params,\n            lr = 0.001, lr2 = .005, lr3 = .0005,\n            beta_1 = 0.999, beta_2 = 0.999, beta_3 = 0.9999,\n            epsilon = 1e-8, **kwargs):\n        if not 0.0 <= lr:\n            raise ValueError(\"Invalid lr: {}\".format(lr))\n        if not 0.0 <= lr2:\n            raise ValueError(\"Invalid lr2: {}\".format(lr))\n        if not 0.0 <= lr3:\n            raise ValueError(\"Invalid lr3: {}\".format(lr))\n        if not 0.0 <= epsilon:\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n        if not 0.0 <= beta_1 < 1.0:\n            raise ValueError(\"Invalid beta_1 parameter: {}\".format(betas[0]))\n        if not 0.0 <= beta_2 < 1.0:\n            raise ValueError(\"Invalid beta_2 parameter: {}\".format(betas[1]))\n        if not 0.0 <= beta_3 < 1.0:\n            raise ValueError(\"Invalid beta_3 parameter: {}\".format(betas[2]))\n        defaults = dict(lr=lr, lr2=lr2, lr3=lr3, beta_1=beta_1, beta_2=beta_2, beta_3=beta_3, epsilon=epsilon)\n        self._varn = None\n        self._is_create_slots = None\n        self._curr_var = None\n        self._lr = lr\n        self._lr2 = lr2\n        self._lr3 = lr3\n        self._beta_1 = beta_1\n        self._beta_2 = beta_2\n        self._beta_3 = beta_3\n        self._epsilon = epsilon\n        super(Adas, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(Adas, self).__setstate__(state)\n\n    @torch.no_grad()\n    def _add(self,x,y):\n        x.add_(y)\n        return x\n\n    @torch.no_grad()\n    # TODO: fix variables' names being too convoluted in _derivatives_normalizer and _get_updates_universal_impl\n    def _derivatives_normalizer(self,derivative,beta):\n        steps = self._make_variable(0,(),derivative.dtype)\n        self._add(steps,1)\n        factor = (1. - (self._beta_1 ** steps)).sqrt()\n        m = self._make_variable(0,derivative.shape,derivative.dtype)\n        moments = self._make_variable(0,derivative.shape,derivative.dtype)\n        m.mul_(self._beta_1).add_((1 - self._beta_1) * derivative * derivative)\n        np_t = derivative * factor / (m.sqrt() + self._epsilon)\n        #the third returned value should be called when the moments is finally unused, so it's updated\n        return (moments,np_t,lambda: moments.mul_(beta).add_((1 - beta) * np_t))\n\n    def _make_variable(self,value,shape,dtype):\n        self._varn += 1\n        name = 'unnamed_variable' + str(self._varn)\n        if self._is_create_slots:\n            self.state[self._curr_var][name] = torch.full(size=shape,fill_value=value,dtype=dtype,device=self._curr_var.device)\n        return self.state[self._curr_var][name]\n\n    @torch.no_grad()\n    def _get_updates_universal_impl(self, grad, param):\n        lr = self._make_variable(value = self._lr,shape=param.shape[1:], dtype=param.dtype)\n        moment, deriv, f = self._derivatives_normalizer(grad,self._beta_3)\n        param.add_( - torch.unsqueeze(lr,0) * deriv)\n        lr_deriv = torch.sum(moment * grad,0)\n        f()\n        master_lr = self._make_variable(self._lr2,(),dtype=torch.float32)\n        m2,d2, f = self._derivatives_normalizer(lr_deriv,self._beta_2)\n        self._add(lr,master_lr * lr * d2)\n        master_lr_deriv2 = torch.sum(m2 * lr_deriv)\n        f()\n        m3,d3,f = self._derivatives_normalizer(master_lr_deriv2,0.)\n        self._add(master_lr,self._lr3 * master_lr * d3)\n        f()\n\n    @torch.no_grad()\n    def _get_updates_universal(self, param, grad, is_create_slots):\n        self._curr_var = param\n        self._is_create_slots = is_create_slots\n        self._varn = 0\n        return self._get_updates_universal_impl(grad,self._curr_var.data)\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        \"\"\"Performs a single optimization step.\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = None\n        if closure is not None:\n            with torch.enable_grad():\n                loss = closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                if grad.is_sparse:\n                    raise RuntimeError('Adas does not support sparse gradients')\n                self._get_updates_universal(p,grad,len(self.state[p]) == 0)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_birds = 24\n\nif torch.cuda.is_available():\n    device=torch.device('cuda:0')\nelse:\n    device=torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\nlearning_rate = 2e-4\nepochs = 20\nloss_fn = nn.CrossEntropyLoss()\n\ndef train(model, loss_fn, train_loader, valid_loader, epochs, optimizer):\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    train_losses = []\n    valid_losses = []\n    \n    for epoch in tqdm(range(1,epochs+1)):\n        model.train()\n        batch_losses=[]\n        for i, data in enumerate(train_loader):\n            x, y = data\n            optimizer.zero_grad()\n            x = x.to(device, dtype=torch.float32)\n            y = y.to(device, dtype=torch.long)\n            y_hat = model(x)\n            loss = loss_fn(y_hat, y)\n            loss.backward()\n            batch_losses.append(loss.item())\n            optimizer.step()\n            \n        train_losses.append(batch_losses)\n        print(f'Epoch - {epoch} Train-Loss : {np.mean(train_losses[-1])}')\n        model.eval()\n        batch_losses=[]\n        trace_y = []\n        trace_yhat = []\n        \n        for i, data in enumerate(valid_loader):\n            x, y = data\n            x = x.to(device, dtype=torch.float32)\n            y = y.to(device, dtype=torch.long)\n            y_hat = model(x)\n            loss = loss_fn(y_hat, y)\n            trace_y.append(y.cpu().detach().numpy())\n            trace_yhat.append(y_hat.cpu().detach().numpy())      \n            batch_losses.append(loss.item())\n        valid_losses.append(batch_losses)\n        trace_y = np.concatenate(trace_y)\n        trace_yhat = np.concatenate(trace_yhat)\n        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n        print(f'Epoch - {epoch} Valid-Loss : {np.mean(valid_losses[-1])} Valid-Accuracy : {accuracy}')\n        # deep copy the model\n        if accuracy > best_acc:\n            best_acc = accuracy\n            best_model_wts = copy.deepcopy(model.state_dict())\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generating Mel spectrograms for training from true positive data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import csv\nimport random\n# from specAugment import spec_augment_tensorflow\n\nfft = 2048\nhop = 512\n# Less rounding errors this way\nsr = 48000\nlength = 10 * sr\n\nwith open('/kaggle/input/rfcx-species-audio-detection/train_tp.csv') as f:\n    reader = csv.reader(f)\n    next(reader, None)\n    data = list(reader)\n\n# Check minimum/maximum frequencies for bird calls\n# Not neccesary, but there are usually plenty of noise in low frequencies, and removing it helps\nfmin = 24000\nfmax = 0\n\n# Skip header row (recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max) and start from 1 instead of 0\nfor i in range(0, len(data)):\n    if fmin > float(data[i][4]):\n        fmin = float(data[i][4])\n    if fmax < float(data[i][6]):\n        fmax = float(data[i][6])\n        \n# Get some safety margin\nfmin = int(fmin * 0.9)\nfmax = int(fmax * 1.1)\nprint('Minimum frequency: ' + str(fmin) + ', maximum frequency: ' + str(fmax))\n\n\nlabel_list = []\ndata_list = []\naudio_data = {}\nfor i in range(0, len(data)):\n    recording_id = data[i][0]\n    species_id = data[i][1]\n    data_list.append(recording_id)\n    label_list.append(species_id)\n    audio_data[recording_id] = {}\n\n    # All sound files are 48000 bitrate, no need to slowly resample\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/train/' + recording_id + '.flac', sr=None)\n    t_min = float(data[i][3]) * sr\n    t_max = float(data[i][5]) * sr\n    # Positioning sound slice\n    center = np.round((t_min + t_max) / 2)\n    beginning = center - length / 2\n    if beginning < 0:\n        beginning = 0\n    ending = beginning + length\n    if ending > len(wav):\n        ending = len(wav)\n        beginning = ending - length\n    slice = wav[int(beginning):int(ending)]\n    \n    spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)\n    spec_db=librosa.power_to_db(spec,top_db=80)\n    \n    img = spec_to_image(spec_db)\n    mel_spec = np.stack((img, img, img))\n    \n    audio_data[recording_id][\"original\"] = mel_spec\n    audio_data[recording_id][\"augmentation\"] = []\n    augmentation_functions = [\n        addNoisy, contrast_stretching,\n        randomGaussian, grayScale,\n        randomGamma, vertical_flip,\n        horizontal_flip\n    ]\n    \n    for fun in augmentation_functions:\n        mel_spec = fun(img)\n        audio_data[recording_id][\"augmentation\"].append(mel_spec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nnfold = 5\nskf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=32)\n\nfor fold_id, (train_index, val_index) in enumerate(skf.split(data_list, label_list)):\n    print(\"Fold\", fold_id)\n    X_train = np.take(data_list, train_index)\n    y_train = np.take(label_list, train_index)\n    X_val = np.take(data_list, val_index)\n    y_val = np.take(label_list, val_index)\n    train_data = AudioData(X_train, y_train, \"train\")\n    valid_data = AudioData(X_val, y_val, \"valid\")\n    train_loader = DataLoader(train_data, batch_size=16, shuffle=True)\n    valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)\n    resnet_model = resnet50(pretrained=True)\n    num_ftrs = resnet_model.fc.in_features\n    resnet_model.fc = nn.Linear(num_ftrs, num_birds)\n    resnet_model = resnet_model.to(device)\n    optimizer = Adas(resnet_model.parameters(), lr=learning_rate)\n    resnet_model = train(resnet_model, loss_fn, train_loader, valid_loader, epochs, optimizer)\n    torch.save(resnet_model.state_dict(), \"model\"+str(fold_id)+\".pt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"members = []\nfor i in range(nfold):\n    model = resnet50(pretrained=True)\n    num_ftrs = model.fc.in_features\n    model.fc = nn.Linear(num_ftrs, num_birds)\n    model = model.to(device)\n    model.load_state_dict(torch.load('/kaggle/working/model'+str(i)+'.pt'))\n    model.eval()\n    members.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Already defined above; for reference\n\n# fft = 2048\n# hop = 512\n# sr = 48000\n# length = 10 * sr\n\ndef load_test_file(f):\n    wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/test/' + f, sr=None)\n\n    # Split for enough segments to not miss anything\n    segments = len(wav) / length\n    segments = int(np.ceil(segments))\n    \n    mel_array = []\n    \n    for i in range(0, segments):\n        # Last segment going from the end\n        if (i + 1) * length > len(wav):\n            slice = wav[len(wav) - length:len(wav)]\n        else:\n            slice = wav[i * length:(i + 1) * length]\n        \n        # Same mel spectrogram as before\n        spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)\n        spec_db=librosa.power_to_db(spec,top_db=80)\n\n        img = spec_to_image(spec_db)\n        mel_spec = np.stack((img, img, img))\n        mel_array.append(mel_spec)\n    \n    return mel_array","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submitting predictions with best model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\n# Scoring does not like many files:(\nif save_to_disk == 0:\n    for f in os.listdir('/kaggle/working/'):\n        os.remove('/kaggle/working/' + f)\n\n# Prediction loop\nprint('Starting prediction loop')\nwith open('submission.csv', 'w', newline='') as csvfile:\n    submission_writer = csv.writer(csvfile, delimiter=',')\n    submission_writer.writerow(['recording_id','s0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11',\n                               's12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23'])\n    \n    test_files = os.listdir('/kaggle/input/rfcx-species-audio-detection/test/')\n    print(len(test_files))\n    \n    # Every test file is split on several chunks and prediction is made for each chunk\n    for i in range(0, len(test_files)):\n        data = load_test_file(test_files[i])\n        data = torch.tensor(data)\n        data = data.float()\n        if torch.cuda.is_available():\n            data = data.cuda()\n\n        output_list = []\n        for m in members:\n            output = m(data)\n            maxed_output = torch.max(output, dim=0)[0]\n            maxed_output = maxed_output.cpu().detach()\n            output_list.append(maxed_output)\n        avg_maxed_output = torch.mean(torch.stack(output_list), dim=0)\n        \n        file_id = str.split(test_files[i], '.')[0]\n        write_array = [file_id]\n        \n        for out in avg_maxed_output:\n            write_array.append(out.item())\n    \n        submission_writer.writerow(write_array)\n        \n        \n        if i % 100 == 0 and i > 0:\n            print('Predicted for ' + str(i) + ' of ' + str(len(test_files) + 1) + ' files')\n\nprint('Submission generated')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### References\nhttps://www.kaggle.com/fffrrt/all-in-one-rfcx-baseline-for-beginners\n\nhttps://www.kaggle.com/tomahim/image-manipulation-augmentation-with-skimage\n\nhttps://www.kaggle.com/safavieh/image-augmentation-using-skimage\n\nhttps://medium.com/@hasithsura/audio-classification-d37a82d6715"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}