{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import additional packages"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# https://github.com/rwightman/gen-efficientnet-pytorch\n!pip install ../input/geffnet-pack/gen-efficientnet-pytorch-master/ > /dev/null\n# https://github.com/zhanghang1989/ResNeSt\n!pip install ../input/resnest-git/ResNeSt-master/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\n\nfrom glob import glob\nfrom os.path import join as pjoin\nfrom tqdm.notebook import tqdm\nfrom typing import Mapping, Any, List, Tuple, Optional, Union\nfrom pathlib import Path\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport geffnet\n\nfrom resnest.torch import resnest50_fast_1s1x64d\nfrom torchvision.models.resnet import ResNet, Bottleneck","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Config Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample rate of audio\nTARGET_SR=32_000\n# Whether to normalize audio by max(np.abs())\nPP_NORMALIZE=True\n# Minimum length of audio to predict\nMIN_SEC = 1\n# Computational device\nDEVICE = 'cuda'\n# Inference batch size\nBATCH_SIZE = 64\n# Thresholding value\nSIGMOID_THRESH = 0.3\n# Maximum amount of birds in raw. If None - no limit\nMAX_BIRDS = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bird code dict \n\nBIRD_CODE = {\n 'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n 'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n 'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n 'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n 'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n 'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n 'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n 'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n 'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n 'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n 'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n 'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n 'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n 'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n 'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n 'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n 'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n 'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n 'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n 'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n 'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n 'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n 'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n 'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n 'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n 'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n 'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n 'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n 'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n 'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n 'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n 'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n 'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n 'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n 'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n 'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n 'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n 'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n 'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n 'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n 'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n 'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n 'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n 'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n 'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n 'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n 'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n 'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n 'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n 'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n 'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n 'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n 'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nCODE2BIRD = {v:k for k,v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Audio Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ReadAudio(object):\n\n def __init__(\n self\n ):\n pass\n\n def __call__(self, filename: str) -> Tuple[np.ndarray, int]:\n au, sr = librosa.load(filename, sr=None, dtype=np.float64)\n return au, sr\n \nclass PreprocessWaveform(object):\n\n def __init__(\n self,\n target_sr: int,\n normalize: bool = True,\n ):\n self.target_sr=target_sr\n self.normalize=normalize\n\n def __call__(self, wave: np.ndarray, sr: int) -> np.ndarray:\n au = librosa.resample(wave, sr, self.target_sr)\n if self.normalize:\n au = librosa.util.normalize(au)\n return au","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Main Pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"class InfernceDataPipelineFileName2Au(object):\n \n def __init__(\n self,\n target_sr: bool,\n pp_normalize: bool,\n ):\n self.au_reader = ReadAudio()\n self.au_pp = PreprocessWaveform(\n target_sr=target_sr,\n normalize=pp_normalize\n )\n \n def __call__(\n self, \n au_path: str\n ): \n readed_au, readed_sr = self.au_reader(au_path)\n au = self.au_pp(readed_au, readed_sr)\n pp_sr = self.au_pp.target_sr\n\n result = {\n 'au':au,\n 'sr':pp_sr,\n }\n \n return result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Torch Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Test2StepDataset(torch.utils.data.Dataset):\n def __init__(\n self, \n df: pd.DataFrame, \n root_path: str,\n data_pipeline_austep: object,\n extension: str = '.mp3',\n duration: int = 5\n ):\n self.df = df\n self.data_pipeline_austep = data_pipeline_austep\n self.root_path = root_path\n self.extension = extension\n self.duration = duration\n \n self.dumped_features = None\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, idx: int):\n # Extract data from dataframe\n sample = self.df.loc[idx, :]\n site = sample.site\n row_id = sample.row_id\n name = sample.audio_id\n \n # We have to load new audio if only\n # 1. We get new filename\n # 2. We just started and we do not have loaded audio\n if self.dumped_features is None or self.dumped_features['name'] != name:\n self.dumped_features = {\n 'name': name,\n 'features': self.data_pipeline_austep(pjoin(self.root_path, name + self.extension))\n }\n print(f'{name} feature loaded')\n \n if site == \"site_3\":\n n_secs = len(self.dumped_features['features']['au']) / self.dumped_features['features']['sr']\n sample_rate = self.dumped_features['features']['sr']\n \n start_sec = 0\n specs = []\n names = []\n sites = []\n row_ids = []\n \n while n_secs >= start_sec:\n \n y_batch = self.dumped_features['features']['au']\n # Get duration num of seconds\n y_batch = y_batch[start_sec*sample_rate:(start_sec+self.duration)*sample_rate]\n y_batch = y_batch.astype(np.float32)\n \n # Do not process audio less then one sec\n if len(y_batch) < MIN_SEC * sample_rate:\n break\n \n # Padd if we do not have 5 secs in segment\n if len(y_batch) < sample_rate * self.duration:\n y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)\n y_pad[:len(y_batch)] = y_batch\n y_batch = y_pad\n \n y_batch = torch.from_numpy(y_batch)\n \n start_sec += self.duration\n \n specs.append(y_batch)\n names.append(name)\n sites.append(sites)\n row_ids.append(row_id)\n \n return specs, names, sites, row_ids\n else:\n end_seconds = int(sample.seconds)\n start_seconds = int(end_seconds - self.duration)\n \n sample_rate = self.dumped_features['features']['sr']\n \n y_batch = self.dumped_features['features']['au']\n y_batch = y_batch[start_seconds*sample_rate:end_seconds*sample_rate]\n y_batch = y_batch.astype(np.float32)\n \n if len(y_batch) < sample_rate * self.duration:\n y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)\n y_pad[:len(y_batch)] = y_batch\n y_batch = y_pad\n \n y_batch = torch.from_numpy(y_batch)\n\n return y_batch, name, site, row_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST = Path(\"../input/birdsong-recognition/test_audio\").exists()\n\nif TEST:\n DATA_DIR = Path(\"../input/birdsong-recognition/\")\nelse:\n # dataset created by @shonenkov, thanks!\n DATA_DIR = Path(\"../input/birdcall-check/\")\n \n\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\ntest_root = DATA_DIR / \"test_audio\"\n\n\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialize Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = Test2StepDataset(\n df=test_df,\n root_path=test_root,\n data_pipeline_austep=InfernceDataPipelineFileName2Au(\n target_sr=TARGET_SR,\n pp_normalize=PP_NORMALIZE\n ),\n extension='.mp3',\n duration=5\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# torchaudio utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_deltas(\n specgram: torch.Tensor,\n win_length: int = 5,\n mode: str = \"replicate\"\n) -> torch.Tensor:\n r\"\"\"Compute delta coefficients of a tensor, usually a spectrogram:\n\n .. math::\n d_t = \\frac{\\sum_{n=1}^{\\text{N}} n (c_{t+n} - c_{t-n})}{2 \\sum_{n=1}^{\\text{N}} n^2}\n\n where :math:`d_t` is the deltas at time :math:`t`,\n :math:`c_t` is the spectrogram coeffcients at time :math:`t`,\n :math:`N` is ``(win_length-1)//2``.\n\n Args:\n specgram (Tensor): Tensor of audio of dimension (..., freq, time)\n win_length (int, optional): The window length used for computing delta (Default: ``5``)\n mode (str, optional): Mode parameter passed to padding (Default: ``\"replicate\"``)\n\n Returns:\n Tensor: Tensor of deltas of dimension (..., freq, time)\n\n Example\n >>> specgram = torch.randn(1, 40, 1000)\n >>> delta = compute_deltas(specgram)\n >>> delta2 = compute_deltas(delta)\n \"\"\"\n device = specgram.device\n dtype = specgram.dtype\n\n # pack batch\n shape = specgram.size()\n specgram = specgram.reshape(1, -1, shape[-1])\n\n assert win_length >= 3\n\n n = (win_length - 1) // 2\n\n # twice sum of integer squared\n denom = n * (n + 1) * (2 * n + 1) / 3\n\n specgram = torch.nn.functional.pad(specgram, (n, n), mode=mode)\n\n kernel = torch.arange(-n, n + 1, 1, device=device, dtype=dtype).repeat(specgram.shape[1], 1, 1)\n\n output = torch.nn.functional.conv1d(specgram, kernel, groups=specgram.shape[1]) / denom\n\n # unpack batch\n output = output.reshape(shape)\n\n return output\n\ndef make_delta(\n input_tensor: torch.Tensor\n):\n input_tensor = input_tensor.transpose(3,2)\n input_tensor = compute_deltas(input_tensor)\n input_tensor = input_tensor.transpose(3,2)\n return input_tensor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DFTBase(nn.Module):\n def __init__(self):\n \"\"\"Base class for DFT and IDFT matrix\"\"\"\n super(DFTBase, self).__init__()\n\n def dft_matrix(self, n):\n (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n omega = np.exp(-2 * np.pi * 1j / n)\n W = np.power(omega, x * y)\n return W\n\n def idft_matrix(self, n):\n (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n omega = np.exp(2 * np.pi * 1j / n)\n W = np.power(omega, x * y)\n return W\n \n \nclass STFT(DFTBase):\n def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n \"\"\"Implementation of STFT with Conv1d. The function has the same output \n of librosa.core.stft\n \"\"\"\n super(STFT, self).__init__()\n\n assert pad_mode in ['constant', 'reflect']\n\n self.n_fft = n_fft\n self.center = center\n self.pad_mode = pad_mode\n\n # By default, use the entire frame\n if win_length is None:\n win_length = n_fft\n\n # Set the default hop, if it's not already specified\n if hop_length is None:\n hop_length = int(win_length // 4)\n\n fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n # Pad the window out to n_fft size\n fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n # DFT & IDFT matrix\n self.W = self.dft_matrix(n_fft)\n\n out_channels = n_fft // 2 + 1\n\n self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n groups=1, bias=False)\n\n self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n groups=1, bias=False)\n\n self.conv_real.weight.data = torch.Tensor(\n np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n # (n_fft // 2 + 1, 1, n_fft)\n\n self.conv_imag.weight.data = torch.Tensor(\n np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n # (n_fft // 2 + 1, 1, n_fft)\n\n if freeze_parameters:\n for param in self.parameters():\n param.requires_grad = False\n\n def forward(self, input):\n \"\"\"input: (batch_size, data_length)\n Returns:\n real: (batch_size, n_fft // 2 + 1, time_steps)\n imag: (batch_size, n_fft // 2 + 1, time_steps)\n \"\"\"\n\n x = input[:, None, :] # (batch_size, channels_num, data_length)\n\n if self.center:\n x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n real = self.conv_real(x)\n imag = self.conv_imag(x)\n # (batch_size, n_fft // 2 + 1, time_steps)\n\n real = real[:, None, :, :].transpose(2, 3)\n imag = imag[:, None, :, :].transpose(2, 3)\n # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n return real, imag\n \n \nclass Spectrogram(nn.Module):\n def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n window='hann', center=True, pad_mode='reflect', power=2.0, \n freeze_parameters=True):\n \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n Conv1d. The function has the same output of librosa.core.stft\n \"\"\"\n super(Spectrogram, self).__init__()\n\n self.power = power\n\n self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n win_length=win_length, window=window, center=center, \n pad_mode=pad_mode, freeze_parameters=True)\n\n def forward(self, input):\n \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n Returns:\n spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n \"\"\"\n\n (real, imag) = self.stft.forward(input)\n # (batch_size, n_fft // 2 + 1, time_steps)\n\n spectrogram = real ** 2 + imag ** 2\n\n if self.power == 2.0:\n pass\n else:\n spectrogram = spectrogram ** (self.power / 2.0)\n\n return spectrogram\n\n \nclass LogmelFilterBank(nn.Module):\n def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n the pytorch implementation of as librosa.filters.mel \n \"\"\"\n super(LogmelFilterBank, self).__init__()\n\n self.is_log = is_log\n self.ref = ref\n self.amin = amin\n self.top_db = top_db\n\n self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n fmin=fmin, fmax=fmax).T\n # (n_fft // 2 + 1, mel_bins)\n\n self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n if freeze_parameters:\n for param in self.parameters():\n param.requires_grad = False\n\n def forward(self, input):\n \"\"\"input: (batch_size, channels, time_steps)\n \n Output: (batch_size, time_steps, mel_bins)\n \"\"\"\n\n # Mel spectrogram\n mel_spectrogram = torch.matmul(input, self.melW)\n\n # Logmel spectrogram\n if self.is_log:\n output = self.power_to_db(mel_spectrogram)\n else:\n output = mel_spectrogram\n\n return output\n\n\n def power_to_db(self, input):\n \"\"\"Power to db, this function is the pytorch implementation of \n librosa.core.power_to_lb\n \"\"\"\n ref_value = self.ref\n log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n if self.top_db is not None:\n if self.top_db < 0:\n raise ValueError('top_db must be non-negative')\n for i in range(log_spec.shape[0]):\n log_spec[i] = torch.clamp(log_spec[i], min=log_spec[i].max().item() - self.top_db, max=np.inf)\n\n return log_spec\n\nclass DropStripes(nn.Module):\n def __init__(self, dim, drop_width, stripes_num):\n \"\"\"Drop stripes. \n Args:\n dim: int, dimension along which to drop\n drop_width: int, maximum width of stripes to drop\n stripes_num: int, how many stripes to drop\n \"\"\"\n super(DropStripes, self).__init__()\n\n assert dim in [2, 3] # dim 2: time; dim 3: frequency\n\n self.dim = dim\n self.drop_width = drop_width\n self.stripes_num = stripes_num\n\n def forward(self, input):\n \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n assert input.ndimension() == 4\n\n if self.training is False:\n return input\n\n else:\n batch_size = input.shape[0]\n total_width = input.shape[self.dim]\n\n for n in range(batch_size):\n self.transform_slice(input[n], total_width)\n\n return input\n\n\n def transform_slice(self, e, total_width):\n \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n for _ in range(self.stripes_num):\n distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n if self.dim == 2:\n e[:, bgn : bgn + distance, :] = 0\n elif self.dim == 3:\n e[:, :, bgn : bgn + distance] = 0\n\nclass SpecAugmentation(nn.Module):\n def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n freq_stripes_num):\n \"\"\"Spec augmetation. \n [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n Args:\n time_drop_width: int\n time_stripes_num: int\n freq_drop_width: int\n freq_stripes_num: int\n \"\"\"\n\n super(SpecAugmentation, self).__init__()\n\n self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n stripes_num=time_stripes_num)\n\n self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n stripes_num=freq_stripes_num)\n\n def forward(self, input):\n x = self.time_dropper(input)\n x = self.freq_dropper(x)\n return x\n \nclass Loudness(nn.Module):\n def __init__(self, sr, n_fft, min_db, device):\n super().__init__()\n self.min_db = min_db\n freqs = librosa.fft_frequencies(\n sr=sr, n_fft=n_fft\n )\n self.a_weighting = torch.nn.Parameter(\n data=torch.from_numpy(librosa.A_weighting(freqs + 1e-10)),\n requires_grad=False\n )\n self.to(device)\n\n def forward(self, spec):\n power_db = torch.log10(spec**0.5 + 1e-10)\n\n loudness = power_db + self.a_weighting\n\n #loudness -= 10 * torch.log10(spec)\n loudness -= 20.7\n\n loudness = torch.clamp(loudness, min=-self.min_db)\n\n # Average over frequency bins.\n loudness = torch.mean(loudness, axis=-1).float()\n return loudness","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resnext initialize functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _resnext(block, layers, pretrained, progress, **kwargs):\n return ResNet(block, layers, **kwargs)\n\ndef resnext50_32x4d_swsl(progress=True, **kwargs):\n \"\"\"Constructs a semi-weakly supervised ResNeXt-50 32x4 model pre-trained on 1B weakly supervised \n image dataset and finetuned on ImageNet. \n `\"Billion-scale Semi-Supervised Learning for Image Classification\" `_\n Args:\n progress (bool): If True, displays a progress bar of the download to stderr.\n \"\"\"\n kwargs['groups'] = 32\n kwargs['width_per_group'] = 4\n return _resnext(Bottleneck, [3, 4, 6, 3], True, progress, **kwargs)\n\ndef resnext101_32x4d_swsl(progress=True, **kwargs):\n \"\"\"Constructs a semi-weakly supervised ResNeXt-101 32x4 model pre-trained on 1B weakly supervised \n image dataset and finetuned on ImageNet. \n `\"Billion-scale Semi-Supervised Learning for Image Classification\" `_\n Args:\n progress (bool): If True, displays a progress bar of the download to stderr.\n \"\"\"\n kwargs['groups'] = 32\n kwargs['width_per_group'] = 4\n return _resnext(Bottleneck, [3, 4, 23, 3], True, progress, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BIG BIG Model class"},{"metadata":{"trusted":true},"cell_type":"code","source":"EFFNETB6_EMB_DIM = 2304\nEFFNETB5_EMB_DIM = 2048\nEFFNETB4_EMB_DIM = 1792\nEFFNETB3_EMB_DIM = 1536\nEFFNETB1_EMB_DIM = 1280\nRESNEST50_FAST_EMB_DIM = 2048\nRESNEXT50_EMB_DIM = 2048\nRESNEXT101_EMB_DIM = 2048\n\nEPS = 1e-6\n\nclass SpectralEffnet(nn.Module):\n def __init__(\n self, \n # Spectral Config ==Start==\n sample_rate: int, \n window_size: int, \n hop_size: int,\n mel_bins: int, \n fmin: int, \n fmax: int, \n top_db: float,\n # Spectral Config ==End==\n # Model Config ==Start==\n classes_num: int,\n encoder_type: str, \n hidden_dims: int, \n first_dropout_rate: float,\n second_dropout_rate: float, \n use_pretrained_encoder: bool, \n classifier_type: str,\n # Model Config ==End==\n # Feature Extraction Config ==Start==\n use_spectral_cutout: bool, \n spec_aggreagation: str,\n use_loudness: bool, \n use_spectral_centroid: bool,\n # Feature Extraction Config ==End==\n # Other Config ==Start==\n device: str, \n use_sigmoid: bool = True, \n # Other Config ==End==\n ):\n super().__init__()\n\n window = 'hann'\n center = True\n pad_mode = 'reflect'\n ref = 1.0\n amin = 1e-10\n self.interpolate_ratio = 32 # Downsampled ratio\n\n # Spectrogram extractor\n self.spectrogram_extractor = Spectrogram(\n n_fft=window_size,\n hop_length=hop_size,\n win_length=window_size,\n window=window,\n center=center,\n pad_mode=pad_mode,\n freeze_parameters=True)\n\n # Logmel feature extractor\n self.logmel_extractor = LogmelFilterBank(\n sr=sample_rate,\n n_fft=window_size,\n n_mels=mel_bins,\n fmin=fmin,\n fmax=fmax,\n ref=ref,\n amin=amin,\n top_db=top_db,\n freeze_parameters=True)\n\n # Spec augmenter\n self.spec_augmenter = SpecAugmentation(\n time_drop_width=64,\n time_stripes_num=2,\n freq_drop_width=8,\n freq_stripes_num=2)\n\n if use_loudness:\n self.loudness_bn = nn.BatchNorm1d(1)\n self.loudness_extractor = Loudness(\n sr=sample_rate, \n n_fft=window_size, \n min_db=120, \n device=device\n )\n if use_spectral_centroid:\n self.spectral_centroid_bn = nn.BatchNorm1d(1)\n\n self.bn0 = nn.BatchNorm2d(mel_bins)\n\n if spec_aggreagation in ['conv1', 'repeat3', 'deltas', 'conv3', 'time_freq_encoding']:\n self.spec_aggreagation = spec_aggreagation\n else:\n raise ValueError('Invalid spec_aggreagation')\n\n if encoder_type == 'effnet1': \n self.encoder = geffnet.tf_efficientnet_b1_ns(pretrained=use_pretrained_encoder)\n self.encoder.classifier = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv_stem = Conv2dSame(\n in_channels=1, \n out_channels=self.encoder.conv_stem.out_channels,\n kernel_size=self.encoder.conv_stem.kernel_size,\n stride=self.encoder.conv_stem.stride,\n bias=self.encoder.conv_stem.bias\n )\n nn_embed_size = EFFNETB1_EMB_DIM\n elif encoder_type == 'effnet3':\n self.encoder = geffnet.tf_efficientnet_b3_ns(pretrained=use_pretrained_encoder)\n self.encoder.classifier = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv_stem = Conv2dSame(\n in_channels=1, \n out_channels=self.encoder.conv_stem.out_channels,\n kernel_size=self.encoder.conv_stem.kernel_size,\n stride=self.encoder.conv_stem.stride,\n bias=self.encoder.conv_stem.bias\n )\n nn_embed_size = EFFNETB3_EMB_DIM\n elif encoder_type == 'effnet4':\n self.encoder = geffnet.tf_efficientnet_b4_ns(pretrained=use_pretrained_encoder)\n self.encoder.classifier = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv_stem = Conv2dSame(\n in_channels=1, \n out_channels=self.encoder.conv_stem.out_channels,\n kernel_size=self.encoder.conv_stem.kernel_size,\n stride=self.encoder.conv_stem.stride,\n bias=self.encoder.conv_stem.bias\n )\n nn_embed_size = EFFNETB4_EMB_DIM\n elif encoder_type == 'effnet5':\n self.encoder = geffnet.tf_efficientnet_b5_ns(pretrained=use_pretrained_encoder)\n self.encoder.classifier = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv_stem = Conv2dSame(\n in_channels=1, \n out_channels=self.encoder.conv_stem.out_channels,\n kernel_size=self.encoder.conv_stem.kernel_size,\n stride=self.encoder.conv_stem.stride,\n bias=self.encoder.conv_stem.bias\n )\n nn_embed_size = EFFNETB5_EMB_DIM\n elif encoder_type == 'effnet6':\n self.encoder = geffnet.tf_efficientnet_b6_ns(pretrained=use_pretrained_encoder)\n self.encoder.classifier = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv_stem = Conv2dSame(\n in_channels=1, \n out_channels=self.encoder.conv_stem.out_channels,\n kernel_size=self.encoder.conv_stem.kernel_size,\n stride=self.encoder.conv_stem.stride,\n bias=self.encoder.conv_stem.bias\n )\n nn_embed_size = EFFNETB6_EMB_DIM\n elif encoder_type == 'resnest50_fast_1s1x64d':\n if self.spec_aggreagation == 'conv1':\n raise ValueError('Invalid spec_aggreagation for this encoder')\n self.encoder = resnest50_fast_1s1x64d(pretrained=use_pretrained_encoder)\n self.encoder.fc = nn.Identity()\n nn_embed_size = RESNEST50_FAST_EMB_DIM\n elif encoder_type == 'resnext50_32x4d_swsl':\n self.encoder = resnext50_32x4d_swsl()\n if use_pretrained_encoder:\n self.encoder.load_state_dict(\n torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_swsl').state_dict()\n )\n self.encoder.fc = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv1 = nn.Conv2d(\n in_channels=1, \n out_channels=self.encoder.conv1.out_channels,\n kernel_size=self.encoder.conv1.kernel_size,\n stride=self.encoder.conv1.stride,\n bias=self.encoder.conv1.bias\n )\n nn_embed_size = RESNEXT50_EMB_DIM\n elif encoder_type == 'resnext101_32x4d_swsl':\n self.encoder = resnext101_32x4d_swsl()\n if use_pretrained_encoder:\n self.encoder.load_state_dict(\n torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext101_32x4d_swsl').state_dict()\n )\n self.encoder.fc = nn.Identity()\n if self.spec_aggreagation == 'conv1':\n self.encoder.conv1 = nn.Conv2d(\n in_channels=1, \n out_channels=self.encoder.conv1.out_channels,\n kernel_size=self.encoder.conv1.kernel_size,\n stride=self.encoder.conv1.stride,\n bias=self.encoder.conv1.bias\n )\n nn_embed_size = RESNEXT101_EMB_DIM\n else:\n raise ValueError(f'{encoder_type} is invalid model_type')\n\n \n if classifier_type == 'relu':\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims), nn.ReLU(), nn.Dropout(p=first_dropout_rate),\n nn.Linear(hidden_dims, hidden_dims), nn.ReLU(), nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n )\n elif classifier_type == 'elu':\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n nn.Dropout(second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n )\n elif classifier_type == 'dima': # inspired by https://github.com/ex4sperans/freesound-classification\n self.classifier = nn.Sequential(\n nn.BatchNorm1d(nn_embed_size),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.BatchNorm1d(hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n )\n elif classifier_type == 'prelu':\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n )\n elif classifier_type == 'multiscale_relu':\n self.big_dropout = nn.Dropout(p=0.5)\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims), nn.ReLU(), nn.Dropout(p=first_dropout_rate),\n nn.Linear(hidden_dims, hidden_dims), nn.ReLU(), nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n )\n else:\n raise ValueError(\"Invalid classifier_type\")\n \n # Classifier type\n self.classifier_type = classifier_type\n # Augmentations\n self.use_spectral_cutout = use_spectral_cutout\n # Final activation\n self.use_sigmoid = use_sigmoid\n # Additional features\n self.use_spectral_centroid = use_spectral_centroid\n self.use_loudness = use_loudness\n # Some additional stuff\n self.encoder_type = encoder_type\n self.device = device\n self.mel_bins = mel_bins\n self.to(self.device)\n\n def _add_frequency_encoding(self, x):\n n, d, h, w = x.size()\n\n vertical = torch.linspace(-1, 1, w, device=x.device).view(1, 1, 1, -1)\n vertical = vertical.repeat(n, 1, h, 1)\n\n return vertical\n\n def _add_time_encoding(self, x):\n n, d, h, w = x.size()\n\n horizontal = torch.linspace(-1, 1, h, device=x.device).view(1, 1, -1, 1)\n horizontal = horizontal.repeat(n, 1, 1, w)\n\n return horizontal\n \n def preprocess(self, input):\n x = self.spectrogram_extractor(input) # (batch_size, 1, time_steps, freq_bins)\n\n additional_features = []\n if self.use_loudness:\n loudness = self.loudness_extractor(x)\n loudness = self.loudness_bn(loudness)\n loudness = loudness.unsqueeze(-1)\n loudness = loudness.repeat(1,1,1,self.mel_bins)\n additional_features.append(loudness)\n if self.use_spectral_centroid:\n spectral_centroid = x.mean(-1)\n spectral_centroid = self.spectral_centroid_bn(spectral_centroid)\n spectral_centroid = spectral_centroid.unsqueeze(-1)\n spectral_centroid = spectral_centroid.repeat(1,1,1,self.mel_bins)\n additional_features.append(spectral_centroid)\n\n x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins)\n\n if self.training and self.use_spectral_cutout:\n x = self.spec_augmenter(x)\n \n frames_num = x.shape[2]\n \n x = x.transpose(1, 3)\n x = self.bn0(x)\n x = x.transpose(1, 3)\n\n if len(additional_features) > 0:\n additional_features.append(x)\n x = torch.cat(additional_features, dim=1)\n\n if self.spec_aggreagation == 'repeat3':\n x = torch.cat([x,x,x], dim=1)\n elif self.spec_aggreagation == 'deltas':\n delta_1 = make_delta(x)\n delta_2 = make_delta(delta_1)\n x = torch.cat([x,delta_1,delta_2], dim=1)\n elif self.spec_aggreagation == 'time_freq_encoding':\n freq_encode = self._add_frequency_encoding(x)\n time_encode = self._add_time_encoding(x)\n x = torch.cat([x, freq_encode, time_encode], dim=1)\n elif self.spec_aggreagation in ['conv1','conv3']:\n pass\n\n return x, frames_num\n \n\n def forward(self, input):\n \"\"\"\n Input: (batch_size, data_length)\n \"\"\"\n # Output shape (batch size, channels, time, frequency)\n x, _ = self.preprocess(input)\n\n # Output shape (batch size, channels)\n x = self.encoder(x)\n\n if self.classifier_type == 'multiscale_relu':\n logits = torch.mean(torch.stack([self.classifier(self.big_dropout(x)) for _ in range(5)],dim=0),dim=0)\n else:\n logits = self.classifier(x)\n\n if self.use_sigmoid:\n return torch.sigmoid(logits)\n else:\n return logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_chkp(\n path: str,\n nn_model: nn.Module\n):\n chckp = torch.load(path, map_location='cpu')\n nn_model.load_state_dict(chckp)\n return nn_model\n\n\ndef initalize_spectral_model(\n model_config: Mapping[str, Any],\n chkp_path: str,\n m_device: str\n):\n temp_model = SpectralEffnet(\n **model_config,\n device=m_device\n )\n temp_model = load_chkp(chkp_path, temp_model)\n temp_model.eval()\n return temp_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ALL My Models"},{"metadata":{},"cell_type":"markdown","source":"All models are taken from 5 folds, so each experiment contains bunch of 5 models.\n\nEach model is created by SWA( simple averaging of weight matrices from 3 best chekpoints taken for `f1_test_median`)"},{"metadata":{"trusted":true},"cell_type":"code","source":"NEW_EFFNET3 = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet3',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"deltas\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\nNEW_EFFNET3_MD = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet3',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"deltas\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"multiscale_relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\nNEW_EFFNET4 = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet4',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"deltas\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\nNEW_EFFNET4_MD = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet4',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"deltas\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"multiscale_relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\nNEW_EFFNET5_MD = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet5',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"deltas\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"multiscale_relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\nNEW_EFFNET3_MD_TFE = {\n \"sample_rate\": TARGET_SR,\n \"window_size\": 2048,\n \"hop_size\": 256,\n \"mel_bins\": 128,\n \"fmin\": 20,\n \"fmax\": 16000,\n \"top_db\":80.0,\n \"classes_num\": len(BIRD_CODE),\n \"encoder_type\": 'effnet3',\n \"hidden_dims\": 1024,\n \"first_dropout_rate\": 0.2,\n \"second_dropout_rate\": 0.2,\n \"use_spectral_cutout\": False,\n \"spec_aggreagation\": \"time_freq_encoding\",\n \"use_pretrained_encoder\": False,\n \"use_sigmoid\": True,\n \"classifier_type\": \"multiscale_relu\",\n \"use_loudness\": False,\n \"use_spectral_centroid\": False,\n}\n\n\nmodels = [\n# effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto\n# Public score: 0.613 (thresh - 0.4) and 0.612 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET3_MD, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3_MD, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3_MD, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3_MD, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3_MD, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto \n# Public score: 0.61 (thresh - 0.4), 0.605 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET4, '../input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET4, '../input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET4, '../input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET4, '../input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET4, '../input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto\n# Public score: 0.608 (thresh - 0.4), 0.607 (thresh - 0.5) \n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0\n# Public score: 0.601 (thresh - 0.4)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005\n# Public score: 0.599 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075\n# Public score: 0.601 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10\n# Public score: 0.606 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_nocalldatax10_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto\n# Public score: 0.607 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET5_MD, '../input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET5_MD, '../input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET5_MD, '../input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET5_MD, '../input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET5_MD, '../input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto\n# Public score: 0.595 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3_MD_TFE, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3_MD_TFE, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3_MD_TFE, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3_MD_TFE, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3_MD_TFE, '../input/cornell-birds-models/effnet3_multiscalereluclas_plato_timefreqencode_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01\n# Public score: 0.589 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_ls01_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05\n# Public score: 0.598 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005\n# Public score: 0.596 (thresh - 0.5)\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_0.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_1.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_2.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_3.pt', DEVICE),\n initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01\n# Public score: 0.593 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls01_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels\n# Public score: 0.582 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto\n# Public score: 0.601 (thresh - 0.5)\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto_swa_f1_test_median_fold_1.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto_swa_f1_test_median_fold_3.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET3, '../input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_extxenocanto_swa_f1_test_median_fold_4.pt', DEVICE),\n# effnet4_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugsless_newbacks_secondarylabels_mixupsumwave05_xeno_canto\n# Public score: not commited\n# Is not used in best Blend\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugsless_newbacks_secondarylabels_mixupsumwave05_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugsless_newbacks_secondarylabels_mixupsumwave05_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),\n# initalize_spectral_model(NEW_EFFNET4_MD, '../input/cornell-birds-models/effnet4_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugsless_newbacks_secondarylabels_mixupsumwave05_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Models in Blend: {len(models)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch = []\nnames = []\nprobs = []\nfor i in tqdm(range(len(test_dataset))):\n sample_spec, _, _, sample_id = test_dataset[i]\n \n # If we have site3 we get list of samples\n # If we have site1 or site2 - only one sample\n if isinstance(sample_spec, list):\n batch += sample_spec\n else:\n batch.append(sample_spec)\n \n if isinstance(sample_id, list):\n names += sample_id\n else:\n names.append(sample_id)\n \n if len(batch) >= BATCH_SIZE or i == (len(test_dataset) - 1):\n with torch.no_grad():\n # If site3 produced very big batch we decompose it \n # in smaller ones\n if len(batch) > BATCH_SIZE + 1:\n n_steps = math.ceil(len(batch) / BATCH_SIZE)\n for step in range(n_steps):\n small_batch = batch[step*BATCH_SIZE:(step+1)*BATCH_SIZE]\n small_batch = torch.stack(small_batch).to(DEVICE).float()\n # Or models are blend with simple Mean\n batch_probs = sum(m(small_batch).detach().cpu() for m in models) / len(models)\n batch_probs = batch_probs.numpy()\n probs.append(batch_probs)\n else: \n batch = torch.stack(batch).to(DEVICE).float()\n # Or models are blend with simple Mean\n batch_probs = sum(m(batch).detach().cpu() for m in models) / len(models)\n batch_probs = batch_probs.numpy()\n probs.append(batch_probs)\n \n batch = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create final submission DF"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Concatanate batches\nprobs = np.concatenate(probs)\n\n# Create DataFrame\nresult_df = pd.DataFrame({\n 'row_id':names,\n 'birds':[probs[i] for i in range(probs.shape[0])]\n})\n\n# Aggregate predictions from site3 by Max\nresult_df = result_df.groupby('row_id')['birds'].apply(lambda x: np.stack(x).max(0)).reset_index()\n\nresult_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def probs2names(\n probs_array: np.ndarray,\n threshold: int = 0.5,\n max_birds_to_take: Optional[int] = None,\n idx2bird_mapping: Mapping[int, str] = CODE2BIRD\n):\n accepted_indices = np.where(probs_array > threshold)[0]\n \n if max_birds_to_take is not None:\n accepted_probs = probs_array[accepted_indices]\n accepted_probs_indices = np.argsort(-accepted_probs)\n accepted_indices = accepted_indices[accepted_probs_indices]\n accepted_indices = accepted_indices[:max_birds_to_take]\n \n if len(accepted_indices) == 0:\n return 'nocall'\n else:\n return ' '.join([idx2bird_mapping[idx] for idx in accepted_indices])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df['birds'] = result_df['birds'].apply(lambda x: probs2names(\n x, \n threshold=SIGMOID_THRESH, \n max_birds_to_take=MAX_BIRDS\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = test_df.merge(result_df, on='row_id', how='left')[['row_id','birds']]\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['birds'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Tips"},{"metadata":{},"cell_type":"markdown","source":"## Augmentations\n\nGreat thanks for [audiomentations package](https://github.com/iver56/audiomentations)"},{"metadata":{},"cell_type":"markdown","source":"### Gain\nTaken from https://github.com/iver56/audiomentations\n\n```python\nclass Gain(audiomentations.BasicTransform):\n \"\"\"\n Multiply the audio by a random amplitude factor to reduce or increase the volume. This\n technique can help a model become somewhat invariant to the overall gain of the input audio.\n Warning: This transform can return samples outside the [-1, 1] range, which may lead to\n clipping or wrap distortion, depending on what you do with the audio in a later stage.\n See also https://en.wikipedia.org/wiki/Clipping_(audio)#Digital_clipping\n \"\"\"\n\n def __init__(self, min_gain_in_db=-12, max_gain_in_db=12, p=0.5):\n \"\"\"\n :param p:\n \"\"\"\n super().__init__(p)\n assert min_gain_in_db <= max_gain_in_db\n self.min_gain_in_db = min_gain_in_db\n self.max_gain_in_db = max_gain_in_db\n\n def randomize_parameters(self, samples, sample_rate):\n super().randomize_parameters(samples, sample_rate)\n if self.parameters[\"should_apply\"]:\n self.parameters[\"amplitude_ratio\"] = convert_decibels_to_amplitude_ratio(\n random.uniform(self.min_gain_in_db, self.max_gain_in_db)\n )\n\n def apply(self, samples, sample_rate):\n return samples * self.parameters[\"amplitude_ratio\"]\n```"},{"metadata":{},"cell_type":"markdown","source":"### Background Noise\n\n```python\nclass SpecifiedNoise(audiomentations.BasicTransform):\n \n def __init__(\n self,\n noise_folder_path: str,\n p: int = 0.5,\n allways_apply: bool = False,\n low_alpha: float = 0.0,\n high_alpha: float = 1.0\n ):\n super().__init__(p)\n filenames = glob(pjoin(noise_folder_path, '*.wav'))\n self.noises = [librosa.load(noise_path, sr=None)[0] for noise_path in filenames]\n self.noises = [librosa.util.normalize(noise) for noise in self.noises]\n self.p = p\n self.allways_apply = allways_apply\n self.low_alpha = low_alpha\n self.high_alpha = high_alpha\n \n def apply(self, au, sr):\n if np.random.binomial(n=1, p=self.p) or self.allways_apply:\n alpha = np.random.uniform(low=self.low_alpha, high=self.high_alpha)\n noise = self.noises[np.random.randint(low=0, high=len(self.noises))]\n au = au*(1 - alpha) + noise * alpha\n \n return au\n```"},{"metadata":{},"cell_type":"markdown","source":"### LowFrequency CutOff\n\n```python\nclass LowFrequencyMask(audiomentations.BasicTransform):\n \n def __init__(\n self,\n p: int = 0.5,\n allways_apply: bool = False,\n max_cutoff: float = 5,\n min_cutoff: float = 4\n ):\n super().__init__(p)\n self.p = p\n self.allways_apply = allways_apply\n self.max_cutoff = max_cutoff\n self.min_cutoff = min_cutoff\n \n def apply(self, au, sr):\n if np.random.binomial(n=1, p=self.p) or self.allways_apply:\n cutoff_value = np.random.uniform(low=self.min_cutoff, high=self.max_cutoff)\n au = butter_lowpass_filter(au, cutoff=cutoff_value, fs=sr / 1000)\n \n return au\n```"},{"metadata":{},"cell_type":"markdown","source":"### All Pipeline\n\n`noisy_samples` you can find in [this dataset](https://www.kaggle.com/vladimirsydor/cornelli-background-noises)\n\n```python\naudiomentations.Compose([\n Gain(p=0.5),\n SpecifiedNoise('/ssd_data/birdsong_recognition/noisy_samples', low_alpha=0.5, high_alpha=0.8, p=0.1),\n audiomentations.AddBackgroundNoise('/ssd_data/birdsong_recognition/noisy_samples', min_snr_in_db=0.001, max_snr_in_db=2, p=0.75),\n LowFrequencyMask(min_cutoff=5, max_cutoff=7, p=0.75)\n])\n```"},{"metadata":{},"cell_type":"markdown","source":"## Mixup \n\nI have used OR Mixup, that was proposed by [Dmytro Danevskyi](https://www.kaggle.com/ddanevskyi) [here](https://www.kaggle.com/c/freesound-audio-tagging-2019/discussion/97926)\n\nI have used high mixup probability - 75%\n\nHere is full loss function, which includes Mixup and Label Smoothing. But Label Smoothing did not work for me in all cases"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport numpy as np\n\nclass StrongBCEwithLogitsMixUp(nn.Module):\n\n def __init__(\n self,\n n_classes: int,\n use_onehot_target: bool = True,\n label_smoothing_coef: float = 0.0,\n mixup_p: float = 0.5,\n mixup_type: str = 'wave_sum'\n ):\n super().__init__()\n self.n_classes = n_classes\n self.use_onehot_target = use_onehot_target\n self.label_smoothing_coef = label_smoothing_coef\n\n self.mixup_p = mixup_p\n\n if mixup_type not in ['wave_sum']:\n raise ValueError(\"Wrong mixup_type\")\n self.mixup_type = mixup_type\n\n self.loss_f = nn.BCEWithLogitsLoss()\n\n def _sum_mixup(self, inputs, targets):\n indices = torch.randperm(inputs.size(0))\n shuffled_inputs = inputs[indices]\n shuffled_targets = targets[indices]\n\n inputs = inputs + shuffled_inputs\n targets = targets + shuffled_targets\n\n targets = targets.clamp(min=0, max=1.0)\n for i in range(inputs.shape[0]):\n inputs[i,:] = inputs[i,:] / inputs[i,:].abs().max()\n\n return inputs, targets\n\n def forward(self, batch, model):\n\n wave = batch['waveform']\n target = batch['targets']\n if self.use_onehot_target:\n one_hot_target = torch.nn.functional.one_hot(target, self.n_classes)\n primary_target = target\n else:\n one_hot_target = target\n primary_target = batch['primary_label']\n\n if model.training and np.random.binomial(n=1,p=self.mixup_p):\n if self.mixup_type == 'wave_sum':\n wave, one_hot_target = self._sum_mixup(wave, one_hot_target)\n\n logits = model(wave)\n\n # Handle output for clipwise/framewise model\n if isinstance(logits, dict):\n logits = logits['clipwise_output']\n\n if self.label_smoothing_coef > 0 and model.training:\n one_hot_target = torch.abs(one_hot_target.float() - self.label_smoothing_coef)\n else:\n one_hot_target = one_hot_target.float()\n\n loss = self.loss_f(logits, one_hot_target)\n\n losses = {\n \"loss\":loss\n }\n outputs = {\n \"logits\":logits\n }\n inputs = {\n \"targets\":primary_target,\n \"all_targets\": target if not self.use_onehot_target else None\n }\n\n return losses, inputs, outputs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multi-Sample Dropout\n\nOriginally proposed in [this paper](https://arxiv.org/pdf/1905.09788.pdf) and used by winners of [QA competition](https://www.kaggle.com/c/google-quest-challenge/overview)\n\n```python\nself.big_dropout = nn.Dropout(p=0.5)\nself.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims), nn.ReLU(), nn.Dropout(p=first_dropout_rate),\n nn.Linear(hidden_dims, hidden_dims), nn.ReLU(), nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num)\n)\nlogits = torch.mean(torch.stack([self.classifier(self.big_dropout(x)) for _ in range(5)],dim=0),dim=0)\n```"},{"metadata":{},"cell_type":"markdown","source":"## Energy trimming\n\nI have created RMS feature with window size of 5 seconds and step size of 1 second. Then normalized it in order to use as probability distribution for sampling start second of the audio\n\n```python\ndef compute_normalized_energy(\n wave: np.array\n):\n wave = wave / np.abs(wave).max()\n return np.power(wave,2)\n\ndef compute_sampling_distribution(\n feature: np.ndarray,\n hop_size: int,\n window_size: int,\n probs_comp: str = 'softmax'\n):\n n_steps = max(math.ceil((len(feature) - window_size) / hop_size),1)\n if probs_comp == 'softmax':\n areas = [feature[hop_size*i:window_size + hop_size*i].sum() for i in range(n_steps)]\n probs = softmax(areas)\n elif probs_comp == 'uniform':\n areas = [feature[hop_size*i:window_size + hop_size*i].sum() for i in range(n_steps)]\n probs = np.array(areas) / sum(areas)\n elif probs_comp == 'rm_softmax':\n areas = [feature[hop_size*i:window_size + hop_size*i].mean() for i in range(n_steps)]\n probs = softmax(np.power(np.array(areas), 0.5))\n else:\n raise ValueError('Invalid probs_comp')\n return probs\n\nh_s = sr * 1\nw_s = sr * 5\n\nt_pobs = compute_sampling_distribution(\n feature=compute_normalized_energy(au),\n hop_size=h_s,\n window_size=w_s,\n probs_comp='uniform'\n)\n\nstart = np.random.choice(len(t_pobs),size=1, p=t_pobs)[0]\nsegment = au[start*h_s:start*h_s + w_s ]\n```"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}