{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install \"../input/timmpytorch/timm-0.2.1-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, random, time, warnings, logging\nimport librosa\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom functools import partial\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\nfrom pathlib import Path\nfrom fastprogress import progress_bar\n\n\nimport timm\nfrom timm.models.efficientnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, \\\n    tf_efficientnet_b5_ns, tf_efficientnet_b2_ns, tf_efficientnet_b6_ns, tf_efficientnet_b7_ns, tf_efficientnet_b0_ns\nfrom torch.nn.modules.dropout import Dropout\nfrom torch.nn.modules.linear import Linear\nfrom torch.nn.modules.pooling import AdaptiveAvgPool2d, AdaptiveMaxPool2d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n    \nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, data_length)\n        Returns:\n          real: (batch_size, n_fft // 2 + 1, time_steps)\n          imag: (batch_size, n_fft // 2 + 1, time_steps)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n    \n    \nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.core.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        Returns:\n          spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        \"\"\"Power to db, this function is the pytorch implementation of \n        librosa.core.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec\n    \nclass DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n \n    if hasattr(layer, 'bias'):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n    \ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.conv2 = nn.Conv2d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x\n\n\nclass ConvBlock5x5(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock5x5, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(5, 5), stride=(1, 1),\n                              padding=(2, 2), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self, n_in, n_out, activation='linear', temperature=1.):\n        super(AttBlock, self).__init__()\n        \n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)\n        self.cla = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)\n        \n        self.bn_att = nn.BatchNorm1d(n_out)\n        self.init_weights()\n        \n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n         \n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass Cnn14_16k(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        \n        super(Cnn14_16k, self).__init__() \n\n        assert sample_rate == 16000\n        assert window_size == 512\n        assert hop_size == 160\n        assert mel_bins == 64\n        assert fmin == 50\n        assert fmax == 8000\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.fc_audioset1 = nn.Linear(2048, classes_num, bias=True)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset1)\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        \n        #x = input\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        #if self.training:\n        #    x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        #if self.training and mixup_lambda is not None\n        #    x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        #embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset1(x))\n        \n        #output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return clipwise_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_params = {\n    \"resnest50d\" : {\n        \"features\" : 2048,\n        \"init_op\"  : partial(timm.models.resnest50d, pretrained=False, in_chans=1)\n    },\n    \"densenet201\" : {\n        \"features\": 1920,\n        \"init_op\": partial(timm.models.densenet201, pretrained=False)\n    },\n    \"dpn92\" : {\n        \"features\": 2688,\n        \"init_op\": partial(timm.models.dpn92, pretrained=False)\n    },\n    \"dpn131\": {\n        \"features\": 2688,\n        \"init_op\": partial(timm.models.dpn131, pretrained=False)\n    },\n    \"tf_efficientnet_b0_ns\": {\n        \"features\": 1280,\n        \"init_op\": partial(tf_efficientnet_b0_ns, pretrained=False, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b3_ns\": {\n        \"features\": 1536,\n        \"init_op\": partial(tf_efficientnet_b3_ns, pretrained=False, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b2_ns\": {\n        \"features\": 1408,\n        \"init_op\": partial(tf_efficientnet_b2_ns, pretrained=False, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b4_ns\": {\n        \"features\": 1792,\n        \"init_op\": partial(tf_efficientnet_b4_ns, pretrained=False, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b5_ns\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=False, drop_path_rate=0.2, in_chans=1)\n    }\n}\n\n\nclass BirdClassifier(nn.Module):\n    def __init__(self, encoder, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        \n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        self.dropout = Dropout(0.2)\n        self.fc = Linear(encoder_params[encoder]['features'], classes_num)\n\n    def forward(self, input, spec_aug=False, mixup_lambda=None):\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        #x = input\n        \n        if spec_aug:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n\n        x = self.encoder.forward_features(x)\n        x = self.avg_pool(x).flatten(1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        return torch.sigmoid(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ncnn14_config = {\n    \"sample_rate\": 16000,\n    \"window_size\": 512,\n    \"hop_size\": 160,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 8000,\n    \"classes_num\": 264\n}\n\ncnn14_stage3 = Cnn14_16k(**cnn14_config)\ncnn14_stage3.load_state_dict(torch.load(\"../input/stage3-final-models/cnn14-fold-0.bin\", map_location=device))\ncnn14_stage3 = cnn14_stage3.to(device)\ncnn14_stage3.eval()\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nresnest50d_config = {\n    \"encoder\" : \"resnest50d\",\n    \"sample_rate\": 16000,\n    \"window_size\": 512,\n    \"hop_size\": 160,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 8000,\n    \"classes_num\": 264\n}\n\nresnest50d_stage3 = BirdClassifier(**resnest50d_config)\nresnest50d_stage3.load_state_dict(torch.load(\"../input/stage3-final-models/resnest50d-fold-1.bin\", map_location=device))\nresnest50d_stage3 = resnest50d_stage3.to(device)\nresnest50d_stage3.eval()\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\neff3_config = {\n    \"encoder\" : \"tf_efficientnet_b3_ns\",\n    \"sample_rate\": 16000,\n    \"window_size\": 512,\n    \"hop_size\": 160,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 8000,\n    \"classes_num\": 264\n}\n\n\neff3_stage3 = BirdClassifier(**eff3_config)\neff3_stage3.load_state_dict(torch.load(\"../input/stage3-final-models/eff3-fold-1.bin\", map_location=device))\neff3_stage3 =  eff3_stage3.to(device)\neff3_stage3.eval()\n\neff3_stage3_f1 = BirdClassifier(**eff3_config)\neff3_stage3_f1.load_state_dict(torch.load(\"../input/stage3-final-models/eff3-fold-3.bin\", map_location=device))\neff3_stage3_f1 =  eff3_stage3_f1.to(device)\neff3_stage3_f1.eval()\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\neff4_config = {\n    \"encoder\" : \"tf_efficientnet_b4_ns\",\n    \"sample_rate\": 16000,\n    \"window_size\": 512,\n    \"hop_size\": 160,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 8000,\n    \"classes_num\": 264\n}\n\neff4_stage3 = BirdClassifier(**eff4_config)\neff4_stage3.load_state_dict(torch.load(\"../input/stage3-final-models/eff4-fold-1.bin\", map_location=device))\neff4_stage3 =  eff4_stage3.to(device)\neff4_stage3.eval()\n\n\neff4_stage3_f1 = BirdClassifier(**eff4_config)\neff4_stage3_f1.load_state_dict(torch.load(\"../input/stage3-final-models/eff4-fold-4.bin\", map_location=device))\neff4_stage3_f1 =  eff4_stage3_f1.to(device)\neff4_stage3_f1.eval()\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\neff5_config = {\n    \"encoder\" : \"tf_efficientnet_b5_ns\",\n    \"sample_rate\": 16000,\n    \"window_size\": 512,\n    \"hop_size\": 160,\n    \"mel_bins\": 120,\n    \"fmin\": 50,\n    \"fmax\": 8000,\n    \"classes_num\": 264\n}\n\neff5_stage3 = BirdClassifier(**eff5_config)\neff5_stage3.load_state_dict(torch.load(\"../input/stage3-final-models/eff5-fold-0.bin\", map_location=device))\neff5_stage3 =  eff5_stage3.to(device)\neff5_stage3.eval()\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SR = 16000\nPERIOD = SR * 5\nbatch_size = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray,\n                        threshold=0.5):\n    \n    clip = clip.astype(np.float32)\n    \n    y_torch = torch.from_numpy(clip)\n    y_batch = list(y_torch.split(PERIOD))\n    \n    if y_batch[-1].shape[0] < PERIOD:\n        last = torch.zeros(PERIOD)\n        last[:y_batch[-1].shape[0]] = y_batch[-1]\n        y_batch[-1] = last\n    y_batch = torch.stack(y_batch)\n    \n    whole_size = y_batch.size(0)\n    \n    if whole_size % batch_size == 0:\n        n_iter = whole_size // batch_size\n    else:\n        n_iter = whole_size // batch_size + 1\n        \n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    \n    if site in {\"site_1\", \"site_2\"}:\n        \n        #nocall_outs = []\n        call_outs   = []\n        \n        for batch_i in range(n_iter):\n            \n            batch = y_batch[batch_i * batch_size:(batch_i + 1) * batch_size]\n            \n            with torch.no_grad():\n                \n                y = batch.to(device)\n                \n                cnn14_s3 = cnn14_stage3(y, None).data.cpu().numpy()\n\n                resnet_s3 = resnest50d_stage3(y, None).data.cpu().numpy()\n\n                eff3_s3 = eff3_stage3(y, None).data.cpu().numpy()\n                eff3_s3_f1 = eff3_stage3_f1(y, None).data.cpu().numpy()\n\n                eff4_s3 = eff4_stage3(y, None).data.cpu().numpy()\n                eff4_s3_f1 = eff4_stage3_f1(y, None).data.cpu().numpy()\n\n                eff5_s3 = eff5_stage3(y, None).data.cpu().numpy()\n                \n            \n                call_out = np.mean([cnn14_s3, resnet_s3, eff3_s3, eff3_s3_f1, eff4_s3, eff4_s3_f1, eff5_s3], axis=0).tolist()\n                \n                \n                call_outs.extend(call_out)\n                \n        \n        ebirds = []\n        \n        call_outs = np.array(call_outs)\n        \n        events = call_outs >= threshold\n        \n        for i in range(len(events)):\n            event = events[i, :]\n            labels = np.argwhere(event).reshape(-1).tolist()\n            \n            if len(labels) == 0:\n                label = \"nocall\"\n            else:\n                label = set()\n                for l in labels:\n                    label.add(INV_BIRD_CODE[l])\n                label = \" \".join(list(label))\n            \n            ebirds.append(label)\n        \n        \n        row_ids = [f'{site}_{audio_id}_{i*5}' for i in range(1, len(call_outs)+1)]\n        \n        pred_df = pd.DataFrame({\n            \"row_id\" : row_ids,\n            \"birds\" : ebirds,\n            #\"nocall\" : nocall_outs\n        })\n    \n    else:\n        \n        call_outs   = []\n        \n        for batch_i in range(n_iter):\n            \n            batch = y_batch[batch_i * batch_size:(batch_i + 1) * batch_size]\n            \n            with torch.no_grad():\n                \n                y = batch.to(device)\n                \n                \n                cnn14_s3 = cnn14_stage3(y, None).data.cpu().numpy()\n\n                resnet_s3 = resnest50d_stage3(y, None).data.cpu().numpy()\n\n                eff3_s3 = eff3_stage3(y, None).data.cpu().numpy()\n                eff3_s3_f1 = eff3_stage3_f1(y, None).data.cpu().numpy()\n\n                eff4_s3 = eff4_stage3(y, None).data.cpu().numpy()\n                eff4_s3_f1 = eff4_stage3_f1(y, None).data.cpu().numpy()\n\n                eff5_s3 = eff5_stage3(y, None).data.cpu().numpy()\n                \n            \n                call_out = np.mean([cnn14_s3, resnet_s3, eff3_s3, eff3_s3_f1, eff4_s3, eff4_s3_f1, eff5_s3], axis=0).tolist()\n                \n                \n                call_outs.extend(call_out)\n        \n        ebirds = []\n        \n        call_outs = np.array(call_outs)\n        \n        events = call_outs >= threshold\n        \n        label_set = set()\n        \n        for i in range(len(events)):\n            event = events[i, :]\n            labels = np.argwhere(event).reshape(-1).tolist()\n            \n            if len(labels) != 0:\n                for l in labels:\n                    label_set.add(INV_BIRD_CODE[l])\n        \n        label_list = list(label_set)\n        \n        if len(label_list) != 0:\n            ebirds = \" \".join(label_list)\n        else:\n            ebirds = \"nocall\"\n            \n        row_ids = test_df[\"row_id\"].values[0]\n        \n        pred_df = pd.DataFrame({\n            \"row_id\" : [row_ids],\n            \"birds\" : [ebirds],\n            #\"nocall\" : [0]\n        })\n        \n    return pred_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               threshold=0.5):\n    \n    unique_audio_id = test_df.audio_id.unique()\n    \n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    \n    for audio_id in progress_bar(unique_audio_id):\n       \n        clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                               sr=SR,\n                               mono=True,\n                               res_type=\"kaiser_fast\")\n        \n        \n        #clip, _ = librosa.load(\"../input/birdsong-recognition/example_test_audio/BLKFR-10-CPL_20190611_093000.pt540.mp3\", sr=SR, mono=True, res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        \n        \n        prediction_df = prediction_for_clip(test_df_for_audio_id,clip=clip, threshold=threshold)\n\n        prediction_dfs.append(prediction_df)\n        \n        #break\n     \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    \n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df = prediction(\n    test_df=test,\n    test_audio=TEST_AUDIO_DIR,\n    threshold=0.3\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df.birds.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(prediction_df, on=\"row_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.birds.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}