{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Libraries","metadata":{"_uuid":"9fb2b5fc-d5f7-4fc5-b93b-7b5d32193b92","_cell_guid":"e69813e6-901c-4ee1-98c3-f895bcb4e0ad","trusted":true}},{"cell_type":"code","source":"import sys\nimport cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\nimport glob\nimport pdb\nimport json\n\nimport librosa as lb\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset\n\nfrom collections import Counter\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom tqdm import tqdm\n\npytorch_timm_path = \"../input/timm-pytorch-image-models/pytorch-image-models-master\"\nsys.path.append(pytorch_timm_path)\nimport timm\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(DEVICE)","metadata":{"_uuid":"176bf6e6-b811-4603-aebc-b46c1eedceb8","_cell_guid":"9d523db2-9075-4d67-b73b-7aa97b995fc5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-01T08:28:19.474150Z","iopub.execute_input":"2021-06-01T08:28:19.474513Z","iopub.status.idle":"2021-06-01T08:28:24.962274Z","shell.execute_reply.started":"2021-06-01T08:28:19.474434Z","shell.execute_reply":"2021-06-01T08:28:24.960919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MelSpec GPU code","metadata":{}},{"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        r\"\"\"Base class for DFT and IDFT matrix.\n        \"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)  # shape: (n, n)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)  # shape: (n, n)\n        return W\n\n\nclass DFT(DFTBase):\n    def __init__(self, n, norm):\n        r\"\"\"Calculate discrete Fourier transform (DFT), inverse DFT (IDFT, \n        right DFT (RDFT) RDFT, and inverse RDFT (IRDFT.) \n\n        Args:\n          n: fft window size\n          norm: None | 'ortho'\n        \"\"\"\n        super(DFT, self).__init__()\n\n        self.W = self.dft_matrix(n)\n        self.inv_W = self.idft_matrix(n)\n\n        self.W_real = torch.Tensor(np.real(self.W))\n        self.W_imag = torch.Tensor(np.imag(self.W))\n        self.inv_W_real = torch.Tensor(np.real(self.inv_W))\n        self.inv_W_imag = torch.Tensor(np.imag(self.inv_W))\n\n        self.n = n\n        self.norm = norm\n\n    def dft(self, x_real, x_imag):\n        r\"\"\"Calculate DFT of a signal.\n\n        Args:\n            x_real: (n,), real part of a signal\n            x_imag: (n,), imag part of a signal\n\n        Returns:\n            z_real: (n,), real part of output\n            z_imag: (n,), imag part of output\n        \"\"\"\n        z_real = torch.matmul(x_real, self.W_real) - torch.matmul(x_imag, self.W_imag)\n        z_imag = torch.matmul(x_imag, self.W_real) + torch.matmul(x_real, self.W_imag)\n        # shape: (n,)\n\n        if self.norm is None:\n            pass\n        elif self.norm == 'ortho':\n            z_real /= math.sqrt(self.n)\n            z_imag /= math.sqrt(self.n)\n\n        return z_real, z_imag\n\n    def idft(self, x_real, x_imag):\n        r\"\"\"Calculate IDFT of a signal.\n\n        Args:\n            x_real: (n,), real part of a signal\n            x_imag: (n,), imag part of a signal\n        Returns:\n            z_real: (n,), real part of output\n            z_imag: (n,), imag part of output\n        \"\"\"\n        z_real = torch.matmul(x_real, self.inv_W_real) - torch.matmul(x_imag, self.inv_W_imag)\n        z_imag = torch.matmul(x_imag, self.inv_W_real) + torch.matmul(x_real, self.inv_W_imag)\n        # shape: (n,)\n\n        if self.norm is None:\n            z_real /= self.n\n        elif self.norm == 'ortho':\n            z_real /= math.sqrt(n)\n            z_imag /= math.sqrt(n)\n\n        return z_real, z_imag\n\n    def rdft(self, x_real):\n        r\"\"\"Calculate right RDFT of signal.\n\n        Args:\n            x_real: (n,), real part of a signal\n            x_imag: (n,), imag part of a signal\n\n        Returns:\n            z_real: (n // 2 + 1,), real part of output\n            z_imag: (n // 2 + 1,), imag part of output\n        \"\"\"\n        n_rfft = self.n // 2 + 1\n        z_real = torch.matmul(x_real, self.W_real[..., 0 : n_rfft])\n        z_imag = torch.matmul(x_real, self.W_imag[..., 0 : n_rfft])\n        # shape: (n // 2 + 1,)\n\n        if self.norm is None:\n            pass\n        elif self.norm == 'ortho':\n            z_real /= math.sqrt(self.n)\n            z_imag /= math.sqrt(self.n)\n\n        return z_real, z_imag\n\n    def irdft(self, x_real, x_imag):\n        r\"\"\"Calculate IRDFT of signal.\n        \n        Args:\n            x_real: (n // 2 + 1,), real part of a signal\n            x_imag: (n // 2 + 1,), imag part of a signal\n\n        Returns:\n            z_real: (n,), real part of output\n            z_imag: (n,), imag part of output\n        \"\"\"\n        n_rfft = self.n // 2 + 1\n\n        flip_x_real = torch.flip(x_real, dims=(-1,))\n        flip_x_imag = torch.flip(x_imag, dims=(-1,))\n        # shape: (n // 2 + 1,)\n\n        x_real = torch.cat((x_real, flip_x_real[..., 1 : n_rfft - 1]), dim=-1)\n        x_imag = torch.cat((x_imag, -1. * flip_x_imag[..., 1 : n_rfft - 1]), dim=-1)\n        # shape: (n,)\n\n        z_real = torch.matmul(x_real, self.inv_W_real) - torch.matmul(x_imag, self.inv_W_imag)\n        # shape: (n,)\n\n        if self.norm is None:\n            z_real /= self.n\n        elif self.norm == 'ortho':\n            z_real /= math.sqrt(n)\n\n        return z_real\n\n\nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None,\n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        r\"\"\"PyTorch implementation of STFT with Conv1d. The function has the \n        same output as librosa.stft.\n\n        Args:\n            n_fft: int, fft window size, e.g., 2048\n            hop_length: int, hop length samples, e.g., 441\n            win_length: int, window length e.g., 2048\n            window: str, window function name, e.g., 'hann'\n            center: bool\n            pad_mode: str, e.g., 'reflect'\n            freeze_parameters: bool, set to True to freeze all parameters. Set\n                to False to finetune all parameters.\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.hop_length = hop_length\n        self.win_length = win_length\n        self.window = window\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame.\n        if self.win_length is None:\n            self.win_length = n_fft\n\n        # Set the default hop, if it's not already specified.\n        if self.hop_length is None:\n            self.hop_length = int(self.win_length // 4)\n\n        fft_window = lb.filters.get_window(window, self.win_length, fftbins=True)\n\n        # Pad the window out to n_fft size.\n        fft_window = lb.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix.\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels,\n            kernel_size=n_fft, stride=self.hop_length, padding=0, dilation=1,\n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels,\n            kernel_size=n_fft, stride=self.hop_length, padding=0, dilation=1,\n            groups=1, bias=False)\n\n        # Initialize Conv1d weights.\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        r\"\"\"Calculate STFT of batch of signals.\n\n        Args: \n            input: (batch_size, data_length), input signals.\n\n        Returns:\n            real: (batch_size, 1, time_steps, n_fft // 2 + 1)\n            imag: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n\n\nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=22050, n_fft=2048, n_mels=64, fmin=0.0, fmax=None, \n        is_log=True, ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        r\"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n        if fmax == None:\n            fmax = sr//2\n\n        self.melW = lb.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        r\"\"\"Calculate (log) mel spectrogram from spectrogram.\n\n        Args:\n            input: (*, n_fft), spectrogram\n        \n        Returns: \n            output: (*, mel_bins), (log) mel spectrogram\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n        # (*, mel_bins)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n    def power_to_db(self, input):\n        r\"\"\"Power to db, this function is the pytorch implementation of \n        librosa.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise lb.util.exceptions.ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec\n\n\nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None,\n        window='hann', center=True, pad_mode='reflect', power=2.0,\n        freeze_parameters=True):\n        r\"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length,\n            win_length=win_length, window=window, center=center,\n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        r\"\"\"Calculate spectrogram of input signals.\n        Args: \n            input: (batch_size, data_length)\n\n        Returns:\n            spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (self.power / 2.0)\n\n        return spectrogram\n\n\nclass DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n\n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n#         if self.training is False:\n#             return input\n\n#         else:\n        batch_size = input.shape[0]\n        total_width = input.shape[self.dim]\n\n        for n in range(batch_size):\n            self.transform_slice(input[n], total_width)\n\n        return input\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:28:24.964826Z","iopub.execute_input":"2021-06-01T08:28:24.965191Z","iopub.status.idle":"2021-06-01T08:28:25.143648Z","shell.execute_reply.started":"2021-06-01T08:28:24.965152Z","shell.execute_reply":"2021-06-01T08:28:25.142671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset and Loading","metadata":{"_uuid":"f507766d-ab46-47bc-892e-e0871de8dbcb","_cell_guid":"ac20c5da-8c13-4e6c-9cb8-ad7e5e45c6ed","trusted":true}},{"cell_type":"code","source":"TEST = (len(list(Path(\"../input/birdclef-2021/test_soundscapes/\").glob(\"*.ogg\"))) != 0)\n# SAMPLE_SUB_PATH = None\nif TEST:\n    DATADIR = Path(\"../input/birdclef-2021/test_soundscapes/\")\n    SAMPLE_SUB_PATH = \"../input/birdclef-2021/sample_submission.csv\"\nelse:\n    DATADIR = Path(\"../input/birdclef-2021/train_soundscapes/\")\n    SAMPLE_SUB_PATH = None","metadata":{"_uuid":"cca50c9f-68f4-49fc-a811-050c176f32e4","_cell_guid":"191d848f-afab-4886-82ae-ad263c95bebf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-01T08:28:25.145707Z","iopub.execute_input":"2021-06-01T08:28:25.146011Z","iopub.status.idle":"2021-06-01T08:28:25.163638Z","shell.execute_reply.started":"2021-06-01T08:28:25.145953Z","shell.execute_reply":"2021-06-01T08:28:25.162937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.DataFrame(\n     [(path.stem, *path.stem.split(\"_\"), path) for path in DATADIR.glob(\"*.ogg\")],\n    columns = [\"filename\", \"id\", \"site\", \"date\", \"filepath\"]\n)\nmap_Latitude = {\n    'COL': 5.57,\n    'COR': 10.12,\n    'SNE': 38.49,\n    'SSW': 42.47,\n}\nmap_Longitude = {\n    'COL': -75.85,\n    'COR': -84.51,\n    'SNE': -119.95,\n    'SSW': -76.45\n}\n\ndata['month'] = data['date'].apply(lambda x: int(x[4:6]) - 1)\ndata['latitude'] = data['site'].map(map_Latitude)\ndata['longitude'] = data['site'].map(map_Longitude)\nprint(data.shape)\n\ndf_train = pd.read_csv(\"../input/birdclef-2021/train_metadata.csv\")\n\nTARG_NAMES = np.unique(df_train['primary_label'].values).tolist()\nprint(f\"targ length: {len(TARG_NAMES)}\")","metadata":{"_uuid":"4202ef18-18d3-4a64-974e-b4566c085529","_cell_guid":"e503b47d-9bed-4116-be52-637880cbcd6c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-01T08:28:25.166670Z","iopub.execute_input":"2021-06-01T08:28:25.166902Z","iopub.status.idle":"2021-06-01T08:28:25.653110Z","shell.execute_reply.started":"2021-06-01T08:28:25.166880Z","shell.execute_reply":"2021-06-01T08:28:25.652157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.tail(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:28:25.655971Z","iopub.execute_input":"2021-06-01T08:28:25.656309Z","iopub.status.idle":"2021-06-01T08:28:25.675033Z","shell.execute_reply.started":"2021-06-01T08:28:25.656276Z","shell.execute_reply":"2021-06-01T08:28:25.674074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n    \n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = torch.clamp(X, _min, _max)\n        V = (V - _min) / (_max - _min)\n    else:\n        V = torch.zeros_like(X)\n\n    return V\n\n\nclass MelSpecComputer:\n    def __init__(self, sr, n_mels, fmin, fmax, n_fft, hop_length):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax\n        self.n_fft = n_fft\n        self.hop_length = hop_length\n\n    def __call__(self, y):\n\n        melspec = lb.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax, n_fft=self.n_fft, hop_length=self.hop_length\n        )\n\n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec\n\n\nclass TestDataset(Dataset):\n    def __init__(self, data, sr=32000, n_mels=128, fmin=0, fmax=None, n_fft=2048, hop_length=512, duration=5, step=None, res_type=\"kaiser_fast\", resample=True):\n        \n        self.data = data\n        \n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n        self.n_fft = n_fft\n        self.hop_length = hop_length\n\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.step = step or self.audio_length\n        \n        self.res_type = res_type\n        self.resample = resample\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax, n_fft=self.n_fft, hop_length=self.hop_length)\n\n    def __len__(self):\n        return len(self.data)\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n    \n    def audio_to_image(self, audio):\n        melspec = self.mel_spec_computer(audio) \n        image = mono_to_color(melspec)\n        image = self.normalize(image)\n        return image\n\n    def read_file(self, filepath):\n        audio, orig_sr = sf.read(filepath, dtype=\"float32\")\n\n        if self.resample and orig_sr != self.sr:\n            audio = lb.resample(audio, orig_sr, self.sr, res_type=self.res_type)\n          \n        audios = []\n        for i in range(self.audio_length, len(audio) + self.step, self.step):\n            start = max(0, i - self.audio_length)\n            end = start + self.audio_length\n            audios.append(audio[start:end])\n            \n        if len(audios[-1]) < self.audio_length:\n            audios = audios[:-1]\n            \n        images = np.stack(audios)\n        \n        return images\n    \n    def __getitem__(self, idx):\n        sample = self.data.loc[idx]\n        images = self.read_file(sample[\"filepath\"])        \n        ### meta feat\n        meta = np.zeros(4)\n        if sample.month >= 0:\n            meta[0] = month2cosine(sample.month)\n            meta[1] = month2sin(sample.month)\n        meta[2] = float(sample.latitude)\n        meta[3] = float(sample.longitude)\n        meta = torch.tensor(meta).float()\n        ###\n        return images, meta\n\ndef month2cosine(month):\n    month_norm = 2 * np.pi * month / 12\n    return np.cos(month_norm)\n\ndef month2sin(month):\n    month_norm = 2 * np.pi * month / 12\n    return np.sin(month_norm)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:28:25.676921Z","iopub.execute_input":"2021-06-01T08:28:25.677298Z","iopub.status.idle":"2021-06-01T08:28:25.697676Z","shell.execute_reply.started":"2021-06-01T08:28:25.677262Z","shell.execute_reply":"2021-06-01T08:28:25.696598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{"_uuid":"a74c8307-278e-4d6f-86cb-4fb3e15fea10","_cell_guid":"79cd3ebd-faad-4321-a88b-c5cb0088e329","trusted":true}},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\nclass DownPool(nn.Module):\n    def __init__(self, pool_stride, conv_stride):\n        super().__init__()\n        self.avgpool = nn.AvgPool2d(3, stride=pool_stride, padding=1)\n        self.downconv = nn.Sequential(\n                            nn.Conv2d(1, 2, kernel_size=5, stride=conv_stride, padding=2, bias=False),\n                            nn.BatchNorm2d(2),\n                            nn.ReLU()\n                        )\n    def forward(self, x):\n        x = torch.cat((self.avgpool(x), self.downconv(x)), dim=1)\n        return x\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, \n                 num_classes=397, in_channels=1, norm_free=False, mix_up=False, downpool=False,\n                 n_fft=2048, hop_length=800, fmin=20, fmax=16000, n_mels=128, pool_stride=(2,2), conv_stride=(2,2)):\n        super(TimmSED, self).__init__()\n        self.num_channel = in_channels\n        self.mix_up = mix_up\n        self.downpool=None\n        if downpool:\n            self.downpool = DownPool(pool_stride, conv_stride)\n            in_channels = 3\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=n_fft, hop_length=hop_length,\n                                                 win_length=n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=32000, n_fft=n_fft,\n                                                 n_mels=n_mels, \n                                                 fmin=fmin, fmax=fmax, \n                                                 ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n        \n        base_model = timm.create_model(base_model_name, pretrained=pretrained, in_chans=in_channels)\n        if 'rexnet' in base_model_name:\n            layers = list(base_model.children())[:-1]\n            fc = list(base_model.children())[-1].fc\n        else:\n            layers = list(base_model.children())[:-2]\n            fc = list(base_model.children())[-1]\n        self.encoder = nn.Sequential(*layers)\n        if norm_free:\n            fc = fc.fc        \n        in_features = fc.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(in_features, num_classes, activation=\"sigmoid\")\n        \n        self.MLP = nn.Sequential(\n            nn.Linear(num_classes+4, 256),\n            nn.BatchNorm1d(256),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, num_classes)\n        )\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.fc1)\n\n    def preprocess(self, input_x, mixup_lambda=None):\n\n        x = self.spectrogram_extractor(input_x)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n        x = mono_to_color(x)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        return x, frames_num\n\n    def forward(self, x, meta):\n        # input shape: (batch_size, audio_len)\n        x, frames_num = self.preprocess(x)\n        \n        x = x.transpose(2, 3)  # (batch_size, 1, mel_bins, time_steps)\n        if self.num_channel == 3:\n            x = torch.cat([x,x,x], dim=1)\n        if self.downpool:\n            x = self.downpool(x)\n\n        x = self.encoder(x)  # (batch_size, channels, freq, frames)\n        x = torch.mean(x, dim=2)  # (batch_size, channels, frames)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2  # (batch_size, channels, frames)\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)  # (batch_size, frames, channels)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)  # (batch_size, channels, frames)\n        x = F.dropout(x, p=0.5, training=self.training)\n        \n        # calculate attention\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n#         logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        xl = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        ### add region and month feature here?\n        xl = torch.cat([xl, meta], 1)\n        logit = self.MLP(xl)\n        ###\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # # Get framewise output\n        # framewise_output = interpolate(segmentwise_output, interpolate_ratio)\n        # framewise_output = pad_framewise_output(framewise_output, frames_num)\n        \n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            # \"framewise_output\": framewise_output,\n            # \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            # \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict\n\n\nclass BirdCallModel(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, \n                 num_classes=397, in_channels=3):\n        super(BirdCallModel, self).__init__()\n\n        base_model = timm.create_model(base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        fc = list(base_model.children())[-1]\n        self.encoder = nn.Sequential(*layers)\n        \n        self.in_features = fc.in_features\n        \n        # self.pooling = GeM()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.classifier = nn.Linear(self.in_features, num_classes, bias=True)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.classifier)\n\n    def forward(self, x):\n        mb_size = x.shape[0]\n        x = self.encoder(x)\n        x = self.pooling(x)\n        # x = torch.mean(x, dim=2)  # (batch_size, channels, frames)\n        out = self.classifier(x.view(mb_size, self.in_features))\n        return {'logit': out}\n","metadata":{"_uuid":"f2856857-ed1b-4620-b6f4-87a58ae3f08d","_cell_guid":"4cbcba18-efb0-4d13-821c-7e480c5bb699","execution":{"iopub.status.busy":"2021-06-01T08:28:25.699792Z","iopub.execute_input":"2021-06-01T08:28:25.700303Z","iopub.status.idle":"2021-06-01T08:28:25.743413Z","shell.execute_reply.started":"2021-06-01T08:28:25.700263Z","shell.execute_reply":"2021-06-01T08:28:25.742524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Models","metadata":{"_uuid":"bb5a8a84-26b4-4d47-9a7c-0b10fbbcfefe","_cell_guid":"8d6cfecb-7ea6-4b27-b6a5-4db36337920e","trusted":true}},{"cell_type":"code","source":"def prepare_model_for_inference(model, path: Path):\n    try:\n        ckpt = torch.load(path, map_location=\"cpu\")\n        model.load_state_dict(ckpt[\"model\"])\n    except:\n        model.load_state_dict(torch.load(path, map_location=\"cpu\"))\n    model.eval()\n    return model","metadata":{"_uuid":"70706858-4d8a-470c-88f4-44db0f7d3790","_cell_guid":"0c96efae-7c7d-4077-b00b-49c3e2cb8777","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-01T08:28:25.744710Z","iopub.execute_input":"2021-06-01T08:28:25.745106Z","iopub.status.idle":"2021-06-01T08:28:25.757247Z","shell.execute_reply.started":"2021-06-01T08:28:25.745070Z","shell.execute_reply":"2021-06-01T08:28:25.756511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**10 seconds models**","metadata":{}},{"cell_type":"code","source":"weights_10s_path = [\n    (\"SED\", \"tf_efficientnet_b4_ns\", Path(\"../input/birdclef-2021-models-qishen/b4_10s_cyclic_v1_bs64_lr2e3_50epo_best_loss_fold3.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"tf_efficientnet_b4_ns\", Path(\"../input/birdclef-2021-models-qishen/b4_10s_cyclic_v1_bs64_lr2e3_50epo_best_loss_fold4.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    \n    (\"SED\", \"tf_efficientnet_b5_ns\", Path(\"../input/birdclef-2021-models-qishen/b5_10s_cyclic_v1_bs64_lr2e3_50epo_best_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"tf_efficientnet_b5_ns\", Path(\"../input/birdclef-2021-models-qishen/b5_10s_cyclic_v1_bs64_lr2e3_50epo_best_fold4.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"rexnet_150\", Path(\"../input/d/underwearfitting/bird-clef-10s-models/rexnet_150_10s_best_fold4.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    # (\"SED\", \"rexnet_150\", Path(\"../input/d/underwearfitting/bird-clef-10s-models/rexnet_150_10s_best_fold1.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"rexnet_200\", Path(\"../input/d/underwearfitting/bird-clef-10s-models/rexnet_200_10s_best_fold1.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    # (\"SED\", \"rexnet_200\", Path(\"../input/d/underwearfitting/bird-clef-10s-models/rexnet_200_10s_best_fold2.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"nf_resnet50\", Path(\"../input/lee-nf50-fold0-10s/nf50_128_cyclic_10s_lr2e3_50epo_bestscore_fold0.pth/nf50_128_cyclic_10s_lr2e3_50epo_bestscore_fold0.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n    (\"SED\", \"nf_resnet50\", Path(\"../input/nf50-10s-cyclic/nf50_128_cyclic_10s_bestscore_fold4.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"eca_nfnet_l0\", Path(\"../input/eca-nfnet-f34-10s/_eca_nfnet_l0-10s_50epo_bestloss_fold3.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n    (\"SED\", \"eca_nfnet_l0\", Path(\"../input/eca-nfnet-f34-10s/_eca_nfnet_l0-10s_50epo_bestloss_fold4.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"eca_nfnet_l1\", Path(\"../input/bird-eca-l1-10s/ecal1_128_cyclic_bestloss_fold1.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n]\n\nmodels_10s = []\nfor wp in tqdm(weights_10s_path):\n    if wp[0] == \"SED\":\n        net = TimmSED(wp[1], num_classes=397, in_channels=wp[4], n_fft=wp[5], hop_length=wp[6], fmin=50, fmax=16000, n_mels=wp[9], \n                      norm_free=wp[7], downpool=wp[8], pool_stride=wp[10], conv_stride=wp[11])\n    else:\n        net = BirdCallModel(wp[1], num_classes=397, in_channels=3)\n    net = net.to(DEVICE)\n    net = prepare_model_for_inference(net, wp[2])\n    models_10s.append(net)\n\nlen(models_10s)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:28:25.759100Z","iopub.execute_input":"2021-06-01T08:28:25.759708Z","iopub.status.idle":"2021-06-01T08:29:11.015570Z","shell.execute_reply.started":"2021-06-01T08:28:25.759671Z","shell.execute_reply":"2021-06-01T08:29:11.014863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**15 seconds models**","metadata":{}},{"cell_type":"code","source":"weights_15s_path = [\n    (\"SED\", \"tf_efficientnet_b4_ns\", Path(\"../input/bird-clef-128-v12-fold0-b4ns/b4_cyclic_best_fold3.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),  \n\n    (\"SED\", \"nf_resnet50\", Path(\"../input/birdclefnf50meta/nf50_128_cyclic_v1_bs64_lr2e3_50epo_bestscore_fold0.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n    (\"SED\", \"nf_resnet50\", Path(\"../input/birdclefnf50meta/nf50_128_cyclic_v1_bs64_lr2e3_50epo_bestscore_fold4.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n    (\"SED\", \"nf_resnet50\", Path(\"../input/birdclefnf50meta/nf50_128_cyclic_v1_bs64_lr2e3_50epo_bestscore_fold2.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"dm_nfnet_f0\", Path(\"../input/dimf0-temp-birdelf/nff0_128_cyclic_v1_bs64_lr2e3_50epo_bestscore_fold3.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"rexnet_200\", Path(\"../input/bird-clef-rexnet200-fold2/rexnet_200_cyclic_v1_bs64_lr2e3_50epo_best_fold2.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"rexnet_200\", Path(\"../input/birdclef-2021-models-qishen/rexnet_200_cyclic_v1_bs62_lr2e3_50epo_best_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    # (\"SED\", \"rexnet_200\", Path(\"../input/birdclef-2021-models-qishen/rexnet_200_cyclic_v1_bs62_lr2e3_50epo_best_fold4.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"efficientnetv2_rw_s\", Path(\"../input/effnetv2-birdelf-temp/effnetv2_best_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"eca_nfnet_l0\", Path(\"../input/bird-eca-nfnet-l0-cyclic-fold0/nf50_128_cyclic_v1_bs64_lr2e3_50epo_bestloss_fold0.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"eca_nfnet_l1\", Path(\"../input/birdclef2021-eca-nfnet-l1/eca_nfnet_l1_128_cyclic_v1_bs64_lr2e3_50epo_bestloss_fold0.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"rexnet_150\", Path(\"../input/rex150-birdelf/rexnet_150_cyclic_v1_bs64_lr2e3_50epo_best_fold2.pthbest_sc\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"rexnet_150\", Path(\"../input/rex150-birdelf/rexnet_150_cyclic_v1_bs64_lr2e3_50epo_best_fold4.pthbest_sc\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n\n    (\"SED\", \"rexnet_200\", Path(\"../input/bird-25d-rex200-40epochs-fold0/rex200_cyclic_v1_sample4bs16_lr2e3_40epo_best_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n]\n\nmodels_15s = []\nfor wp in tqdm(weights_15s_path):\n    if wp[0] == \"SED\":\n        net = TimmSED(wp[1], num_classes=397, in_channels=wp[4], n_fft=wp[5], hop_length=wp[6], fmin=50, fmax=16000, n_mels=wp[9], \n                      norm_free=wp[7], downpool=wp[8], pool_stride=wp[10], conv_stride=wp[11])\n    else:\n        net = BirdCallModel(wp[1], num_classes=397, in_channels=3)\n    net = net.to(DEVICE)\n    net = prepare_model_for_inference(net, wp[2])\n    models_15s.append(net)\n    \nlen(models_15s)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:29:11.016932Z","iopub.execute_input":"2021-06-01T08:29:11.017277Z","iopub.status.idle":"2021-06-01T08:29:56.839707Z","shell.execute_reply.started":"2021-06-01T08:29:11.017242Z","shell.execute_reply":"2021-06-01T08:29:56.839054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**20 seconds models**","metadata":{}},{"cell_type":"code","source":"weights_20s_path = [\n    (\"SED\", \"rexnet_200\", Path(\"../input/birdclef-2021-models-qishen/rexnet_200_20s_cyclic_v1_bs56_lr2e3_50epo_best_loss_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"rexnet_200\", Path(\"../input/birdclef-2021-models-qishen/rexnet_200_20s_cyclic_v1_bs56_lr2e3_50epo_best_loss_fold1.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    \n    (\"SED\", \"tf_efficientnet_b5_ns\", Path(\"../input/d/underwearfitting/birdclef-model-20s/b5ns_20s_best_fold3.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    (\"SED\", \"tf_efficientnet_b5_ns\", Path(\"../input/birdclef2021-eca-nfnet-l1/tf_efficientnet_b5_ns_20s_128_cyclic_v1_bs64_lr2e3_bestloss_fold3.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), \n    \n    (\"SED\", \"tf_efficientnet_b4_ns\", Path(\"../input/d/underwearfitting/birdclef-model-20s/b4ns_20s_best_fold3.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), # 0.781 @0.42\n    (\"SED\", \"tf_efficientnet_b4_ns\", Path(\"../input/birdclef2021-eca-nfnet-l1/tf_efficientnet_b4_20s_ns_128_cyclic_v1_bs64_lr2e3_bestscore_fold4.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), \n    \n    (\"SED\", \"tf_efficientnet_b3_ns\", Path(\"../input/birdclef-2021-models-qishen/b3_20s_cyclic_v1_bs64_lr2e3_50epo_best_loss_fold2.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), #0.759 @0.45\n    (\"SED\", \"tf_efficientnet_b3_ns\", Path(\"../input/birdclef-2021-models-qishen/b3_20s_cyclic_v1_bs64_lr2e3_50epo_best_loss_fold4.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n    \n    (\"SED\", \"nf_resnet50\", Path(\"../input/birdclef2021-eca-nfnet-l1/nf50_128_cyclic_20s_lr2e3_50epo_bestloss_fold0.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)), # 0.772 @0.42\n    (\"SED\", \"nf_resnet50\", Path(\"../input/birdclefnf50meta/nf50_128_cyclic_20s_lr2e3_bestloss_fold2.pth\"), 0.3, 1, 2048, 512, True, False,128,(1,2), (1,2)), \n    \n    (\"SED\", \"efficientnetv2_rw_s\", Path(\"../input/birdenetv2-20s-fold0/enetv2_20s_best_fold0.pth\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), # 0.772 @0.42\n    # (\"SED\", \"efficientnetv2_rw_s\", Path(\"../input/birdenetv2-20s-fold0/enetv2_20s_best_fold0.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)), \n    \n    (\"SED\", \"rexnet_150\", Path(\"../input/d/underwearfitting/birdclef-model-20s/rex150_20s_best_fold2.pthbest_loss\"), 0.3, 1, 2048, 512, False, False,128,(1,2), (1,2)),\n]\n\nmodels_20s = []\nfor wp in tqdm(weights_20s_path):\n    if wp[0] == \"SED\":\n        net = TimmSED(wp[1], num_classes=397, in_channels=wp[4], n_fft=wp[5], hop_length=wp[6], fmin=50, fmax=16000, n_mels=wp[9], \n                      norm_free=wp[7], downpool=wp[8], pool_stride=wp[10], conv_stride=wp[11])\n    else:\n        net = BirdCallModel(wp[1], num_classes=397, in_channels=3)\n    net = net.to(DEVICE)\n    net = prepare_model_for_inference(net, wp[2])\n    models_20s.append(net)\n    \nlen(models_20s)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:29:56.841958Z","iopub.execute_input":"2021-06-01T08:29:56.842471Z","iopub.status.idle":"2021-06-01T08:30:35.063666Z","shell.execute_reply.started":"2021-06-01T08:29:56.842431Z","shell.execute_reply":"2021-06-01T08:30:35.063000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:30:35.065090Z","iopub.execute_input":"2021-06-01T08:30:35.065434Z","iopub.status.idle":"2021-06-01T08:30:35.853573Z","shell.execute_reply.started":"2021-06-01T08:30:35.065398Z","shell.execute_reply":"2021-06-01T08:30:35.851797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Probabilities","metadata":{}},{"cell_type":"code","source":"test_data = TestDataset(data=data)\nlen(test_data), test_data[0][0].shape, test_data[0][1].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:30:35.855182Z","iopub.execute_input":"2021-06-01T08:30:35.855541Z","iopub.status.idle":"2021-06-01T08:30:37.118725Z","shell.execute_reply.started":"2021-06-01T08:30:35.855505Z","shell.execute_reply":"2021-06-01T08:30:37.118020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"little_bs = 32\n\n\ndef get_prob_zero_pad(x, meta, models, pad_seconds):\n    ### zero pad center\n    xa = torch.cat([\n        torch.zeros(x.shape[0], int(32000 * pad_seconds / 2)).to(DEVICE),\n        x,\n        torch.zeros(x.shape[0], int(32000 * pad_seconds / 2)).to(DEVICE),\n    ], 1)\n    meta_a = meta.unsqueeze(0).repeat(xa.shape[0], 1)\n    prob_a = torch.stack([\n        torch.cat([\n                    m(xa[b:b+little_bs], meta_a[b:b+little_bs])['logit'].sigmoid()\n                        for b in range(0, xa.shape[0], little_bs)\n                 ])\n        for m in models\n    ], 0).mean(0)\n\n    ### zero pad left\n    xb = torch.cat([\n        x,\n        torch.zeros(x.shape[0], int(32000 * pad_seconds)).to(DEVICE),\n    ], 1)\n    prob_b = torch.stack([\n        torch.cat([\n                    m(xb[b:b+little_bs], meta_a[b:b+little_bs])['logit'].sigmoid()\n                        for b in range(0, xb.shape[0], little_bs)\n                 ])\n        for m in models\n    ], 0).mean(0)\n\n    ### zero pad right\n    xc = torch.cat([\n        torch.zeros(x.shape[0], int(32000 * pad_seconds)).to(DEVICE),\n        x,\n    ], 1)\n    prob_c = torch.stack([\n        torch.cat([\n                    m(xc[b:b+little_bs], meta_a[b:b+little_bs])['logit'].sigmoid()\n                        for b in range(0, xc.shape[0], little_bs)\n                 ])\n        for m in models\n    ], 0).mean(0)\n\n    return (prob_a + prob_b + prob_c) / 3.\n\n\ndef get_prob_avg_pool(x, meta, models, mode):\n    ### concat then avg pool\n    if mode == '10s':\n        zero_pad = 1\n    elif mode == '15s':\n        zero_pad = 2\n    elif mode == '20s':\n        zero_pad = 3\n    else:\n        raise\n\n    xd = torch.cat([torch.zeros(zero_pad, int(32000 * 5)).to(DEVICE), x, torch.zeros(zero_pad, int(32000 * 5)).to(DEVICE)])\n    \n    if mode == '10s':\n        xd = torch.cat([xd[:-1], xd[1:]], 1)\n    elif mode == '15s':\n        xd = torch.cat([xd[:-2], xd[1:-1], xd[2:]], 1)\n    elif mode == '20s':\n        xd = torch.cat([xd[:-3],xd[1:-2],xd[2:-1],xd[3:]], 1)\n    \n    meta_d = meta.unsqueeze(0).repeat(xd.shape[0], 1)\n    prob = torch.stack([\n        torch.cat([\n                    m(xd[b:b+little_bs], meta_d[b:b+little_bs])['logit'].sigmoid()\n                        for b in range(0, xd.shape[0], little_bs)\n                 ])\n        for m in models\n    ], 0).mean(0)\n\n    prob_d = F.avg_pool1d(prob.transpose(1,0).unsqueeze(1), kernel_size=zero_pad+1, stride=1).squeeze(1).transpose(1,0)\n    return prob_d","metadata":{"execution":{"iopub.status.busy":"2021-06-01T08:30:37.119956Z","iopub.execute_input":"2021-06-01T08:30:37.120281Z","iopub.status.idle":"2021-06-01T08:30:37.138428Z","shell.execute_reply.started":"2021-06-01T08:30:37.120255Z","shell.execute_reply":"2021-06-01T08:30:37.137557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_prob = []\nwith torch.no_grad():\n    for idx in tqdm(list(range(len(test_data)))):\n        x, meta = test_data[idx]\n        x = torch.tensor(x).to(DEVICE)\n        meta = meta.to(DEVICE)\n        \n        ### 10 Sec\n        prob_zero_pad = get_prob_zero_pad(x, meta, models_10s, pad_seconds=5)\n        prob_avg_pool = get_prob_avg_pool(x, meta, models_10s, mode='10s')\n        prob_10s = (prob_zero_pad + prob_avg_pool) / 2.\n\n        ### 15 Sec\n        prob_zero_pad = get_prob_zero_pad(x, meta, models_15s, pad_seconds=10)\n        prob_avg_pool = get_prob_avg_pool(x, meta, models_15s, mode='15s')\n        prob_15s = (prob_zero_pad + prob_avg_pool) / 2.\n\n        ### 20 Sec\n        prob_zero_pad = get_prob_zero_pad(x, meta, models_20s, pad_seconds=15)\n        prob_avg_pool = get_prob_avg_pool(x, meta, models_20s, mode='20s')\n        prob_20s = (prob_zero_pad + prob_avg_pool) / 2.\n\n        ### Final\n        prob_final = (prob_15s * len(models_15s) + prob_10s * len(models_10s) + prob_20s * len(models_20s)) / \\\n                     (len(models_15s) + len(models_10s) + len(models_20s))\n\n        model_prob.append(prob_final.cpu())\n\nmodel_prob = torch.cat(model_prob)","metadata":{"_uuid":"7c233686-5b67-4a05-bae0-a4853ea61760","_cell_guid":"1160296c-1152-4fc3-a3c7-8375615c5f90","execution":{"iopub.status.busy":"2021-06-01T08:30:37.139581Z","iopub.execute_input":"2021-06-01T08:30:37.140069Z","iopub.status.idle":"2021-06-01T09:11:22.545000Z","shell.execute_reply.started":"2021-06-01T08:30:37.140015Z","shell.execute_reply":"2021-06-01T09:11:22.544140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Post Process Probabilities","metadata":{}},{"cell_type":"code","source":"def get_thresh_preds(probs, thresh=0.5):\n    o = (-probs).argsort(dim=1)\n    npreds = torch.sum(probs > thresh, dim=1)\n    preds = []\n    for prob_idx, npred in zip(o, npreds):\n        preds.append(prob_idx[:npred].cpu().numpy().tolist())\n    return preds\n\n\ndef get_bird_names(preds, ebird_names):\n    bird_names = []\n    for pred in preds:\n        if not pred:\n            bird_names.append(\"nocall\")\n        else:\n            bird_names.append(\" \".join([ebird_names[bird_id] for bird_id in pred]))\n    return bird_names\n\n\ndef preds_as_df(data, preds):\n    sub = {\n        \"row_id\": [],\n        \"birds\": [],\n    }\n\n    for row in data.itertuples():\n        row_id = [f\"{row.id}_{row.site}_{5*i}\" for i in range(1, 121)]\n#         sub[\"birds\"] += pred\n        sub[\"row_id\"] += row_id\n\n    sub[\"birds\"] += preds[0]\n    sub = pd.DataFrame(sub)\n\n    if SAMPLE_SUB_PATH:\n        sample_sub = pd.read_csv(SAMPLE_SUB_PATH, usecols=[\"row_id\"])\n        sub = sample_sub.merge(sub, on=\"row_id\", how=\"left\")\n        sub[\"birds\"] = sub[\"birds\"].fillna(\"nocall\")\n    return sub\n\ndef get_metrics(s_true, s_pred):\n    s_true = set(s_true.split())\n    s_pred = set(s_pred.split())\n    n, n_true, n_pred = len(s_true.intersection(s_pred)), len(s_true), len(s_pred)\n    \n    prec = n/n_pred\n    rec = n/n_true\n    f1 = 2*prec*rec/(prec + rec) if prec + rec else 0\n    \n    return {\"f1\": f1, \"prec\": prec, \"rec\": rec, \"n_true\": n_true, \"n_pred\": n_pred, \"n\": n}\n","metadata":{"execution":{"iopub.status.busy":"2021-06-01T09:11:27.785764Z","iopub.execute_input":"2021-06-01T09:11:27.786108Z","iopub.status.idle":"2021-06-01T09:11:27.798400Z","shell.execute_reply.started":"2021-06-01T09:11:27.786077Z","shell.execute_reply":"2021-06-01T09:11:27.797504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = get_thresh_preds(model_prob, 0.36)\nnamed_pred = get_bird_names(preds, TARG_NAMES)\nprediction_df = preds_as_df(data, [named_pred])\nprediction_df.to_csv(\"submission.csv\", index=False)\nif \"train_soundscapes\" in DATADIR.name:\n    train_labels = pd.read_csv(\"../input/birdclef-2021/train_soundscape_labels.csv\")\n#     remove_sound = [\"7019_COR_20190904\", \"7954_COR_20190923\",\"31928_COR_20191004\"]\n    remove_sound = [\"7019_COR\", \"7954_COR\",\"31928_COR\"]\n    each_sub = pd.read_csv(\"submission.csv\")\n    sub_target = train_labels.merge(each_sub, how=\"left\", on=\"row_id\")\n\n    print(sub_target[\"birds_x\"].notnull().sum(), sub_target[\"birds_x\"].notnull().sum())\n    assert sub_target[\"birds_x\"].notnull().all()\n    assert sub_target[\"birds_y\"].notnull().all()\n\n    df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target.birds_x, sub_target.birds_y)])\n    df_metrics_1 = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred, row_idx in zip(sub_target.birds_x, sub_target.birds_y, sub_target.row_id) if \"_\".join(row_idx.split(\"_\")[:-1]) not in remove_sound])\n\n    print(f\"{df_metrics.mean()}\")\n    print(\"=================\")\n    print(f\"{df_metrics_1.mean()}\")\n    print(\"=================\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-06-01T09:12:16.108640Z","iopub.execute_input":"2021-06-01T09:12:16.111314Z","iopub.status.idle":"2021-06-01T09:12:16.301732Z","shell.execute_reply.started":"2021-06-01T09:12:16.111266Z","shell.execute_reply":"2021-06-01T09:12:16.300846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}