{"cells":[{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:16:58.757038Z","iopub.status.busy":"2020-09-10T12:16:58.756327Z","iopub.status.idle":"2020-09-10T12:17:09.205611Z","shell.execute_reply":"2020-09-10T12:17:09.204791Z"},"papermill":{"duration":10.46751,"end_time":"2020-09-10T12:17:09.205781","exception":false,"start_time":"2020-09-10T12:16:58.738271","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\n\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\nfrom fastprogress import progress_bar\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, average_precision_score","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-09-10T12:17:09.249744Z","iopub.status.busy":"2020-09-10T12:17:09.248886Z","iopub.status.idle":"2020-09-10T12:17:09.267376Z","shell.execute_reply":"2020-09-10T12:17:09.266671Z"},"papermill":{"duration":0.044056,"end_time":"2020-09-10T12:17:09.267548","exception":false,"start_time":"2020-09-10T12:17:09.223492","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-09-10T12:17:09.311043Z","iopub.status.busy":"2020-09-10T12:17:09.310296Z","iopub.status.idle":"2020-09-10T12:17:09.314947Z","shell.execute_reply":"2020-09-10T12:17:09.315467Z"},"papermill":{"duration":0.030425,"end_time":"2020-09-10T12:17:09.315622","exception":false,"start_time":"2020-09-10T12:17:09.285197","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:09.358844Z","iopub.status.busy":"2020-09-10T12:17:09.357975Z","iopub.status.idle":"2020-09-10T12:17:09.641433Z","shell.execute_reply":"2020-09-10T12:17:09.640329Z"},"papermill":{"duration":0.310084,"end_time":"2020-09-10T12:17:09.641576","exception":false,"start_time":"2020-09-10T12:17:09.331492","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.010531,"end_time":"2020-09-10T12:17:09.663065","exception":false,"start_time":"2020-09-10T12:17:09.652534","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### torchlibrosa\n\n\nIn PANNs, `torchlibrosa`, a PyTorch based implementation are used to replace some of the `librosa`'s functions. Here I use some functions of `torchlibrosa`.\n\nRef: https://github.com/qiuqiangkong/torchlibrosa"},{"metadata":{"papermill":{"duration":0.010367,"end_time":"2020-09-10T12:17:09.683879","exception":false,"start_time":"2020-09-10T12:17:09.673512","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### LICENSE\n\n```\nISC License\nCopyright (c) 2013--2017, librosa development team.\n\nPermission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n```"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:09.739365Z","iopub.status.busy":"2020-09-10T12:17:09.728925Z","iopub.status.idle":"2020-09-10T12:17:09.7487Z","shell.execute_reply":"2020-09-10T12:17:09.748214Z"},"papermill":{"duration":0.054515,"end_time":"2020-09-10T12:17:09.7488","exception":false,"start_time":"2020-09-10T12:17:09.694285","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n    \nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, data_length)\n        Returns:\n          real: (batch_size, n_fft // 2 + 1, time_steps)\n          imag: (batch_size, n_fft // 2 + 1, time_steps)\n        \"\"\"\n\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        # (batch_size, 1, time_steps, n_fft // 2 + 1)\n\n        return real, imag\n    \n    \nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        \"\"\"Calculate spectrogram using pytorch. The STFT is implemented with \n        Conv1d. The function has the same output of librosa.core.stft\n        \"\"\"\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        Returns:\n          spectrogram: (batch_size, 1, time_steps, n_fft // 2 + 1)\n        \"\"\"\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        \"\"\"Power to db, this function is the pytorch implementation of \n        librosa.core.power_to_lb\n        \"\"\"\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:09.786463Z","iopub.status.busy":"2020-09-10T12:17:09.785219Z","iopub.status.idle":"2020-09-10T12:17:09.787742Z","shell.execute_reply":"2020-09-10T12:17:09.788136Z"},"papermill":{"duration":0.028773,"end_time":"2020-09-10T12:17:09.788245","exception":false,"start_time":"2020-09-10T12:17:09.759472","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        \"\"\"Drop stripes. \n        Args:\n          dim: int, dimension along which to drop\n          drop_width: int, maximum width of stripes to drop\n          stripes_num: int, how many stripes to drop\n        \"\"\"\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, \n        freq_stripes_num):\n        \"\"\"Spec augmetation. \n        [ref] Park, D.S., Chan, W., Zhang, Y., Chiu, C.C., Zoph, B., Cubuk, E.D. \n        and Le, Q.V., 2019. Specaugment: A simple data augmentation method \n        for automatic speech recognition. arXiv preprint arXiv:1904.08779.\n        Args:\n          time_drop_width: int\n          time_stripes_num: int\n          freq_drop_width: int\n          freq_stripes_num: int\n        \"\"\"\n\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.010426,"end_time":"2020-09-10T12:17:09.809175","exception":false,"start_time":"2020-09-10T12:17:09.798749","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### audioset_tagging_cnn\n\nI also use `Cnn14_DecisionLevelAtt` model from [PANNs models](https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/models.py), which is a SED model."},{"metadata":{"papermill":{"duration":0.010391,"end_time":"2020-09-10T12:17:09.830069","exception":false,"start_time":"2020-09-10T12:17:09.819678","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### LICENSE\n\n```\nThe MIT License\n  \nCopyright (c) 2010-2017 Google, Inc. http://angularjs.org\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n```"},{"metadata":{"papermill":{"duration":0.010357,"end_time":"2020-09-10T12:17:09.850758","exception":false,"start_time":"2020-09-10T12:17:09.840401","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Building blocks"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:09.895898Z","iopub.status.busy":"2020-09-10T12:17:09.890731Z","iopub.status.idle":"2020-09-10T12:17:09.90535Z","shell.execute_reply":"2020-09-10T12:17:09.904926Z"},"papermill":{"duration":0.044395,"end_time":"2020-09-10T12:17:09.905464","exception":false,"start_time":"2020-09-10T12:17:09.861069","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:09.9449Z","iopub.status.busy":"2020-09-10T12:17:09.934401Z","iopub.status.idle":"2020-09-10T12:17:09.961025Z","shell.execute_reply":"2020-09-10T12:17:09.960566Z"},"papermill":{"duration":0.045122,"end_time":"2020-09-10T12:17:09.961118","exception":false,"start_time":"2020-09-10T12:17:09.915996","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class PANNsCNN14Att(nn.Module):\n    def __init__(self, sample_rate: int, window_size: int, hop_size: int,\n                 mel_bins: int, fmin: int, fmax: int, classes_num: int):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(\n            n_fft=window_size,\n            hop_length=hop_size,\n            win_length=window_size,\n            window=window,\n            center=center,\n            pad_mode=pad_mode,\n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(\n            sr=sample_rate,\n            n_fft=window_size,\n            n_mels=mel_bins,\n            fmin=fmin,\n            fmax=fmax,\n            ref=ref,\n            amin=amin,\n            top_db=top_db,\n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2,\n            freq_drop_width=8,\n            freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def cnn_feature_extractor(self, x):\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        return x\n    \n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n        \n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n\n        # Output shape (batch size, channels, time, frequency)\n        x = self.cnn_feature_extractor(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.010507,"end_time":"2020-09-10T12:17:09.982335","exception":false,"start_time":"2020-09-10T12:17:09.971828","status":"completed"},"tags":[]},"cell_type":"markdown","source":"What is good in PANNs models is that they accept raw audio clip as input. Let's put a chunk into the CNN feature extractor of the model above."},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:10.008962Z","iopub.status.busy":"2020-09-10T12:17:10.008302Z","iopub.status.idle":"2020-09-10T12:17:10.020793Z","shell.execute_reply":"2020-09-10T12:17:10.020271Z"},"papermill":{"duration":0.028131,"end_time":"2020-09-10T12:17:10.020907","exception":false,"start_time":"2020-09-10T12:17:09.992776","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"SR = 32000\n\ny, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"aldfly\" / \"XC134874.wav\",\n                    sr=SR,\n                    res_type=\"kaiser_fast\",\n                    mono=True)\n\n# Audio(y, rate=SR)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:10.047527Z","iopub.status.busy":"2020-09-10T12:17:10.046575Z","iopub.status.idle":"2020-09-10T12:17:10.239629Z","shell.execute_reply":"2020-09-10T12:17:10.23918Z"},"papermill":{"duration":0.208005,"end_time":"2020-09-10T12:17:10.239731","exception":false,"start_time":"2020-09-10T12:17:10.031726","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"display.waveplot(y, sr=SR)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:10.269749Z","iopub.status.busy":"2020-09-10T12:17:10.268685Z","iopub.status.idle":"2020-09-10T12:17:11.586266Z","shell.execute_reply":"2020-09-10T12:17:11.585792Z"},"papermill":{"duration":1.334593,"end_time":"2020-09-10T12:17:11.586374","exception":false,"start_time":"2020-09-10T12:17:10.251781","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# model_config = {\n#     \"sample_rate\": 32000,\n#     \"window_size\": 1024,\n#     \"hop_size\": 256,\n#     \"mel_bins\": 128,\n#     \"fmin\": 50,\n#     \"fmax\": 14000,\n#     \"classes_num\": 264\n# }\n\nmodel_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 20,\n    \"fmax\": 16000,\n    \"classes_num\": 264\n}\n\nmodel = PANNsCNN14Att(**model_config)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.010888,"end_time":"2020-09-10T12:17:11.608898","exception":false,"start_time":"2020-09-10T12:17:11.59801","status":"completed"},"tags":[]},"cell_type":"markdown","source":"In `PANNsCNN14Att`, input raw waveform will be converted into log-melspectrogram using `torchlibrosa`'s utilities. I put this functionality in `PANNsCNN14Att.preprocess()` method. Let's check the output."},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:11.637322Z","iopub.status.busy":"2020-09-10T12:17:11.636734Z","iopub.status.idle":"2020-09-10T12:17:11.744687Z","shell.execute_reply":"2020-09-10T12:17:11.745131Z"},"papermill":{"duration":0.1255,"end_time":"2020-09-10T12:17:11.745255","exception":false,"start_time":"2020-09-10T12:17:11.619755","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"chunk = torch.from_numpy(y[:SR * 5]).unsqueeze(0)\nmelspec, _ = model.preprocess(chunk)\nmelspec.size()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:11.772532Z","iopub.status.busy":"2020-09-10T12:17:11.77199Z","iopub.status.idle":"2020-09-10T12:17:11.876442Z","shell.execute_reply":"2020-09-10T12:17:11.876847Z"},"papermill":{"duration":0.120534,"end_time":"2020-09-10T12:17:11.876975","exception":false,"start_time":"2020-09-10T12:17:11.756441","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"melspec_numpy = melspec.detach().numpy()[0, 0].transpose(1, 0)\ndisplay.specshow(melspec_numpy, sr=SR, y_axis=\"mel\");","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011348,"end_time":"2020-09-10T12:17:11.900348","exception":false,"start_time":"2020-09-10T12:17:11.889","status":"completed"},"tags":[]},"cell_type":"markdown","source":"`PANNsCNN14Att.cnn_feature_extractor()` method will take this as input and output feature map. Let's check the output of the feature extractor."},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:11.927991Z","iopub.status.busy":"2020-09-10T12:17:11.927482Z","iopub.status.idle":"2020-09-10T12:17:12.490079Z","shell.execute_reply":"2020-09-10T12:17:12.489652Z"},"papermill":{"duration":0.578184,"end_time":"2020-09-10T12:17:12.49018","exception":false,"start_time":"2020-09-10T12:17:11.911996","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"feature_map = model.cnn_feature_extractor(melspec)\nfeature_map.size()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011566,"end_time":"2020-09-10T12:17:12.514223","exception":false,"start_time":"2020-09-10T12:17:12.502657","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Although it's downsized through several convolution and pooling layers, the size of it's third dimension is 15 and it still contains time information. Each element of this dimension is *segment*. In SED model, we provide prediction for each of this."},{"metadata":{"papermill":{"duration":0.011537,"end_time":"2020-09-10T12:17:12.537558","exception":false,"start_time":"2020-09-10T12:17:12.526021","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Now, let's try to train this model in weakly-supervised manner."},{"metadata":{"papermill":{"duration":0.011535,"end_time":"2020-09-10T12:17:12.561008","exception":false,"start_time":"2020-09-10T12:17:12.549473","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:12.596532Z","iopub.status.busy":"2020-09-10T12:17:12.595619Z","iopub.status.idle":"2020-09-10T12:17:12.647519Z","shell.execute_reply":"2020-09-10T12:17:12.646803Z"},"papermill":{"duration":0.074623,"end_time":"2020-09-10T12:17:12.647627","exception":false,"start_time":"2020-09-10T12:17:12.573004","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:12.687747Z","iopub.status.busy":"2020-09-10T12:17:12.685883Z","iopub.status.idle":"2020-09-10T12:17:12.688471Z","shell.execute_reply":"2020-09-10T12:17:12.688987Z"},"papermill":{"duration":0.028849,"end_time":"2020-09-10T12:17:12.689115","exception":false,"start_time":"2020-09-10T12:17:12.660266","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"PERIOD = 10\n\nclass PANNsDataset(data.Dataset):\n    def __init__(\n            self,\n            file_list: List[List[str]],\n            waveform_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.waveform_transforms = waveform_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        for b_code in ebird_code:\n            labels[BIRD_CODE[b_code]] = 1\n\n        return {\"waveform\": y, \"targets\": labels}","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.023032,"end_time":"2020-09-10T12:17:12.737413","exception":false,"start_time":"2020-09-10T12:17:12.714381","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Criterion"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:12.772515Z","iopub.status.busy":"2020-09-10T12:17:12.771598Z","iopub.status.idle":"2020-09-10T12:17:12.774514Z","shell.execute_reply":"2020-09-10T12:17:12.774085Z"},"papermill":{"duration":0.023178,"end_time":"2020-09-10T12:17:12.774616","exception":false,"start_time":"2020-09-10T12:17:12.751438","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                     torch.zeros_like(input_),\n                     input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        target = target.float()\n\n        input_ = torch.clamp(input_, 0.0, 1.0)\n\n        return self.bce(input_, target)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.012396,"end_time":"2020-09-10T12:17:12.799714","exception":false,"start_time":"2020-09-10T12:17:12.787318","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Callbacks"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:12.870328Z","iopub.status.busy":"2020-09-10T12:17:12.869661Z","iopub.status.idle":"2020-09-10T12:17:12.872984Z","shell.execute_reply":"2020-09-10T12:17:12.873395Z"},"papermill":{"duration":0.060785,"end_time":"2020-09-10T12:17:12.873539","exception":false,"start_time":"2020-09-10T12:17:12.812754","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class F1Callback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"f1\"):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output.argmax(axis=1)\n        y_true = targ.argmax(axis=1)\n\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0).argmax(axis=1)\n        y_true = np.concatenate(self.target, axis=0).argmax(axis=1)\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        score = average_precision_score(targ, clipwise_output, average=None)\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.012741,"end_time":"2020-09-10T12:17:12.899518","exception":false,"start_time":"2020-09-10T12:17:12.886777","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Train\n\nSome code are taken from https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast .\nThanks @ttahara!"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:12.934154Z","iopub.status.busy":"2020-09-10T12:17:12.933402Z","iopub.status.idle":"2020-09-10T12:17:13.692895Z","shell.execute_reply":"2020-09-10T12:17:13.692094Z"},"papermill":{"duration":0.780719,"end_time":"2020-09-10T12:17:13.693047","exception":false,"start_time":"2020-09-10T12:17:12.912328","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:13.783405Z","iopub.status.busy":"2020-09-10T12:17:13.782579Z","iopub.status.idle":"2020-09-10T12:17:13.958177Z","shell.execute_reply":"2020-09-10T12:17:13.959034Z"},"papermill":{"duration":0.244945,"end_time":"2020-09-10T12:17:13.959249","exception":false,"start_time":"2020-09-10T12:17:13.714304","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:14.012341Z","iopub.status.busy":"2020-09-10T12:17:14.010716Z","iopub.status.idle":"2020-09-10T12:17:14.014607Z","shell.execute_reply":"2020-09-10T12:17:14.011525Z"},"papermill":{"duration":0.036195,"end_time":"2020-09-10T12:17:14.014757","exception":false,"start_time":"2020-09-10T12:17:13.978562","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"code_map =dict(zip(train_all['primary_label'].tolist(),train_all['ebird_code'].tolist()))\nlen(code_map)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:14.069535Z","iopub.status.busy":"2020-09-10T12:17:14.068647Z","iopub.status.idle":"2020-09-10T12:17:14.344623Z","shell.execute_reply":"2020-09-10T12:17:14.345636Z"},"papermill":{"duration":0.310696,"end_time":"2020-09-10T12:17:14.345808","exception":false,"start_time":"2020-09-10T12:17:14.035112","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_all['sec_code'] = train_all.secondary_labels.apply(lambda x : [code_map[i] for i in eval(x)  if i in code_map.keys()])\ntrain_all[train_all.sec_code.apply(len)>0].shape","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:14.404431Z","iopub.status.busy":"2020-09-10T12:17:14.403434Z","iopub.status.idle":"2020-09-10T12:17:15.517311Z","shell.execute_reply":"2020-09-10T12:17:15.516879Z"},"papermill":{"duration":1.15057,"end_time":"2020-09-10T12:17:15.51741","exception":false,"start_time":"2020-09-10T12:17:14.36684","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_all[\"final_bird\"] = train_all.apply(lambda x : [x['ebird_code']] + x['sec_code'] ,axis=1 )\ntrain_all[['ebird_code','final_bird','sec_code']]\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:15.552488Z","iopub.status.busy":"2020-09-10T12:17:15.551926Z","iopub.status.idle":"2020-09-10T12:17:15.596201Z","shell.execute_reply":"2020-09-10T12:17:15.596646Z"},"papermill":{"duration":0.065955,"end_time":"2020-09-10T12:17:15.596776","exception":false,"start_time":"2020-09-10T12:17:15.530821","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"use_fold = 3\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"final_bird\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"final_bird\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:15.636183Z","iopub.status.busy":"2020-09-10T12:17:15.635033Z","iopub.status.idle":"2020-09-10T12:17:24.674335Z","shell.execute_reply":"2020-09-10T12:17:24.673025Z"},"papermill":{"duration":9.064647,"end_time":"2020-09-10T12:17:24.674502","exception":false,"start_time":"2020-09-10T12:17:15.609855","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n\n# loaders\nloaders = {\n    \"train\": data.DataLoader(PANNsDataset(train_file_list, None), \n                             batch_size=40, \n                             shuffle=True, \n                             num_workers=2, \n                             pin_memory=True, \n                             drop_last=True),\n    \"valid\": data.DataLoader(PANNsDataset(val_file_list, None), \n                             batch_size=64, \n                             shuffle=False,\n                             num_workers=2,\n                             pin_memory=True,\n                             drop_last=False)\n}\n\n# model\nmodel_config[\"classes_num\"] = 527\nmodel = PANNsCNN14Att(**model_config)\nweights = torch.load(\"../input/pannscnn14-decisionlevelatt-weight/Cnn14_DecisionLevelAtt_mAP0.425.pth\")\n# Fixed in V3\nmodel.load_state_dict(weights[\"model\"])\nmodel.att_block = AttBlock(2048, 264, activation='sigmoid')\nmodel.att_block.init_weights()\nmodel.to(device)\n\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Scheduler\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n\n# Loss\ncriterion = PANNsLoss().to(device)\n\n# callbacks\ncallbacks = [\n    F1Callback(input_key=\"targets\", output_key=\"logits\", prefix=\"f1\"),\n    mAPCallback(input_key=\"targets\", output_key=\"logits\", prefix=\"mAP\"),\n    CheckpointCallback(save_n_best=0)\n]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:24.708742Z","iopub.status.busy":"2020-09-10T12:17:24.707271Z","iopub.status.idle":"2020-09-10T12:17:24.711292Z","shell.execute_reply":"2020-09-10T12:17:24.710849Z"},"papermill":{"duration":0.023276,"end_time":"2020-09-10T12:17:24.711389","exception":false,"start_time":"2020-09-10T12:17:24.688113","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model_config","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-09-10T12:17:24.743897Z","iopub.status.busy":"2020-09-10T12:17:24.742495Z","iopub.status.idle":"2020-09-10T12:17:24.744924Z","shell.execute_reply":"2020-09-10T12:17:24.745375Z"},"papermill":{"duration":0.020597,"end_time":"2020-09-10T12:17:24.745501","exception":false,"start_time":"2020-09-10T12:17:24.724904","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# warnings.simplefilter(\"ignore\")\n\n# runner = SupervisedRunner(\n#     device=device,\n#     input_key=\"waveform\",\n#     input_target_key=\"targets\")\n\n# runner.train(\n#     model=model,\n#     criterion=criterion,\n#     loaders=loaders,\n#     optimizer=optimizer,\n#     scheduler=scheduler,\n#     num_epochs=50,\n#     verbose=True,\n#     logdir=f\"fold0\",\n#     callbacks=callbacks,\n#     main_metric=\"epoch_f1\",\n#     minimize_metric=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:24.777425Z","iopub.status.busy":"2020-09-10T12:17:24.775749Z","iopub.status.idle":"2020-09-10T12:17:24.778202Z","shell.execute_reply":"2020-09-10T12:17:24.778656Z"},"papermill":{"duration":0.019868,"end_time":"2020-09-10T12:17:24.778768","exception":false,"start_time":"2020-09-10T12:17:24.7589","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# torch.save(model,\"pan.pth\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:24.809673Z","iopub.status.busy":"2020-09-10T12:17:24.809063Z","iopub.status.idle":"2020-09-10T12:17:24.813194Z","shell.execute_reply":"2020-09-10T12:17:24.812775Z"},"papermill":{"duration":0.021291,"end_time":"2020-09-10T12:17:24.813292","exception":false,"start_time":"2020-09-10T12:17:24.792001","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# model.load_state_dict(torch.load(\"pan.pth\").state_dict())","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.013805,"end_time":"2020-09-10T12:17:24.840929","exception":false,"start_time":"2020-09-10T12:17:24.827124","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Seems it's learning something.\n\nNow I'll show how this model works in the inference phase. I'll use trained model of this which I trained by myself using the data of this competition in my local environment.\n\nSince [several concerns](https://www.kaggle.com/c/birdsong-recognition/discussion/172356) are expressed about over-sharing of top solutions during competition, and since I do respect those people who have worked hard to improve their scores, I would not make trained weight in common and would not share how I trained this model."},{"metadata":{"papermill":{"duration":0.013223,"end_time":"2020-09-10T12:17:24.910864","exception":false,"start_time":"2020-09-10T12:17:24.897641","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Prediction with SED model"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:24.943928Z","iopub.status.busy":"2020-09-10T12:17:24.943191Z","iopub.status.idle":"2020-09-10T12:17:24.947203Z","shell.execute_reply":"2020-09-10T12:17:24.946761Z"},"papermill":{"duration":0.022497,"end_time":"2020-09-10T12:17:24.947298","exception":false,"start_time":"2020-09-10T12:17:24.924801","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 20,\n    \"fmax\": 16000,\n    \"classes_num\": 264\n}\n\nweights_path = \"../input/pan-model-new/fold1/checkpoints/best.pth\"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:24.981563Z","iopub.status.busy":"2020-09-10T12:17:24.979847Z","iopub.status.idle":"2020-09-10T12:17:24.982201Z","shell.execute_reply":"2020-09-10T12:17:24.982626Z"},"papermill":{"duration":0.021832,"end_time":"2020-09-10T12:17:24.982741","exception":false,"start_time":"2020-09-10T12:17:24.960909","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_model(config: dict, weights_path: str):\n    model = PANNsCNN14Att(**config)\n    checkpoint = torch.load(weights_path)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    device = torch.device(\"cuda\")\n    model.to(device)\n    model.eval()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model  = get_model(model_config, \"../input/pan-model/pan_model/best.pth\")\nmodel_2 = get_model(model_config, \"../input/pan-model/pan_model/fold2.pth\")\nmodel_4 = get_model(model_config, \"../input/pan-model-new/fold4.pth\")\nmodel_5 = get_model(model_config, \"../input/pan-model/pan_model/fold0.pth\")\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:25.01741Z","iopub.status.busy":"2020-09-10T12:17:25.012017Z","iopub.status.idle":"2020-09-10T12:17:25.03339Z","shell.execute_reply":"2020-09-10T12:17:25.033856Z"},"papermill":{"duration":0.037845,"end_time":"2020-09-10T12:17:25.033992","exception":false,"start_time":"2020-09-10T12:17:24.996147","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame,\n                        clip: np.ndarray, \n                        model: PANNsCNN14Att,\n                        threshold=0.5):\n    PERIOD = 10\n    audios = []\n    y = clip.astype(np.float32)\n    len_y = len(y)\n    start = 0\n    end = PERIOD * SR\n    while True:\n        y_batch = y[start:end].astype(np.float32)\n        if len(y_batch) != PERIOD * SR:\n            y_pad = np.zeros(PERIOD * SR, dtype=np.float32)\n            y_pad[:len(y_batch)] = y_batch\n            audios.append(y_pad)\n            break\n        start = end\n        end += PERIOD * SR\n        audios.append(y_batch)\n        \n    array = np.asarray(audios)\n    tensors = torch.from_numpy(array)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    estimated_event_list = []\n    global_time = 0.0\n    site = test_df[\"site\"].values[0]\n    audio_id = test_df[\"audio_id\"].values[0]\n    for image in progress_bar(tensors):\n        image = image.view(1, image.size(0))\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)[\"framewise_output\"].detach().cpu().numpy()[0]\n            prediction_2 = best_model(image)[\"framewise_output\"].detach().cpu().numpy()[0]\n            prediction_3 = model_2(image)[\"framewise_output\"].detach().cpu().numpy()[0]\n            prediction_4 = model_4(image)[\"framewise_output\"].detach().cpu().numpy()[0]\n            prediction_5 = model_5(image)[\"framewise_output\"].detach().cpu().numpy()[0]\n            \n            framewise_outputs = (prediction + prediction_2 + prediction_3 +prediction_4 +prediction_5 )/5\n            \n\n        thresholded = framewise_outputs >= threshold\n\n        for target_idx in range(thresholded.shape[1]):\n            if thresholded[:, target_idx].mean() == 0:\n                pass\n            else:\n                detected = np.argwhere(thresholded[:, target_idx]).reshape(-1)\n                head_idx = 0\n                tail_idx = 0\n                while True:\n                    if (tail_idx + 1 == len(detected)) or (\n                            detected[tail_idx + 1] - \n                            detected[tail_idx] != 1):\n                        onset = 0.01 * detected[\n                            head_idx] + global_time\n                        offset = 0.01 * detected[\n                            tail_idx] + global_time\n                        onset_idx = detected[head_idx]\n                        offset_idx = detected[tail_idx]\n                        max_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].max()\n                        mean_confidence = framewise_outputs[\n                            onset_idx:offset_idx, target_idx].mean()\n                        estimated_event = {\n                            \"site\": site,\n                            \"audio_id\": audio_id,\n                            \"ebird_code\": INV_BIRD_CODE[target_idx],\n                            \"onset\": onset,\n                            \"offset\": offset,\n                            \"max_confidence\": max_confidence,\n                            \"mean_confidence\": mean_confidence\n                        }\n                        estimated_event_list.append(estimated_event)\n                        head_idx = tail_idx + 1\n                        tail_idx = tail_idx + 1\n                        if head_idx >= len(detected):\n                            break\n                    else:\n                        tail_idx += 1\n        global_time += PERIOD\n        \n    prediction_df = pd.DataFrame(estimated_event_list)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:25.070992Z","iopub.status.busy":"2020-09-10T12:17:25.070281Z","iopub.status.idle":"2020-09-10T12:17:25.074067Z","shell.execute_reply":"2020-09-10T12:17:25.073497Z"},"papermill":{"duration":0.026292,"end_time":"2020-09-10T12:17:25.074171","exception":false,"start_time":"2020-09-10T12:17:25.047879","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               model_config: dict,\n               weights_path: str,\n               threshold=0.5):\n    model = get_model(model_config, weights_path)\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=SR,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_df = prediction_for_clip(test_df_for_audio_id,\n                                                clip=clip,\n                                                model=model,\n                                                threshold=threshold)\n\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:25.105669Z","iopub.status.busy":"2020-09-10T12:17:25.105104Z","iopub.status.idle":"2020-09-10T12:17:43.680307Z","shell.execute_reply":"2020-09-10T12:17:43.679766Z"},"papermill":{"duration":18.592832,"end_time":"2020-09-10T12:17:43.68041","exception":false,"start_time":"2020-09-10T12:17:25.087578","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"prediction_df = prediction(test_df=test,\n                           test_audio=TEST_AUDIO_DIR,\n                           model_config=model_config,\n                           weights_path=weights_path,\n                           threshold=0.5)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:43.732106Z","iopub.status.busy":"2020-09-10T12:17:43.73112Z","iopub.status.idle":"2020-09-10T12:17:43.735596Z","shell.execute_reply":"2020-09-10T12:17:43.735107Z"},"papermill":{"duration":0.038243,"end_time":"2020-09-10T12:17:43.735705","exception":false,"start_time":"2020-09-10T12:17:43.697462","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"prediction_df","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.016738,"end_time":"2020-09-10T12:17:43.769842","exception":false,"start_time":"2020-09-10T12:17:43.753104","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Postprocess\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:43.854517Z","iopub.status.busy":"2020-09-10T12:17:43.844438Z","iopub.status.idle":"2020-09-10T12:17:43.871254Z","shell.execute_reply":"2020-09-10T12:17:43.870785Z"},"papermill":{"duration":0.084524,"end_time":"2020-09-10T12:17:43.871364","exception":false,"start_time":"2020-09-10T12:17:43.78684","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"labels = {}\n\nfor audio_id, sub_df in prediction_df.groupby(\"audio_id\"):\n    events = sub_df[[\"ebird_code\", \"onset\", \"offset\", \"max_confidence\", \"site\"]].values\n    n_events = len(events)\n    removed_event = []\n    # Overlap deletion: this part may not be necessary\n    # I deleted this part in other model and found there's no difference on the public LB score.\n    for i in range(n_events):\n        for j in range(n_events):\n            if i == j:\n                continue\n            if i in removed_event:\n                continue\n            if j in removed_event:\n                continue\n            \n            event_i = events[i]\n            event_j = events[j]\n            \n            if (event_i[1] - event_j[2] >= 0) or (event_j[1] - event_i[2] >= 0):\n                pass\n            else:\n                later_onset = max(event_i[1], event_j[1])\n                sooner_onset = min(event_i[1], event_j[1])\n                sooner_offset = min(event_i[2], event_j[2])\n                later_offset = max(event_i[2], event_j[2])\n\n                intersection = sooner_offset - later_onset\n                union = later_offset - sooner_onset\n                \n                iou = intersection / union\n                if iou > 0.4:\n                    if event_i[3] > event_j[3]:\n                        removed_event.append(j)\n                    else:\n                        removed_event.append(i)\n\n    site = events[0][4]\n    for i in range(n_events):\n        if i in removed_event:\n            continue\n        event = events[i][0]\n        onset = events[i][1]\n        offset = events[i][2]\n        if site in {\"site_1\", \"site_2\"}:\n            start_section = int((onset // 5) * 5) + 5\n            end_section = int((offset // 5) * 5) + 5\n            cur_section = start_section\n\n            row_id = f\"{site}_{audio_id}_{start_section}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)\n\n            while cur_section != end_section:\n                cur_section += 5\n                row_id = f\"{site}_{audio_id}_{cur_section}\"\n                if labels.get(row_id) is not None:\n                    labels[row_id].add(event)\n                else:\n                    labels[row_id] = set()\n                    labels[row_id].add(event)\n        else:\n            row_id = f\"{site}_{audio_id}\"\n            if labels.get(row_id) is not None:\n                labels[row_id].add(event)\n            else:\n                labels[row_id] = set()\n                labels[row_id].add(event)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:43.918444Z","iopub.status.busy":"2020-09-10T12:17:43.917683Z","iopub.status.idle":"2020-09-10T12:17:43.920831Z","shell.execute_reply":"2020-09-10T12:17:43.92125Z"},"papermill":{"duration":0.032475,"end_time":"2020-09-10T12:17:43.921368","exception":false,"start_time":"2020-09-10T12:17:43.888893","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for key in labels:\n    labels[key] = \" \".join(sorted(list(labels[key])))\n    \n    \nrow_ids = list(labels.keys())\nbirds = list(labels.values())\npost_processed = pd.DataFrame({\n    \"row_id\": row_ids,\n    \"birds\": birds\n})\npost_processed.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:43.971631Z","iopub.status.busy":"2020-09-10T12:17:43.970937Z","iopub.status.idle":"2020-09-10T12:17:44.036662Z","shell.execute_reply":"2020-09-10T12:17:44.036146Z"},"papermill":{"duration":0.097845,"end_time":"2020-09-10T12:17:44.036767","exception":false,"start_time":"2020-09-10T12:17:43.938922","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"all_row_id = test[[\"row_id\"]]\nsubmission = all_row_id.merge(post_processed, on=\"row_id\", how=\"left\")\nsubmission = submission.fillna(\"nocall\")\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(20)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-10T12:17:44.080698Z","iopub.status.busy":"2020-09-10T12:17:44.079935Z","iopub.status.idle":"2020-09-10T12:17:44.083611Z","shell.execute_reply":"2020-09-10T12:17:44.083179Z"},"papermill":{"duration":0.028501,"end_time":"2020-09-10T12:17:44.08371","exception":false,"start_time":"2020-09-10T12:17:44.055209","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission.birds.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017459,"end_time":"2020-09-10T12:17:44.118642","exception":false,"start_time":"2020-09-10T12:17:44.101183","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## EOF"},{"metadata":{"papermill":{"duration":0.017463,"end_time":"2020-09-10T12:17:44.153934","exception":false,"start_time":"2020-09-10T12:17:44.136471","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}