{"cells":[{"metadata":{},"cell_type":"markdown","source":"##Thanks @hidehisaarai1213 for this notebooks: https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Updated:+ Use extracter_layer from EFFNET B0\n        + add augmetation\n        + Write new Trainer\n        + Add Mixup\n        + fp16 option\n        + GRU layer\n        + Gradient accumulation steps","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#  !pip install --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/apexpytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import library","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n# from apex import amp\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\nfrom fastprogress import progress_bar\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, average_precision_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### torchlibrosa\n\n\nIn PANNs, `torchlibrosa`, a PyTorch based implementation are used to replace some of the `librosa`'s functions. Here I use some functions of `torchlibrosa`.\n\nRef: https://github.com/qiuqiangkong/torchlibrosa","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DFTBase(nn.Module):\n    def __init__(self):\n        \"\"\"Base class for DFT and IDFT matrix\"\"\"\n        super(DFTBase, self).__init__()\n\n    def dft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(-2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n\n    def idft_matrix(self, n):\n        (x, y) = np.meshgrid(np.arange(n), np.arange(n))\n        omega = np.exp(2 * np.pi * 1j / n)\n        W = np.power(omega, x * y)\n        return W\n    \n    \nclass STFT(DFTBase):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', freeze_parameters=True):\n        \"\"\"Implementation of STFT with Conv1d. The function has the same output \n        of librosa.core.stft\n        \"\"\"\n        super(STFT, self).__init__()\n\n        assert pad_mode in ['constant', 'reflect']\n\n        self.n_fft = n_fft\n        self.center = center\n        self.pad_mode = pad_mode\n\n        # By default, use the entire frame\n        if win_length is None:\n            win_length = n_fft\n\n        # Set the default hop, if it's not already specified\n        if hop_length is None:\n            hop_length = int(win_length // 4)\n\n        fft_window = librosa.filters.get_window(window, win_length, fftbins=True)\n\n        # Pad the window out to n_fft size\n        fft_window = librosa.util.pad_center(fft_window, n_fft)\n\n        # DFT & IDFT matrix\n        self.W = self.dft_matrix(n_fft)\n\n        out_channels = n_fft // 2 + 1\n\n        self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels, \n            kernel_size=n_fft, stride=hop_length, padding=0, dilation=1, \n            groups=1, bias=False)\n\n        self.conv_real.weight.data = torch.Tensor(\n            np.real(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        self.conv_imag.weight.data = torch.Tensor(\n            np.imag(self.W[:, 0 : out_channels] * fft_window[:, None]).T)[:, None, :]\n        # (n_fft // 2 + 1, 1, n_fft)\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        x = input[:, None, :]   # (batch_size, channels_num, data_length)\n\n        if self.center:\n            x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)\n\n        real = self.conv_real(x)\n        imag = self.conv_imag(x)\n        real = real[:, None, :, :].transpose(2, 3)\n        imag = imag[:, None, :, :].transpose(2, 3)\n        return real, imag\n    \n    \nclass Spectrogram(nn.Module):\n    def __init__(self, n_fft=2048, hop_length=None, win_length=None, \n        window='hann', center=True, pad_mode='reflect', power=2.0, \n        freeze_parameters=True):\n        super(Spectrogram, self).__init__()\n\n        self.power = power\n\n        self.stft = STFT(n_fft=n_fft, hop_length=hop_length, \n            win_length=win_length, window=window, center=center, \n            pad_mode=pad_mode, freeze_parameters=True)\n\n    def forward(self, input):\n\n        (real, imag) = self.stft.forward(input)\n        # (batch_size, n_fft // 2 + 1, time_steps)\n\n        spectrogram = real ** 2 + imag ** 2\n\n        if self.power == 2.0:\n            pass\n        else:\n            spectrogram = spectrogram ** (power / 2.0)\n\n        return spectrogram\n\n    \nclass LogmelFilterBank(nn.Module):\n    def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True, \n        ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):\n        \"\"\"Calculate logmel spectrogram using pytorch. The mel filter bank is \n        the pytorch implementation of as librosa.filters.mel \n        \"\"\"\n        super(LogmelFilterBank, self).__init__()\n\n        self.is_log = is_log\n        self.ref = ref\n        self.amin = amin\n        self.top_db = top_db\n\n        self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n            fmin=fmin, fmax=fmax).T\n        # (n_fft // 2 + 1, mel_bins)\n\n        self.melW = nn.Parameter(torch.Tensor(self.melW))\n\n        if freeze_parameters:\n            for param in self.parameters():\n                param.requires_grad = False\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps)\n        \n        Output: (batch_size, time_steps, mel_bins)\n        \"\"\"\n\n        # Mel spectrogram\n        mel_spectrogram = torch.matmul(input, self.melW)\n\n        # Logmel spectrogram\n        if self.is_log:\n            output = self.power_to_db(mel_spectrogram)\n        else:\n            output = mel_spectrogram\n\n        return output\n\n\n    def power_to_db(self, input):\n        ref_value = self.ref\n        log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))\n        log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))\n\n        if self.top_db is not None:\n            if self.top_db < 0:\n                raise ParameterError('top_db must be non-negative')\n            log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)\n\n        return log_spec","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DropStripes(nn.Module):\n    def __init__(self, dim, drop_width, stripes_num):\n        super(DropStripes, self).__init__()\n\n        assert dim in [2, 3]    # dim 2: time; dim 3: frequency\n\n        self.dim = dim\n        self.drop_width = drop_width\n        self.stripes_num = stripes_num\n\n    def forward(self, input):\n        \"\"\"input: (batch_size, channels, time_steps, freq_bins)\"\"\"\n\n        assert input.ndimension() == 4\n\n        if self.training is False:\n            return input\n\n        else:\n            batch_size = input.shape[0]\n            total_width = input.shape[self.dim]\n\n            for n in range(batch_size):\n                self.transform_slice(input[n], total_width)\n\n            return input\n\n\n    def transform_slice(self, e, total_width):\n        \"\"\"e: (channels, time_steps, freq_bins)\"\"\"\n\n        for _ in range(self.stripes_num):\n            distance = torch.randint(low=0, high=self.drop_width, size=(1,))[0]\n            bgn = torch.randint(low=0, high=total_width - distance, size=(1,))[0]\n\n            if self.dim == 2:\n                e[:, bgn : bgn + distance, :] = 0\n            elif self.dim == 3:\n                e[:, :, bgn : bgn + distance] = 0\n\n\nclass SpecAugmentation(nn.Module):\n    def __init__(self, time_drop_width, time_stripes_num, freq_drop_width, freq_stripes_num):\n        super(SpecAugmentation, self).__init__()\n\n        self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width, \n            stripes_num=time_stripes_num)\n\n        self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width, \n            stripes_num=freq_stripes_num)\n\n    def forward(self, input):\n        x = self.time_dropper(input)\n        x = self.freq_dropper(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Building blocks","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"## Code copied from Lukemelas github repository have a look\n## https://github.com/lukemelas/EfficientNet-PyTorch/tree/master/efficientnet_pytorch\n\"\"\"\nThis file contains helper functions for building the model and for loading model parameters.\nThese helper functions are built to mirror those in the official TensorFlow implementation.\n\"\"\"\n\nimport re\nimport math\nimport collections\nfrom functools import partial\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils import model_zoo\n\n########################################################################\n############### HELPERS FUNCTIONS FOR MODEL ARCHITECTURE ###############\n########################################################################\n\n\n# Parameters for the entire model (stem, all blocks, and head)\nGlobalParams = collections.namedtuple('GlobalParams', [\n    'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',\n    'num_classes', 'width_coefficient', 'depth_coefficient',\n    'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',\n    'expand_ratio', 'id_skip', 'stride', 'se_ratio'])\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_tensors[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training: return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self, ):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n\n\n########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=[int(options['s'][0])])\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,\n                 drop_connect_rate=0.2, image_size=None, num_classes=1000):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b1-f1951068.pth',\n    'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b2-8bb594d6.pth',\n    'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b3-5fb5a3c3.pth',\n    'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b4-6ed6700e.pth',\n    'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b5-b6417697.pth',\n    'efficientnet-b6': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b6-c76e70fd.pth',\n    'efficientnet-b7': 'http://storage.googleapis.com/public-models/efficientnet/efficientnet-b7-dcc49843.pth',\n}\n\n## This below function is modified to use the pretrained weight for single channel . Its nothing but summing the weight across one axis .\ndef load_pretrained_weights(model, model_name, load_fc=True,ch=1):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    state_dict = torch.load('../input/efficientnet-pytorch/efficientnet-b0-08094119.pth')\n    if load_fc:\n        if ch == 1:\n            conv1_weight = state_dict['_conv_stem.weight']\n            state_dict['_conv_stem.weight'] = conv1_weight.sum(dim=1, keepdim=True)\n        model.load_state_dict(state_dict)\n        \n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        if ch == 1:\n            conv1_weight = state_dict['_conv_stem.weight']\n            state_dict['_conv_stem.weight'] = conv1_weight.sum(dim=1, keepdim=True)\n        res = model.load_state_dict(state_dict, strict=False)\n        print(res.missing_keys)\n        assert set(res.missing_keys) == set(['_fc.weight', '_fc.bias','fc1.weight', 'fc1.bias','fc2.weight', 'fc2.bias','fc3.weight', 'fc3.bias']), 'issue loading pretrained weights'\n    print('Loaded pretrained weights for {}'.format(model_name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n\nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n        self._swish = MemoryEfficientSwish()\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = self._swish(self._bn0(self._expand_conv(inputs)))\n        x = self._swish(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Stem\n        in_channels = 1  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params))\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params))\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self._dropout = nn.Dropout(self._global_params.dropout_rate)\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n        # vowel_diacritic\n        self.fc1 = nn.Linear(out_channels,11)\n        # grapheme_root\n        self.fc2 = nn.Linear(out_channels,20)\n        # consonant_diacritic\n        self.fc3 = nn.Linear(out_channels,4)\n        self._swish = MemoryEfficientSwish()\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        for block in self._blocks:\n            block.set_swish(memory_efficient)\n\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        bs = inputs.size(0)\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n#         x = self._avg_pooling(x)\n#         x = x.view(bs, -1)\n#         x = self._dropout(x)\n        return x\n       # x = self._fc(x)\n#         x1 = self.fc1(x)\n#         x2= self.fc2(x)\n#         x3 = self.fc3(x)\n#         return x1,x2,x3\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return cls(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000, in_channels = 1):\n        model = cls.from_name(model_name, override_params={'num_classes': num_classes})\n        load_pretrained_weights(model, model_name, load_fc=False)\n        if in_channels != 3:\n            Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)\n            out_channels = round_filters(32, model._global_params)\n            model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        return model\n    \n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000):\n        model = cls.from_name(model_name, override_params={'num_classes': num_classes})\n        load_pretrained_weights(model, model_name, load_fc=False)\n\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):\n        \"\"\" Validates model name. None that pretrained weights are only available for\n        the first four models (efficientnet-b{i} for i in 0,1,2,3) at the moment. \"\"\"\n        num_models = 4 if also_need_pretrained_weights else 8\n        valid_models = ['efficientnet-b'+str(i) for i in range(num_models)]\n        if model_name not in valid_models:\n            raise ValueError('model_name should be one of: ' + ', '.join(valid_models))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EfficientNet.from_pretrained('efficientnet-b0')\nmodel.fc1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        elif pool_type == 'frac':\n            fractional_maxpool2d = nn.FractionalMaxPool2d(kernel_size=pool_size, output_ratio=1/np.sqrt(2))\n            x = fractional_maxpool2d(x)\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"sigmoid\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n        \n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return np.array(mixup_lambdas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANNsCNN14Att(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(PANNsCNN14Att, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.mixup_coff = Mixup(1.)\n        \n        self.interpolate_ratio = 32\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)  \n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=3)\n        self.effnet0 = EfficientNet.from_pretrained('efficientnet-b0')\n        self.gru = torch.nn.GRU(input_size=1280, hidden_size=640, \n                        num_layers=1, batch_first=True, bidirectional=True)\n        self.fc1 = nn.Linear(1280, 2048)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n        \n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n#         init_layer(self.fc_audioset)\n\n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n    \n    def forward(self, input, mixup_lambda=None):\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n        x = self.effnet0(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)        \n        x = torch.mean(x, dim=3)\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        (x, _) = self.gru(x)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, _, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n        framewise_output = interpolate(segmentwise_output, self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {'framewise_output': framewise_output, \n            'clipwise_output': clipwise_output}\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What is good in PANNs models is that they accept raw audio clip as input. Let's put a chunk into the CNN feature extractor of the model above.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SR = 32000\n\ny, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"aldfly\" / \"XC134874.wav\",\n                    sr=SR,\n                    res_type=\"kaiser_fast\",\n                    mono=True)\n\nAudio(y, rate=SR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display.waveplot(y, sr=SR);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 50,\n    \"fmax\": 14000,\n    \"classes_num\": 264\n}\n\nmodel = PANNsCNN14Att(**model_config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> In `PANNsCNN14Att`, input raw waveform will be converted into log-melspectrogram using `torchlibrosa`'s utilities. I put this functionality in `PANNsCNN14Att.preprocess()` method. Let's check the output.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"chunk = torch.from_numpy(y[:SR * 5]).unsqueeze(0)\nmelspec, fram_nums = model.preprocess(chunk)\nmelspec.size()\nmelspec_numpy = melspec.detach().numpy()[0, 0].transpose(1, 0)\ndisplay.specshow(melspec_numpy, sr=SR, y_axis=\"mel\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model(chunk)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Although it's downsized through several convolution and pooling layers, the size of it's third dimension is 15 and it still contains time information. Each element of this dimension is *segment*. In SED model, we provide prediction for each of this.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dataset","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\n\nfrom albumentations.core.transforms_interface import DualTransform, BasicTransform\n\nclass AudioTransform(BasicTransform):\n    \"\"\"Transform for Audio task\"\"\"\n\n    @property\n    def targets(self):\n        return {\"data\": self.apply}\n    \n    def update_params(self, params, **kwargs):\n        if hasattr(self, \"interpolation\"):\n            params[\"interpolation\"] = self.interpolation\n        if hasattr(self, \"fill_value\"):\n            params[\"fill_value\"] = self.fill_value\n        return params\n        \nclass RandomAudio(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self,  seconds=5, always_apply=False, p=0.5):\n        super(RandomAudio, self).__init__(always_apply, p)\n\n        self.seconds = seconds\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift = np.random.randint(len(sound))\n        trim_sound = np.roll(sound, shift)\n\n        min_samples = int(sr * self.seconds)\n\n        if len(trim_sound) < min_samples:\n            padding = min_samples - len(trim_sound)\n            offset = padding // 2\n            trim_sound = np.pad(trim_sound, (offset, padding - offset), \"constant\")\n        else:\n            trim_sound = trim_sound[:min_samples]\n\n        return trim_sound, sr\n    \n\n    \nclass AddGaussianNoise(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(AddGaussianNoise, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n        noise = 0.005*np.random.uniform()*np.amax(sound)\n        augmented_sound = np.array(sound).astype('float64') + noise * np.random.normal(size=sound.shape[0])\n        return augmented_sound, sr\n    \naudio_augs = albumentations.Compose([\n    RandomAudio(always_apply=True),\n     AddGaussianNoise(p=0.5),\n    \n])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"PERIOD = 5\n\nclass PANNsDataset(data.Dataset):\n    def __init__(\n            self,\n            file_list: List[List[str]],\n            waveform_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.waveform_transforms = waveform_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            dat = y,sr\n            y, sr = self.waveform_transforms(data=dat)['data']\n            y = y.astype(np.float32)\n            \n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return {\"waveform\": y, \"targets\": labels}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train\n\nSome code are taken from https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast .\nThanks @ttahara!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_fold = 0\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n# loaders\nloaders = {\n    \"train\": data.DataLoader(PANNsDataset(train_file_list, audio_augs), \n                             batch_size=64, \n                             shuffle=True, \n                             num_workers=2, \n                             pin_memory=True, \n                             drop_last=True),\n    \"valid\": data.DataLoader(PANNsDataset(val_file_list, None), \n                             batch_size=64, \n                             shuffle=False,\n                             num_workers=2,\n                             pin_memory=True,\n                             drop_last=False)\n}\n\n# model\nmodel_config[\"classes_num\"] = 527\nmodel = PANNsCNN14Att(**model_config)\nmodel.att_block = AttBlock(2048, 264, activation='sigmoid')\nmodel.att_block.init_weights()\n# weights=torch.load(\"../input/birdcalleffnet0/model_fold_0.bin\")\n# model.load_state_dict(weights)\n\nmodel.to(device)\noptimizer = optim.AdamW(model.parameters(), lr=0.0001)\n# Scheduler\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \n        \ndef do_mixup(x, mixup_lambda):\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out.float()   \n\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.bce = nn.BCELoss()\n    def forward(self, inputs, target):\n        input_ = torch.mean(inputs['framewise_output'], axis=1)\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        target = target['targets']\n        input_ = torch.clamp(input_, 0, 1)\n        return self.bce(input_, target)\n\nclass EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\", delta=0.0001, tpu=False):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.tpu = tpu\n        self.delta = delta\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n    def __call__(self, epoch_score, model, model_path):\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(\n                    \"EarlyStopping counter: {} out of {}\".format(\n                        self.counter, self.patience\n                    )\n                )\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            print(\"Validation score improved ({} --> {}). Saving model!\".format(\n                        self.val_score, epoch_score\n                    )\n                )\n            torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score\n        \nfrom sklearn.metrics import f1_score   \ndef f1_score_func(output_dict, target_dict):\n    y_pred = output_dict['clipwise_output'].argmax(axis=1).detach().cpu().numpy()\n    y_true = target_dict['targets'].argmax(axis=1).detach().cpu().numpy()\n    return f1_score(y_true, y_pred, average=\"macro\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nfrom torch.cuda import amp\npanns_loss = PANNsLoss()\n\ndef model_fn(batch_data_dict, device, model, mixup=True):\n        if mixup:\n            batch_output_dict = model(batch_data_dict['waveform'], \n                                    batch_data_dict['mixup_lambda'].float())\n            batch_target_dict = {'targets': do_mixup(batch_data_dict['targets'], \n                                    batch_data_dict['mixup_lambda'].float())}\n        else:\n            batch_output_dict = model(batch_data_dict['waveform'], None)\n            batch_target_dict = {'targets': batch_data_dict['targets']}\n        pan_loss = panns_loss(batch_output_dict, batch_target_dict)\n        f1_score_loss = f1_score_func(batch_output_dict, batch_target_dict)\n        return pan_loss,f1_score_loss\nclass Engine:\n    def __init__(\n        self,\n        model,\n        optimizer,\n        device,\n        scheduler=None,\n        accumulation_steps=1,\n        use_tpu=False,\n        tpu_print=10,\n        fp16=False,\n        model_fn=None,\n        use_mean_loss=False,\n        mixup=True\n    ):\n        self.model = model\n        self.optimizer = optimizer\n        self.device = device\n        self.scheduler = scheduler\n        self.accumulation_steps = accumulation_steps\n        self.use_tpu = use_tpu\n        self.tpu_print = tpu_print\n        self.model_fn = model_fn\n        self.fp16 = fp16\n        self.scaler = None\n        self.mixup = mixup\n        if self.fp16:\n            self.scaler = amp.GradScaler()\n        self.model = self.model.to(self.device)\n\n    def train(self, data_loader):\n        losses = AverageMeter()\n        self.model.train()\n        print_idx = int(len(data_loader) * self.tpu_print / 100)\n        if self.mixup:\n            mixup_augmenter = Mixup(mixup_alpha=1.)\n            \n        if self.accumulation_steps > 1:\n            self.optimizer.zero_grad()\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n\n        for b_idx, batch_data_dict in enumerate(tk0):\n            \n            if self.mixup:\n                batch_data_dict['mixup_lambda'] = torch.tensor(mixup_augmenter.get_lambda(len(batch_data_dict['waveform'])))\n            for key, value in batch_data_dict.items():\n                    batch_data_dict[key] = value.to(self.device)\n            if self.accumulation_steps == 1 and b_idx == 0:\n                self.optimizer.zero_grad()\n\n            if self.model_fn is None:\n                _, loss = self.model(**batch_data_dict)\n            else:\n                if self.fp16:\n                    with amp.autocast():\n                        pan_loss, f1_score_loss = self.model_fn(batch_data_dict, self.device, self.model, self.mixup)\n                        loss= pan_loss\n                else:\n                    pan_loss,f1_score_loss = self.model_fn(batch_data_dict, self.device, self.model, self.mixup)\n                    loss = pan_loss \n\n            if not self.use_tpu:\n                with torch.set_grad_enabled(True):\n                    \n                    if self.fp16:\n                        self.scaler.scale(loss).backward()\n                    else:\n                        loss.backward()\n\n                    self.optimizer.step()\n                    self.scheduler.step()\n\n                    if self.fp16:\n                            self.scaler.update()\n\n                    if b_idx > 0:\n                            self.optimizer.zero_grad()\n\n            \n            losses.update(loss.item(), data_loader.batch_size)\n            tk0.set_postfix(loss=losses.avg)\n            \n        tk0.close()\n        return losses.avg\n\n    def evaluate(self, data_loader, return_predictions=False):\n        losses = AverageMeter()\n        f1_score_losses =  AverageMeter()\n        print_idx = int(len(data_loader) * self.tpu_print / 100)\n        self.model.eval()\n        final_predictions = []\n        with torch.no_grad():\n            tk0 = tqdm(data_loader, total=len(data_loader))\n            for b_idx, batch_data_dict in enumerate(tk0):\n                for key, value in batch_data_dict.items():\n                    batch_data_dict[key] = value.to(self.device)\n                if self.fp16:\n                    with amp.autocast():\n                        output_dict = self.model(batch_data_dict['waveform'], None)\n                        target_dict = {'targets': batch_data_dict['targets']}\n#                         nll_loss = clip_nll(output_dict, target_dict)\n                        pan_loss = panns_loss(output_dict, target_dict)\n                        f1_score_loss = f1_score_func(output_dict, target_dict)\n                        loss = pan_loss #+ pan_loss\n                else:\n                        output_dict = self.model(batch_data_dict['waveform'], None)\n                        target_dict = {'targets': batch_data_dict['targets']}\n#                         nll_loss = clip_nll(output_dict, target_dict)\n                        pan_loss = panns_loss(output_dict, target_dict)\n                        f1_score_loss = f1_score_func(output_dict, target_dict)\n                        loss = pan_loss #+ pan_loss\n                        \n                if return_predictions:\n                    final_predictions.append(output_dict)\n                    \n                f1_score_losses.update(f1_score_loss.item(), data_loader.batch_size)\n                losses.update(loss.item(), data_loader.batch_size)\n                tk0.set_postfix(loss=losses.avg)\n            tk0.close()\n        if return_predictions:\n            return losses.avg,f1_score_losses, final_predictions\n        else:\n            return losses.avg, f1_score_losses.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ntrain_engine = Engine(model=model,\n                      optimizer=optimizer,\n                      device=device,\n                      scheduler=scheduler,\n                      model_fn=model_fn,\n                      accumulation_steps=1,\n                      mixup=False,\n                      fp16=False\n                     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 1\nmin_loss = 1\nstep = 0\n\nfor epoch in range(EPOCHS):\n        train_loss = train_engine.train(loaders['train'])\n        valid_loss, f1_loss = train_engine.evaluate(loaders['valid'])\n        print(f\"Epoch = {epoch}, LOSS = {valid_loss}, F1 Score: {f1_loss}\")\n        if valid_loss < min_loss:\n            min_loss = valid_loss\n            torch.save(train_engine.model.state_dict(), \"model_fold_0.pth\")\n            print(\"saved model!!\")\n            step=0\n            \n        else:\n            step += 1\n            print(f\"out of step!!! STEP: {step}\")\n        if step>10:\n            print(\"break\")\n            break\n            \n        gc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}