{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"! pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchlibrosa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def path_folder(x):\n    folder1 = \"ab\"\n    folder2 = \"cdef\"\n    folder3 = \"ghijklm\"\n    folder4 = \"nopqr\"\n    folder5 = \"stuvwxy\"\n    if x[0] in folder1:\n        return \"/kaggle/input/birdsong-resampled-train-audio-00\"\n    elif x[0] in folder2:\n        return \"/kaggle/input/birdsong-resampled-train-audio-01\"\n    elif x[0] in folder3:\n        return \"/kaggle/input/birdsong-resampled-train-audio-02\"\n    elif x[0] in folder4:\n        return \"/kaggle/input/birdsong-resampled-train-audio-03\"\n    elif x[0] in folder5:\n        return \"/kaggle/input/birdsong-resampled-train-audio-04\"\n    else:\n        raise(\"ERROR\")\ndef path_folder_ext_wav(x):\n    folder1 = \"ab\"\n    folder2 = \"cdef\"\n    folder3 = \"ghijklm\"\n    folder4= \"nopqrs\"\n    folder5 = \"tuvwxyz\"\n    if x[0] in folder1:\n        return \"/kaggle/input/xenoexternalwav0/external-xeno-wav-0/external-xeno-wav\"\n    elif x[0] in folder2:\n        return \"/kaggle/input/xenoexternalwav0/external-xeno-wav-1/external-xeno-wav\"\n    elif x[0] in folder3:\n        return \"/kaggle/input/xenoexternalwav0/external-xeno-wav-2/external-xeno-wav\"    \n    elif x[0] in folder4:\n        return \"/kaggle/input/xenoexternalwav1/external-xeno-wav-3/external-xeno-wav\" \n    elif x[0] in folder5:\n        return \"/kaggle/input/xenoexternalwav1/external-xeno-wav-4/external-xeno-wav\" \n    else:\n        print(x[0])\n        raise(\"Error\")\n# Options for Logmel\nmel_bins = 64\nfmin = 50\nfmax = 14000\nwindow_size = 1024\nhop_size = 320\naudioset_classes_num = 527\n\npretrained_model_path = \"pretrained/Cnn14_mAP0.431.pth\"\n\n\n\ndata = pd.read_csv(\"/kaggle/input/birdsongdataset/trainv2.csv\").sort_values(\"filename\").reset_index().drop(columns=[\"index\"])\ndata[\"path\"] = data.apply(lambda x: os.path.join(path_folder(x.ebird_code), x.ebird_code, x.filename.replace(\".mp3\", \".wav\")), axis=1)\ndata[\"targets2\"] = (~data.background.isna()).astype(np.float32)\n\ndata_ext = pd.read_csv(\"/kaggle/input/birdsongdataset/trainv2_ext.csv\").sort_values(\"filename\").reset_index().drop(columns=[\"index\"])\ndata_ext[\"path\"] = data_ext.apply(lambda x: os.path.join(path_folder_ext_wav(x.ebird_code), x.ebird_code, x.filename.replace(\".mp3\", \".wav\")), axis=1)\ndata_ext[\"targets2\"] = (~data_ext.background.isna()).astype(np.float32)\n\ntrain = data.query(\"fold != 0\").reset_index().drop(columns=[\"index\"])\nval = data.query(\"fold == 0\").reset_index().drop(columns=[\"index\"])\n\ntrain = pd.concat((train, data_ext), axis=0).reset_index().drop(columns=[\"index\"])\n\nval_5sec = pd.read_csv(\"/kaggle/input/birdsongdataset/validation_5sec.csv\")\nval_5sec[\"path\"] = val_5sec.apply(lambda x: os.path.join(path_folder(x.ebird_code) if x.split == \"train\" else path_folder_ext_wav(x.ebird_code), x.ebird_code, x.filename.replace(\".mp3\", \".wav\")), axis=1)\nval_5sec[\"targets2\"] = (~val_5sec.background.isna()).astype(np.float32)\nnoise = pd.read_csv(\"/kaggle/input/esc50dataset/esc_environment.csv\")\nnoise[\"path\"] = noise.apply(lambda x: os.path.join(\"/kaggle/input/esc50dataset/ESC-50-master/ESC-50-master/audio\", x.filename), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_5sec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noise","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler \nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation\nfrom torch.nn.utils.rnn import *\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    os.environ['PYTHONHASHSEED'] = str(seed_value)\n    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = False\n        torch.backends.cudnn.benchmark = False\n        \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler \n\nimport torchvision\nimport torchvision.models as models\n\nimport librosa\nimport librosa.display\n\nfrom joblib import Parallel, delayed\nimport os\nimport numpy as np\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import Audio, IFrame, display\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.metrics import f1_score\nimport pickle\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class ConvPreWavBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvPreWavBlock, self).__init__()\n        \n        self.conv1 = nn.Conv1d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1,\n                              padding=1, bias=False)\n                              \n        self.conv2 = nn.Conv1d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1, dilation=2, \n                              padding=2, bias=False)\n                              \n        self.bn1 = nn.BatchNorm1d(out_channels)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        x = F.max_pool1d(x, kernel_size=pool_size)\n        \n        return x\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.conv2 = nn.Conv2d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x\n\n\ndef init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n \n    if hasattr(layer, 'bias'):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n    \ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)\n    \n    \nclass Wavegram_Logmel_Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(Wavegram_Logmel_Cnn14, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.classes_num = classes_num\n        # Wavegram\n        self.pre_conv0 = nn.Conv1d(in_channels=1, out_channels=64, kernel_size=11, stride=5, padding=5, bias=False)\n        self.pre_bn0 = nn.BatchNorm1d(64)\n        self.pre_block1 = ConvPreWavBlock(64, 64)\n        self.pre_block2 = ConvPreWavBlock(64, 128)\n        self.pre_block3 = ConvPreWavBlock(128, 128)\n        self.pre_block4 = ConvBlock(in_channels=4, out_channels=64)\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=128, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.pre_conv0)\n        init_bn(self.pre_bn0)\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\" Input: (batch_size, data_length)\"\"\"\n\n        a1 = F.relu_(self.pre_bn0(self.pre_conv0(input[:, None, :])))\n        a1 = self.pre_block1(a1, pool_size=4)\n        a1 = self.pre_block2(a1, pool_size=4)\n        a1 = self.pre_block3(a1, pool_size=4)\n        a1 = a1.reshape((a1.shape[0], -1, 32, a1.shape[-1])).transpose(2, 3)\n        a1 = self.pre_block4(a1, pool_size=(2, 1))\n        #print(\"a1:\", a1)\n        x1 = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        #print(\"spect\", x.isnan().sum(), x.shape, x[0])\n        with autocast(False):\n            x = self.logmel_extractor(x1)    # (batch_size, 1, time_steps, mel_bins)\n        #if x.isnan().sum()>0:\n            #print(x1[0,0,0], x[0,0,0])\n            #print(\"logmel\", x.isnan().sum(), x.shape)\n            #x[x.isnan()] = 0.\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n            a1 = do_mixup(a1, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n\n        # Concate wavegram and spectrogram along the channel dimention\n        x = torch.cat((x, a1), dim=1)\n\n\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        \n        #embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = self.fc_audioset(x)\n        #print(\"clipwise:\",clipwise_output)\n        #output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return torch.sigmoid(clipwise_output)#output_dict \n\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def _resnet_conv3x3(in_planes, out_planes):\n    #3x3 convolution with padding\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,\n                     padding=1, groups=1, bias=False, dilation=1)\n\n\ndef _resnet_conv1x1(in_planes, out_planes):\n    #1x1 convolution\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)\n\n\nclass _ResnetBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in _ResnetBasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n\n        self.stride = stride\n\n        self.conv1 = _resnet_conv3x3(inplanes, planes)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = _resnet_conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        nn.init.constant_(self.bn2.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            out = F.avg_pool2d(x, kernel_size=(2, 2))\n        else:\n            out = x\n\n        out = self.conv1(out)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass _ResnetBottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        self.stride = stride\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = _resnet_conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = _resnet_conv3x3(width, width)\n        self.bn2 = norm_layer(width)\n        self.conv3 = _resnet_conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        init_layer(self.conv3)\n        init_bn(self.bn3)\n        nn.init.constant_(self.bn3.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            x = F.avg_pool2d(x, kernel_size=(2, 2))\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass _ResNet(nn.Module):\n    def __init__(self, block, layers, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(_ResNet, self).__init__()\n\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n\n        self.layer1 = self._make_layer(block, 64, layers[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            if stride == 1:\n                downsample = nn.Sequential(\n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[0])\n                init_bn(downsample[1])\n            elif stride == 2:\n                downsample = nn.Sequential(\n                    nn.AvgPool2d(kernel_size=2), \n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[1])\n                init_bn(downsample[2])\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class ResNet22(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(ResNet22, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.classes_num = classes_num\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        # self.conv_block2 = ConvBlock(in_channels=64, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[2, 2, 2, 2], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        with autocast(False):\n            x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        #embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        #output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return clipwise_output#output_dict\n\n\nclass ResNet38(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num, background=False):\n        \n        super(ResNet38, self).__init__()\n        self.classes_num = classes_num\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        \n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        # self.conv_block2 = ConvBlock(in_channels=64, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n        \n        self.background = background\n        if background:\n            self.fc_background = nn.Linear(2048, 1, bias=True)\n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        with autocast(False):\n            x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        #embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        if self.background:\n            background_output = torch.sigmoid(self.fc_background(x))\n            return clipwise_output, background_output#output_dict\n        return clipwise_output\n\nclass ResNet54(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        \n        super(ResNet54, self).__init__()\n        self.classes_num = classes_num\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        # self.conv_block2 = ConvBlock(in_channels=64, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBottleneck, layers=[3, 4, 6, 3], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=2048, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        with autocast(False):\n            x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        #embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        #output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return clipwise_output #output_dict\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GlobalAttention(nn.Module):\n    def __init__(self, hidden_size, padding_value=0):\n        super(GlobalAttention , self).__init__()\n        \n        self.W = nn.Linear(hidden_size, hidden_size)#, bias=False)\n        self.padding_value = padding_value\n    def forward(self, h_i_pack):\n        \"\"\"\n            input is a pack seq format\n        \"\"\"\n        \n        h_i, _ = pad_packed_sequence(h_i_pack, batch_first=True, padding_value=self.padding_value)\n        #print(\"pad seq\", h_i.shape)\n        #print(self.W.weight.shape, h_i_pack[0].shape)\n        u_i = self.W(h_i_pack[0])\n        #print(\"u_i :\", u_i.isnan().sum(), u_i.shape, u_i)\n        #print(\"compute w\")\n        u_i = PackedSequence(u_i, h_i_pack[1])\n        \n        u_i, _ = pad_packed_sequence(u_i, batch_first=True, padding_value=-float(\"inf\"))\n        #print(\"pad soft\")\n        a_i =  F.softmax(u_i, dim=1)#µ/2.0\n        #if a_i.isnan().sum()>0:\n        #    print(\"Attention globale : \", a_i.shape, a_i)\n        \n        #print(a_i.shape, a_i.sum(1), a_i.sum(1).shape)\n        v = (a_i * h_i).sum(1)        \n        #print(v)\n        return v, a_i\n\nclass BirdSongClassifierBackground(nn.Module):\n    def __init__(self, backbone, freeze_base=False, background=False):\n        \"\"\"Classifier for a new task using pretrained Cnn14 as a sub module.\n        \"\"\"\n        super(BirdSongClassifierBackground, self).__init__()        \n        self.backbone = backbone\n\n        self.global_attention = GlobalAttention(backbone.classes_num)\n        self.background = background\n        if background:\n            self.global_attention_background = GlobalAttention(1)\n\n        if freeze_base:\n            # Freeze AudioSet pretrained layers\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n\n   \n\n \n\n    def load_from_pretrain(self, pretrained_checkpoint_path):\n        checkpoint = torch.load(pretrained_checkpoint_path,\n                                map_location=\"cpu\")\n        \n        try:\n            del checkpoint[\"model\"][\"fc_audioset.weight\"]\n            del checkpoint[\"model\"][\"fc_audioset.bias\"]\n            print(\"checkpoint audioset deleted\")\n        except:\n            try:\n                del checkpoint[\"model\"][\"att_block.att.weight\"]\n                del checkpoint[\"model\"][\"att_block.att.bias\"]\n                del checkpoint[\"model\"][\"att_block.cla.weight\"]\n                del checkpoint[\"model\"][\"att_block.cla.bias\"]\n                del checkpoint[\"model\"][\"att_block.bn_att.weight\"]\n                del checkpoint[\"model\"][\"att_block.bn_att.bias\"]\n                del checkpoint[\"model\"][\"att_block.bn_att.running_mean\"]\n                del checkpoint[\"model\"][\"att_block.bn_att.running_var\"]\n                print(\"checkpoint attention deleted\")\n            except:\n                pass\n        weights = self.backbone.state_dict()\n        weights.update(checkpoint['model'])\n        self.backbone.load_state_dict(weights)\n\n    def forward(self, input, mixup_lambda=None, lengths=None):\n        \"\"\"Input: (batch_size, length, data_length)\n        \"\"\"\n\n        if self.background:\n            features = []\n            features2 = []\n            for i in range(input.shape[1]):\n                preds_chunk, preds_chunk2 = self.backbone(input[:, i], mixup_lambda)\n                #print(\"nan : \",preds_chunk.isnan().sum())\n                features.append(preds_chunk)\n                features2.append(preds_chunk2)\n\n            features = torch.stack(features, dim=1) # prediction per chunk of 5 sec\n            features2 = torch.stack(features2, dim=1) # prediction per chunk of 5 sec\n            #print(len(lengths), features.shape)\n\n            if mixup_lambda is not None:\n                lengths = do_mixup_length(lengths)\n            features_pack = pack_padded_sequence(features, batch_first=True, lengths=lengths)\n            features2_pack = pack_padded_sequence(features2, batch_first=True, lengths=lengths)\n\n            x , att = self.global_attention(features_pack)\n            x2 , att2 = self.global_attention_background(features2_pack)\n\n            return x , features , x2, features2\n        \n        else:\n            features = []\n        \n            for i in range(input.shape[1]):\n                preds_chunk = self.backbone(input[:, i], mixup_lambda)\n                features.append(preds_chunk)\n\n            features = torch.stack(features, dim=1) # prediction per chunk of 5 sec\n\n            if mixup_lambda is not None:\n                lengths = do_mixup_length(lengths)\n            features_pack = pack_padded_sequence(features, batch_first=True, lengths=lengths)\n\n            x , att = self.global_attention(features_pack)\n\n            return x , features\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttBlock(nn.Module):\n    def __init__(self, n_in, n_out, activation='linear', temperature=1.):\n        super(AttBlock, self).__init__()\n        \n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)\n        self.cla = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)\n        \n        self.bn_att = nn.BatchNorm1d(n_out)\n        self.init_weights()\n        \n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n         \n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\nclass Cnn14_DecisionLevelAtt(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num, background=False):\n        \n        super(Cnn14_DecisionLevelAtt, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.classes_num = classes_num\n        self.interpolate_ratio = 32     # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, self.classes_num, activation='sigmoid')\n        self.background = background\n        if background:\n            self.att_block2 = AttBlock(2048, 1, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        with autocast(False):\n            x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, _, segmentwise_output) = self.att_block(x)\n\n        (clipwise_output2, _, segmentwise_output2) = self.att_block2(x)\n\n        #segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        #framewise_output = interpolate(segmentwise_output, self.interpolate_ratio)\n        #framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        #output_dict = {'framewise_output': framewise_output, \n        #    'clipwise_output': clipwise_output}\n        if self.background:\n\n            return clipwise_output, clipwise_output2\n        return clipwise_output #output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AudioGeneratorDataset(torch.utils.data.Dataset):\n    def __init__(self, path_audio, y, y2, path_augment,  resample_freq = 32000, \n                 max_length=365, augmentation=[], validation=False, num_class=264, background=False):\n        self.labels2idx = {'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263}\n        \n        self.idx2labels = {k:v for v,k in self.labels2idx.items()}\n\n        self.augmentation = set(augmentation)\n        self.samples = path_audio #+ path_augment\n        self.max_length = max_length # 99% are shorter than 365 sec\n        self.resample_freq=resample_freq\n        self.validation = validation\n        self.y = np.array([self.labels2idx[t] for t in y]) #+ [self.labels2idx[t] for t in y_aug]\n        self.background = background\n        if self.background:\n            self.y2 = np.array(y2)\n        self.num_class = num_class\n        if \"noise\" in self.augmentation:\n            self.noise = [self.load_raw_audio_noise(p) for p in path_augment]\n\n    def load_raw_audio_noise(self, x):\n        signal, sr_orig = librosa.load(x, sr=self.resample_freq)\n        shape = len(signal)\n        \n        if shape > self.max_length*self.resample_freq:\n            start = np.random.randint(0, shape - self.max_length*self.resample_freq)\n            signal = signal[start:start+self.max_length*self.resample_freq]\n            if len(signal) != self.max_length*self.resample_freq:\n                signal = np.resize(signal, self.max_length*self.resample_freq).astype(np.float32)\n        else:\n            toadd = (self.resample_freq*5 -  (signal.shape[0]%(self.resample_freq*5)))%(self.resample_freq*5)\n            signal = np.resize(signal, self.max_length*self.resample_freq).astype(np.float32)\n        return signal\n            \n    def load_raw_audio(self, x):\n        signal, sr_orig = librosa.load(x, sr=self.resample_freq)\n        shape = len(signal)\n        if self.validation:\n                signal = signal[:self.max_length*self.resample_freq]\n        else:\n            if shape > self.max_length*self.resample_freq:\n                start = np.random.randint(0, shape - self.max_length*self.resample_freq)\n                signal = signal[start:start+self.max_length*self.resample_freq]\n                #print(len(signal))\n        toadd = (self.resample_freq*5 -  (signal.shape[0]%(self.resample_freq*5)))%(self.resample_freq*5)\n        signal = np.concatenate((signal, np.zeros(toadd))).astype(np.float32)\n        return signal.astype(np.float32)\n    \n    def load_pickle(self, filename):\n        #with open(filename, \"rb\") as f:\n        data = np.load(filename).astype(np.float32)#pickle.load(f)\n        return data\n    \n\n    def load_audio(self, filename):\n\n            signal = self.load_pickle(filename)\n            shape = len(signal)\n            #print(\"shape:\", shape)\n            if self.validation:\n                signal = signal[:self.max_length*self.resample_freq]\n            else:\n                if shape > self.max_length*self.resample_freq:\n                    start = np.random.randint(0, shape - self.max_length*self.resample_freq)\n                    signal = signal[start:start+self.max_length*self.resample_freq]\n                    #print(len(signal))\n            toadd = (self.resample_freq*5 -  (signal.shape[0]%(self.resample_freq*5)))%(self.resample_freq*5)\n            signal = np.concatenate((signal, np.zeros(toadd))).astype(np.float32)\n            return signal\n    def normalise(self, x):\n        \"\"\"\n        \n\n        Parameters\n        ----------\n        x : numpy array\n            DESCRIPTION.\n\n        Returns\n        -------\n        x_norm : Normalise signal between 0 and 1 value\n            DESCRIPTION.\n\n        \"\"\"\n        min_ = x.min()\n        max_ = x.max()\n        if min_ == max_:\n            x_norm = np.zeros(x.shape).astype(np.float32)\n        else:\n            x_norm =  (x-min_)/(max_ - min_)\n        return x_norm\n    \n    def __getitem__(self, index):\n        l = []\n        # label\n        labels_item = set([self.y[index]])\n        labels_one_hot = torch.nn.functional.one_hot(torch.tensor(self.y[index]), self.num_class).type(torch.float32)\n        if self.background:\n            labels2_item = self.y2[index]\n        # load signal\n        signal_raw = self.load_raw_audio(self.samples[index] )\n        # add multiple audio (Augmentation for multi labels)\n        if \"augment\" in self.augmentation:\n            augmented = np.zeros(signal_raw.shape)\n            number = np.random.randint(0, 3)\n            ids = np.random.randint(0, self.__len__(), size=number)\n            counts = 0\n            for id_ in ids:\n                if self.y[id_] not in labels_item: # add only if class not already present\n                    labels_item.add(self.y[id_])\n                    labels_one_hot += torch.nn.functional.one_hot(torch.tensor(self.y[id_]), self.num_class).type(torch.float32) # update label\n                    augmented += np.resize(self.load_raw_audio(self.samples[id_] ), augmented.shape) # update augmented signal\n                    if self.background:\n                        labels2_item += self.y2[id_]\n                    counts += 1\n            if counts > 0:\n                counts += 1\n                signal_raw = (signal_raw + augmented)/counts\n        # add Environment Noise\n        \n        if \"noise\" and \"gaussian\" in self.augmentation:\n            if np.random.uniform()>0.5:\n                noise = np.resize(self.noise[np.random.randint(0, len(self.noise))], len(signal_raw))\n                alpha = np.random.uniform(0.1,0.3)\n                signal_raw = (alpha*signal_raw) + ((1-alpha)*noise)\n                \n            else:\n                signal_raw = AddGaussianNoise(signal_raw)\n        else:\n            if \"noise\" in self.augmentation:\n                noise = np.resize(self.noise[np.random.randint(0, len(self.noise))], len(signal_raw))\n                alpha = np.random.uniform(0.1,0.3)\n                signal_raw = (alpha*signal_raw) + ((1-alpha)*noise)\n                \n            if \"gaussian\" in self.augmentation:\n                signal_raw = AddGaussianNoise(signal_raw)\n        ## convert into mel spectrogram\n        signal_raw = signal_raw.reshape(-1, self.resample_freq*5)\n        \n        \n        l.append( torch.tensor(signal_raw) )\n        l.append(torch.tensor(len(signal_raw)))\n        l.append(labels_one_hot)\n        \n        if self.background:\n            labels2_item = torch.tensor(labels2_item>0).float()\n            l.append(labels2_item)\n        l.append(torch.tensor(index))\n        return tuple(l)\n    \n    def __len__(self):\n        return len(self.samples)\n    \n    \ndef collate_padding_background(batch):\n    lengths = []\n    signals = []\n    indices = []\n    labels = []\n    labels2 = []\n    max_ = 0\n    for item in batch :\n        signals.append(item[0])\n        lengths.append(item[1])\n        max_ = max(max_, item[1])\n        labels.append(item[2])\n        labels2.append(item[3])\n        indices.append(item[4])\n    \n    #print(lengths)\n    lengths = torch.stack(lengths)\n    indices = torch.stack(indices)\n    labels = torch.stack(labels)\n    labels2 = torch.stack(labels2)\n\n    for i in range(len(lengths)):\n        N = max_ - len(signals[i])\n        if N > 0:\n            signals[i] = torch.cat((signals[i], torch.zeros((N, signals[i].shape[1]))), dim=0)\n    signals = torch.stack(signals, dim=0)\n    \n    idx_sort = torch.argsort(lengths, descending=True)\n    \n        \n    lengths = lengths[idx_sort]\n    labels = labels[idx_sort]\n    labels2 = labels2[idx_sort]\n\n    indices = indices[idx_sort]\n    signals = signals[idx_sort]\n    \n    return signals, lengths, labels, labels2, indices\n\ndef collate_padding(batch):\n    lengths = []\n    signals = []\n    indices = []\n    labels = []\n    max_ = 0\n    for item in batch :\n        signals.append(item[0])\n        lengths.append(item[1])\n        max_ = max(max_, item[1])\n        labels.append(item[2])\n        indices.append(item[-1])\n    \n    #print(lengths)\n    lengths = torch.stack(lengths)\n    indices = torch.stack(indices)\n    labels = torch.stack(labels)\n\n    for i in range(len(lengths)):\n        N = max_ - len(signals[i])\n        if N > 0:\n            signals[i] = torch.cat((signals[i], torch.zeros((N, signals[i].shape[1]))), dim=0)\n    signals = torch.stack(signals, dim=0)\n    \n    idx_sort = torch.argsort(lengths, descending=True)\n    \n        \n    lengths = lengths[idx_sort]\n    labels = labels[idx_sort]\n    \n    indices = indices[idx_sort]\n    signals = signals[idx_sort]\n    \n    return signals, lengths, labels, indices    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return np.array(mixup_lambdas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## -- Fonction Loop  -- ##\n    \n\n        \ndef train_fn(model, dataloader, optimizer, loss_fn, cfg, accumulation=2, l_mixup=1.0):\n    total_loss = 0.\n    model.train()\n    t=tqdm(dataloader)\n    scaler = GradScaler()\n    optimizer.zero_grad()\n    N = 0.\n    if l_mixup>0:\n        mixup = Mixup(mixup_alpha=l_mixup)\n\n    for i, batch in enumerate(t):\n        inputs= batch[0]\n        lengths= batch[1]\n        labels= batch[2]\n        if cfg.background:\n            labels2= batch[3]\n            labels2 = labels2.to(device, dtype=torch.float)\n        indices = batch[-1]        \n        \n        inputs = inputs.to(cfg.device, dtype=torch.float)\n        lengths = lengths.to(cfg.device, dtype=torch.float)\n        labels = labels.to(cfg.device, dtype=torch.float)\n \n        lambda_mixup = None\n        if l_mixup>0:\n            lambda_mixup = torch.as_tensor(mixup.get_lambda(batch_size=cfg.batch_size)).to(cfg.device, dtype=torch.float)\n            labels = do_mixup(labels, lambda_mixup)\n            if cfg.background:\n                labels2 = do_mixup(labels2, lambda_mixup)\n                labels2[labels2>1.0] = 1.0\n            labels[labels>1.0] = 1.0\n            \n            \n        with autocast(cfg.use_apex):\n            if cfg.background:\n                outputs, outputs_frames, outputs2, _ = model(inputs, lambda_mixup, lengths)\n                outputs2 = torch.clamp(outputs2,0.0,1.0)\n            else:\n                outputs, outputs_frames = model(inputs, lambda_mixup, lengths)\n            outputs = torch.clamp(outputs,0.0,1.0)\n            labels = torch.clamp(labels,0.0,1.0)\n            \n            \n            \n            loss = loss_fn(outputs, labels )\n            if cfg.background:\n                loss2 = loss_fn(outputs2.view(-1) ,labels2)\n                loss = (loss.sum(1)+loss2).mean()\n            else:\n                loss = loss.sum(1).mean()\n            N += len(inputs)\n\n            if torch.isnan(loss):\n                print(\"loss error\")\n                print(torch.isnan(outputs).sum())\n                loss[torch.isnan(loss)] = 0.0\n\n                total_loss += (loss.item() * len(inputs))\n            else:\n                total_loss += (loss.item() * len(inputs))\n\n\n        if cfg.use_apex:\n            loss = loss/accumulation\n            scaler.scale(loss).backward()\n        else:\n            loss = loss/accumulation\n            loss.backward()\n\n        \n\n\n\n        if (i+1)%accumulation == 0 or i-1 == len(t):\n            if cfg.use_apex:\n                scaler.step(optimizer)\n    \n                # Updates the scale for next iteration.\n                scaler.update()\n                optimizer.zero_grad()\n            else:                \n                optimizer.step()\n                optimizer.zero_grad()\n            \n        \n        t.set_description(\"Loss : {0}\".format(total_loss/N))\n        t.refresh()       \n\n    return total_loss/N\n        \ndef inference_fn(model, dataloader, optimizer, device, loss_fn, activation=False, background=False):\n    total_loss = 0.\n    t=tqdm(dataloader)\n    y_true = []\n    y_preds = []\n    model.eval()\n    with torch.no_grad():\n        for i, batch in enumerate(t):\n            \n            inputs= batch[0]\n            lengths= batch[1]\n            labels= batch[2]\n            if background:\n                labels2= batch[3]\n                labels2 = labels2.to(device, dtype=torch.float)\n            indices = batch[-1]\n\n            inputs = inputs.to(device, dtype=torch.float)\n            lengths = lengths.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            \n\n            #labels_one_hot = torch.cuda.FloatTensor(labels.size(0), 265).to(device, dtype=torch.float)\n            #labels_one_hot = torch.nn.functional.one_hot(labels, 264).to(device, dtype=torch.float)\n            with autocast(cfg.use_apex):\n                if background:\n                    outputs, outputs_frames, outputs2, _ = model(inputs, None, lengths)\n                    outputs2 = torch.clamp(outputs2,0.0,1.0)\n                    labels2 = torch.clamp(labels2,0.0,1.0)\n                else:\n                    outputs, outputs_frames  = model(inputs, None, lengths)\n                    \n                outputs = torch.clamp(outputs,0.0,1.0)\n                labels = torch.clamp(labels,0.0,1.0)\n                                    \n\n                loss = loss_fn(outputs, labels )\n\n                loss = loss.sum(1).mean()\n            \n                total_loss += loss.item()\n            \n            t.set_description(\"Loss : {0}\".format(total_loss/(i+1)))\n            t.refresh()\n        \n            y_true.append(labels.detach().cpu().numpy())\n            if activation:\n                outputs = torch.sigmoid(outputs)\n            y_preds.append(outputs.cpu().detach().numpy())\n            \n    return np.concatenate(y_preds), np.concatenate(y_true), total_loss/(i+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_mixup(x, mixup_lambda):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes \n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_mixup_length(x):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes \n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = torch.stack( (x[0 :: 2] , x[1 :: 2]) ).max(0)[0]\n    return out\n\ndo_mixup_length(torch.as_tensor([5,2,4,5,7,9,12,15,17,2]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## -- FONCTION DE COUTS -- ##\n    \ndef bce(outputs, targets):\n    eps = 1e-5\n    p1=targets*(torch.log(outputs+eps))\n    p0=(1-targets)*torch.log(1-outputs+eps)\n    \n    loss = p0 + p1\n    return -loss\ndef f1_score_multi(y_true, y_preds, threshold=0.5):\n    y_preds = (y_preds>=threshold).astype(np.int32)\n    f1 = []\n    for i in range(y_preds.shape[1]):\n        if len(np.unique(y_true[:,i])) == 2:\n            f1.append(f1_score(y_true[:,i], y_preds[:,i]))\n    \n    return f1\n\ndef f1_micro_row_wise(y_true, y_preds, threshold=0.5):\n    y_preds = (y_preds>=threshold).astype(np.int32)\n    return f1_score(y_true, y_preds, average='samples')\nclass Config():\n    def __init__(self):\n        self.num_class = 264\n        self.resample_freq=32000\n        self.max_length=30 # seconds\n        self.device = \"cuda:0\"\n        self.use_apex = False\n        self.epochs = 6\n        self.accumulation = 2\n        self.batch_size = 16\n        self.background = False\n        self.l_mixup=0.0\n        self.load_name = \"../input/pretrained-pann/ResNet38_mAP0.434.pth\"\n        self.save_name = \"ResNet38-264-birds-gaussian-ext.pth\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = Config()\ncfg.batch_size = cfg.batch_size * 2 if cfg.l_mixup > 0. else cfg.batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def AddGaussianNoise(sound):\n        noise = 0.005*np.random.uniform()*np.amax(sound)\n        augmented_sound = np.array(sound).astype('float32') + noise * np.random.normal(size=sound.shape[0])\n        return augmented_sound","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentations=[\"noise\", \"augment\", \"gaussian\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## -- DATASET / DATALOADER -- #\n\ndataset = AudioGeneratorDataset(train.path.values.tolist(), train.ebird_code.values, train.targets2.values, noise.path.values.tolist(),\n                                 resample_freq=cfg.resample_freq, max_length=cfg.max_length, \n                                augmentation=augmentations, validation=False, num_class=cfg.num_class, background=cfg.background)\n\ndataset_val = AudioGeneratorDataset(val.path.values.tolist(), val.ebird_code.values, val.targets2.values, [],  \n                                    resample_freq=cfg.resample_freq, max_length=cfg.max_length, augmentation=[], validation=True,\n                                    num_class=cfg.num_class, background=cfg.background)\n\n\ndataset_val5 = AudioGeneratorDataset(val_5sec.path.values.tolist(), val_5sec.ebird_code.values,val_5sec.targets2.values, [],  \n                                    resample_freq=cfg.resample_freq, max_length=cfg.max_length, augmentation=[], validation=True,\n                                    num_class=cfg.num_class, background=cfg.background)\n\nloss_fnt = bce#nn.BCEWithLogitsLoss(reduction=\"none\")\nloss_fnt_test = nn.BCELoss(reduction=\"none\")#nn.BCEWithLogitsLoss(reduction=\"none\")\n\ncollate_fn = collate_padding_background if cfg.background else collate_padding\ntrain_dataloader = DataLoader(dataset, batch_size=cfg.batch_size, shuffle=True, collate_fn=collate_fn, num_workers=4,drop_last=True)\nval_dataloader = DataLoader(dataset_val, batch_size=cfg.batch_size, shuffle=False, collate_fn=collate_fn, num_workers=1)\nval_dataloader5 = DataLoader(dataset_val5, batch_size=cfg.batch_size, shuffle=False, collate_fn=collate_fn, num_workers=1)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## -- LOAD MODEL -- ##\n\n#backbone =  Cnn14_DecisionLevelAtt(cfg.resample_freq, window_size, hop_size, mel_bins, fmin, fmax, cfg.num_class, background=True)\n\nbackbone = ResNet38(cfg.resample_freq, window_size, hop_size, mel_bins, fmin, fmax, cfg.num_class, background=cfg.background)\n\nmodel = BirdSongClassifierBackground(backbone, freeze_base=False, background=cfg.background)\n\nmodel.load_from_pretrain(cfg.load_name)\n#weights = model.state_dict()\n#weights.update(torch.load(cfg.load_name))\n#weights = torch.load(cfg.load_name)\n#model.load_state_dict(weights)\nmodel = model.to(cfg.device)\n\n\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3, amsgrad=True, eps=1e-5) # torch.optim.SGD(model.parameters(), lr=1e-3, momentum=5e-4, nesterov=True)#\n#optimizer.load_state_dict(torch.load(\"/kaggle/input/pannsmodel/optimizer-ResNet38-264-birds.h5\"))\nreducer = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=True)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score = 0.\nbest_score5 = 0.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for e in range(cfg.epochs):\n    train_fn(model, train_dataloader, optimizer, loss_fnt, cfg, accumulation=cfg.accumulation, l_mixup=cfg.l_mixup)\n\n    preds, targets, val_loss =inference_fn(model, val_dataloader, optimizer, cfg.device, loss_fnt_test, activation=False, background=cfg.background)\n    preds5, targets5, val_loss =inference_fn(model, val_dataloader5, optimizer, cfg.device, loss_fnt_test, activation=False, background=cfg.background)\n    #preds = np.argmax(preds, axis=1)\n    #targets = np.argmax(targets, axis=1)\n    score = f1_micro_row_wise(targets, preds)#np.mean(f1_score_multi(targets, preds))\n    score5 = f1_micro_row_wise(targets5, preds5)#np.mean(f1_score_multi(targets5, preds5))\n    reducer.step(val_loss)\n    if best_score < score:\n        best_score = score\n        torch.save(model.state_dict(), cfg.save_name)\n        torch.save(optimizer.state_dict(), \"optimizer-\"+ cfg.save_name)\n\n        print(\"F1 - score improved : \", score)\n    else:\n        print(\"F1 score not improve : \", score, \" Best : \", best_score)\n        \n    if best_score5 < score5:\n        best_score5 = score5\n        torch.save(model.state_dict(), \"site1_\"+cfg.save_name)\n\n        print(\"F1 - score improved : \", score5)\n    else:\n        print(\"F1 score not improve : \", score5, \" Best : \", best_score5)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}