{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/timmfork\n!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-01T13:40:04.928636Z","iopub.execute_input":"2021-06-01T13:40:04.929007Z","iopub.status.idle":"2021-06-01T13:41:01.674243Z","shell.execute_reply.started":"2021-06-01T13:40:04.928907Z","shell.execute_reply":"2021-06-01T13:41:01.673232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## prep","metadata":{}},{"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os, sys\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\nimport torchaudio\nimport torchaudio.transforms as T\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom sklearn.metrics import f1_score, classification_report\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:01.676046Z","iopub.execute_input":"2021-06-01T13:41:01.676391Z","iopub.status.idle":"2021-06-01T13:41:06.689968Z","shell.execute_reply.started":"2021-06-01T13:41:01.676352Z","shell.execute_reply":"2021-06-01T13:41:06.689126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n\nset_seed(1213)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:06.6919Z","iopub.execute_input":"2021-06-01T13:41:06.692284Z","iopub.status.idle":"2021-06-01T13:41:06.700725Z","shell.execute_reply.started":"2021-06-01T13:41:06.692242Z","shell.execute_reply":"2021-06-01T13:41:06.699854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    seed = 1213\n    epochs = 35\n    train = True\n    folds = [0]\n    img_size = 224\n    main_metric = \"epoch_f1_at_05\"\n    minimize_metric = False\n\n    ######################\n    # Data #\n    ######################\n    train_datadir = Path(\"../../input/birdclef-2021/birdclef-2021/train_short_audio\")\n    train_csv = \"../../input/birdclef-2021/input/birdclef-2021/train_metadata.csv\"\n    train_soundscape = \"../../input/birdclef-2021/birdclef-2021/train_soundscape_labels.csv\"\n\n    ######################\n    # Dataset #\n    ######################\n    transforms = {\n        \"train\": [],\n        \"valid\": [],\n        \"test\": []\n    }\n    period = 20\n    n_mels = 256\n    fmin = 0\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n    ######################\n    # Loaders #\n    ######################\n    loader_params = {\n        \"train\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        },\n        \"test\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        }\n    }\n\n    ######################\n    # Split #\n    ######################\n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 1213\n    }\n\n    ######################\n    # Model #\n    ######################\n    base_model_name = [\n        \"tf_efficientnet_b0_ns\",\n        \"tf_efficientnet_b0_ns\"\n    ]\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n    ######################\n    # Criterion #\n    ######################\n    loss_name = \"BCEFocal2WayLoss\"\n    loss_params: dict = {}\n\n    ######################\n    # Optimizer #\n    ######################\n    optimizer_name = \"Adam\"\n    base_optimizer = \"Adam\"\n    optimizer_params = {\n        \"lr\": 0.001\n    }\n    # For SAM optimizer\n    base_optimizer = \"Adam\"\n\n    ######################\n    # Scheduler #\n    ######################\n    scheduler_name = \"CosineAnnealingLR\"\n    scheduler_params = {\n        \"T_max\": 10\n    }","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:06.702427Z","iopub.execute_input":"2021-06-01T13:41:06.702769Z","iopub.status.idle":"2021-06-01T13:41:06.729948Z","shell.execute_reply.started":"2021-06-01T13:41:06.702733Z","shell.execute_reply":"2021-06-01T13:41:06.729002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## models","metadata":{}},{"cell_type":"code","source":"## MODELS\ndef init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes\n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(\n                self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n    \n\nclass TimmSED(nn.Module):\n    def __init__(self, encoder: str, pretrained=False, classes=397, in_channels=2, use_coordconv=False, attn_activation=\"sigmoid\", **kwargs):\n        super().__init__()\n\n        base_model = timm.create_model(\n            encoder, pretrained=pretrained, in_chans=in_channels, **kwargs)\n        self.encoder = base_model\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        elif hasattr(base_model, \"num_features\"):\n            in_features = base_model.num_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, classes, activation=attn_activation)\n        self.features = self.encoder.forward_features\n        self.use_coordconv = use_coordconv\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n\n    def forward(self, x):\n\n        # (batch_size, channels, freq, frames)\n        B, C, H, W = x.size()\n\n        if self.use_coordconv:\n            coords = torch.arange(0, H, dtype=x.dtype)[None, None, :, None]\n            coords = coords.repeat(B, C, 1, W).to(x.device) / H\n            x = torch.cat([x, coords], dim=1)\n\n        x = self.features(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = W // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, W)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, W)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"cls\": clipwise_output\n        }\n\n        return output_dict\n    \n## MODELS gpu\n\nfrom dataclasses import dataclass\n\n@dataclass\nclass MelConfig:\n    sample_rate: int = 32000\n    n_fft: int = 2048\n    hop_length: int = 512\n    n_mels: int = 256\n\n        \nclass TimmSEDGPU(nn.Module):\n    def __init__(self, encoder: str,\n                 mel_config: MelConfig,\n                 pretrained=False,\n                 classes=24,\n                 in_channels=1,\n                 attn_activation='sigmoid',\n                 **kwargs\n                 ):\n        super().__init__()\n        # melextractor\n        self.logmel_extractor = nn.Sequential(\n            T.MelSpectrogram(sample_rate=mel_config.sample_rate, n_fft=mel_config.n_fft, win_length=mel_config.n_fft,\n                             hop_length=mel_config.hop_length, power=2.0, n_mels=mel_config.n_mels),\n            T.AmplitudeToDB()\n        )\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_config.n_mels)\n\n        base_model = timm.create_model(\n            encoder, pretrained=pretrained, in_chans=in_channels, **kwargs)\n        self.encoder = base_model\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        elif hasattr(base_model, \"num_features\"):\n            in_features = base_model.num_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.features = self.encoder.forward_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, classes, activation=attn_activation)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        with torch.cuda.amp.autocast(False):\n            x = self.logmel_extractor(input).unsqueeze(1)\n\n        frames_num = x.size(3)\n\n        x = x.transpose(1, 2)\n        x = self.bn0(x)\n        x = x.transpose(1, 2)\n\n        if self.training:\n            x = self.spec_augmenter(x.transpose(2, 3)).transpose(2, 3)\n\n        # (batch_size, channels, freq, frames)\n        x = self.features(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict\n    \nclass TimmRNNSEDGPU(nn.Module):\n    def __init__(self, encoder: str,\n                 mel_config: MelConfig,\n                 pretrained=False,\n                 classes=24,\n                 in_channels=1,\n                 attn_activation='sigmoid',\n                 **kwargs\n                 ):\n        super().__init__()\n        # melextractor\n        self.logmel_extractor = nn.Sequential(\n            T.MelSpectrogram(sample_rate=mel_config.sample_rate, n_fft=mel_config.n_fft, win_length=mel_config.n_fft,\n                             hop_length=mel_config.hop_length, power=2.0, n_mels=mel_config.n_mels),\n            T.AmplitudeToDB()\n        )\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_config.n_mels)\n\n        base_model = timm.create_model(\n            encoder, pretrained=pretrained, in_chans=in_channels, **kwargs)\n        self.encoder = base_model\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        elif hasattr(base_model, \"num_features\"):\n            in_features = base_model.num_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.features = self.encoder.forward_features\n\n        \n        self.gru = torch.nn.GRU(input_size=in_features, hidden_size=in_features, \n                        num_layers=2, dropout=0.3, batch_first=True, bidirectional=True)\n        self.fc1 = nn.Linear(in_features*2, in_features*2, bias=True)\n        self.att_block = AttBlockV2(\n            in_features*2, classes, activation=attn_activation)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        with torch.cuda.amp.autocast(False):\n            x = self.logmel_extractor(input).unsqueeze(1)\n\n        frames_num = x.size(3)\n\n        x = x.transpose(1, 2)\n        x = self.bn0(x)\n        x = x.transpose(1, 2)\n\n        if self.training:\n            x = self.spec_augmenter(x.transpose(2, 3)).transpose(2, 3)\n        \n        # (batch_size, channels, freq, frames)\n        x = self.features(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        (x, _) = self.gru(x)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"segmentwise_logit\": segmentwise_logit,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:06.731743Z","iopub.execute_input":"2021-06-01T13:41:06.732123Z","iopub.status.idle":"2021-06-01T13:41:06.797293Z","shell.execute_reply.started":"2021-06-01T13:41:06.732085Z","shell.execute_reply":"2021-06-01T13:41:06.796546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf = dict({\n    \"network\": \"TimmSED\",\n    \"encoder\": \"tf_efficientnet_b3_ns\",\n    \"encoder_params\": {\n        \"classes\": 397,\n        \"drop_path_rate\": 0.2,\n        \"attn_activation\": \"linear\",\n        \"use_coordconv\": False,\n        \"in_channels\": 1\n    },\n    \"multiplier\": 1,\n    \"use_secondary\": True,\n    \"sample_sec\": 20,\n    \"n_mels\": 256,\n    \"n_fft\": 2048,\n    \"hop_length\": 512,\n    \"crop_width\": 1240,\n    \"optimizer\": {\n        \"batch_size\": 32,\n        \"type\": \"AdamW\",\n        \"weight_decay\": 1e-2,\n        \"learning_rate\": 0.0002,\n        \"schedule\": {\n            \"type\": \"cosine\",\n            \"mode\": \"step\",\n            \"epochs\": 100,\n            \"params\": {\n              \"T_max\": 40000,\n              \"eta_min\": 1e-5\n            }\n        }\n    }\n})\n\ndef load_model(name:str, gpu_mel: bool, path: Path, conf: dict, prefix: str, suffix: str, fold: int):\n    conf['encoder'] = f\"tf_efficientnet_{name.split('_')[0]}_ns\"\n    conf['encoder_params']['use_coordconv'] = False\n    \n    \n    if name == 'v2m_gpu': conf['encoder'] = 'tf_efficientnetv2_m'\n    elif name == 'v2s_gpu': conf['encoder'] = 'tf_efficientnetv2_s_in21k'\n\n    n_mels = conf[\"n_mels\"]\n    n_fft = conf[\"n_fft\"]\n    hop_length = conf[\"hop_length\"]\n    snapshot_name = \"{}{}_{}_{}_{}_{}_{}_{}\".format(prefix, conf['network'], conf['encoder'], fold, n_mels,\n                                                    n_fft, hop_length, suffix)\n    #weights_path = os.path.join(\"weights\", snapshot_name)\n    \n    weights_path = path\n    \n    if 'gpu' in name:\n        \n        \n        if '80' in name:\n            print(\"LOADING RNN(80) GPU MODEL\")\n            conf['network'] = f\"TimmSEDGPU\"\n            del conf['encoder_params']['use_coordconv']\n            mc_80 = MelConfig\n            mc_80.n_mels = 80\n            conf['encoder'] = 'tf_efficientnetv2_b1'\n            model = TimmRNNSEDGPU(encoder=conf['encoder'], mel_config=mc_80, **conf[\"encoder_params\"])\n            conf['encoder_params']['use_coordconv'] = False\n            \n        elif '320' in name:\n            print(\"LOADING 320 GPU MODEL\")\n            conf['network'] = f\"TimmSEDGPU\"\n            del conf['encoder_params']['use_coordconv']\n            if 'b1' in name: conf['encoder'] = 'tf_efficientnetv2_b1'\n            elif 'b3' in name: conf['encoder'] = 'tf_efficientnetv2_b3'\n            mc_320 = MelConfig\n            mc_320.n_mels = 320\n            model = TimmSEDGPU(encoder=conf['encoder'], mel_config=mc_320, **conf[\"encoder_params\"])\n            conf['encoder_params']['use_coordconv'] = False\n        else:\n            print(\"LOADING DEFAULT(256) GPU MODEL\")\n            conf['network'] = f\"TimmSEDGPU\"\n            del conf['encoder_params']['use_coordconv']\n            mc_256 = MelConfig\n            mc_256.n_mels = 256\n            model = TimmSEDGPU(encoder=conf['encoder'], mel_config=mc_256, **conf[\"encoder_params\"])\n            \n    else:\n        model = TimmSED(encoder=conf['encoder'], **conf[\"encoder_params\"])\n        \n        \n    model = torch.nn.DataParallel(model).cuda()\n    print(\"=> loading checkpoint '{}'\".format(weights_path))\n    checkpoint = torch.load(weights_path, map_location=\"cpu\")\n    print(\"epoch\", checkpoint['epoch'])\n    model.load_state_dict(checkpoint['state_dict'])\n    model.eval()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:06.800279Z","iopub.execute_input":"2021-06-01T13:41:06.800577Z","iopub.status.idle":"2021-06-01T13:41:06.818096Z","shell.execute_reply.started":"2021-06-01T13:41:06.800552Z","shell.execute_reply":"2021-06-01T13:41:06.817272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights_paths = [\n    Path(\"../input/ensemble-01/e30oof_TimmSEDGPU_tf_efficientnetv2_s_in21k_0_256_2048_512_last\"),\n    Path(\"../input/ensemble-01/e40oof_TimmSEDGPU_tf_efficientnetv2_s_in21k_1_256_2048_512_best_f1\"),\n    # Path(\"../input/ensemble-01/e30oof_TimmSEDGPU_tf_efficientnetv2_s_in21k_2_256_2048_512_best_f1\"),\n    # Path(\"../input/ensemble-01/oof_TimmSEDGPU_tf_efficientnetv2_s_in21k_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/TimmSEDGPU_tf_efficientnetv2_s_in21k_0_256_2048_512_best_f1 (1)\"),\n    Path(\"../input/ensemble-01/TimmSEDGPU_tf_efficientnetv2_s_in21k_1_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/TimmSEDGPU_tf_efficientnetv2_s_in21k_2_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/TimmSEDGPU_tf_efficientnetv2_s_in21k_4_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1_5secTimmSEDGPU_tf_efficientnet_b1_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1_5secTimmSEDGPU_tf_efficientnet_b1_ns_1_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1_5secTimmSEDGPU_tf_efficientnet_b1_ns_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1_pseudoTimmSEDGPU_tf_efficientnet_b1_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1_pseudoTimmSEDGPU_tf_efficientnet_b1_ns_3_256_2048_512_best_f1\"),\n    # Path(\"../input/ensemble-01/b1TimmSEDGPU_tf_efficientnet_b1_ns_1_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1TimmSEDGPU_tf_efficientnet_b1_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1TimmSEDGPU_tf_efficientnet_b1_ns_2_256_2048_512_best_f1\"),\n    # Path(\"../input/ensemble-01/b1TimmSEDGPU_tf_efficientnet_b1_ns_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b1TimmSEDGPU_tf_efficientnet_b1_ns_4_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b2_TimmSEDGPU_tf_efficientnet_b2_ns_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b3fTimmSEDGPU_tf_efficientnet_b3_ns_2_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/b3fTimmSEDGPU_tf_efficientnet_b3_ns_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/pseudo_TimmSEDGPU_tf_efficientnet_b5_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/pseudo_TimmSEDGPU_tf_efficientnet_b5_ns_1_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/pseudo_TimmSEDGPU_tf_efficientnet_b5_ns_2_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/pseudo_TimmSEDGPU_tf_efficientnet_b5_ns_3_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/e41_oof_TimmSEDGPU_tf_efficientnet_b7_ns_0_256_2048_512_last\"),\n    Path(\"../input/ensemble-01/b1TimmSED_tf_efficientnet_b1_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/e80_TimmSED_tf_efficientnet_b3_ns_0_256_2048_512_best_f1 (1)\"),\n    Path(\"../input/ensemble-01/e63_TimmSED_tf_efficientnet_b5_ns_0_256_2048_512_best_f1\"),\n    Path(\"../input/ensemble-01/e60_TimmSED_tf_efficientnet_b7_ns_0_256_2048_512_best_f1\"),\n    Path('../input/ensemble-01/b1_oof_regular_100epochsTimmSEDGPU_tf_efficientnet_b1_ns_0_256_2048_512_best_f1'),\n    Path('../input/ensemble-01/v2b1_nmels320TimmSEDGPU_tf_efficientnetv2_b1_0_320_2048_512_best_f1'),\n    Path(\"../input/ensemble-01/v2b3_nmels320_psTimmSEDGPU_tf_efficientnetv2_b3_0_320_2048_512_best_f1\")\n]\nmodels = []\nnames = [\n    'v2s_gpu', \n    'v2s_gpu', \n    # 'v2s_gpu', \n    # 'v2s_gpu', \n    'v2s_gpu', \n    'v2s_gpu', \n    'v2s_gpu', \n    'v2s_gpu', \n    'b1_gpu', \n    'b1_gpu', \n    'b1_gpu', \n    'b1_gpu', \n    'b1_gpu', \n    # 'b1_gpu', \n    'b1_gpu', \n    'b1_gpu', \n    #'b1_gpu', \n    'b1_gpu', \n    'b2_gpu', \n    'b3_gpu', 'b3_gpu', \n    'b5_gpu', 'b5_gpu', 'b5_gpu', 'b5_gpu', \n    'b7_gpu', \n    'b1', 'b3', 'b5', 'b7', \n    'b1_gpu', 'b1_gpu_320', 'b3_gpu_320']\n\nfor i in range(len(weights_paths)):\n    if 'gpu' in names[i]: \n        model = load_model(names[i], True, weights_paths[i], conf, \"\", \"best_f1\", 0)\n    else:\n        model = load_model(names[i], False, weights_paths[i], conf, \"\", \"best_f1\", 0)\n\n    models.append(model)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:41:06.820225Z","iopub.execute_input":"2021-06-01T13:41:06.820548Z","iopub.status.idle":"2021-06-01T13:42:00.821216Z","shell.execute_reply.started":"2021-06-01T13:41:06.820519Z","shell.execute_reply":"2021-06-01T13:42:00.820292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data","metadata":{}},{"cell_type":"code","source":"SR = 32000\nF_MIN = 0.0\nF_MAX = SR // 2\nTOP_DB = 80\nMIN_VALUE = -1\n\nclass SoundscapesDataset(torch.utils.data.Dataset):\n    def __init__(self,\n                 data_path,\n                 n_mels,\n                 n_fft,\n                 hop_length,\n                 seconds=20\n                 ):\n        super().__init__()\n        assert 600 % seconds == 0\n        self.data_path = data_path\n        self.n_mels = n_mels\n        self.n_fft = n_fft\n        self.hop_length = hop_length\n        self.names = [f.replace(\".ogg\", \"\") for f in os.listdir(data_path) if f.endswith(\"ogg\")]\n        self.target_columns = CLASSES\n        self.seconds = seconds\n\n    def __len__(self):\n        return len(self.names)\n\n    def as_mel(self, audio: np.ndarray):\n        audio = melspectrogram(audio, sr=SR, n_mels=self.n_mels, hop_length=self.hop_length, n_fft=self.n_fft,\n                               fmax=F_MAX)\n        audio = librosa.power_to_db(audio, ref=np.max, top_db=TOP_DB) / TOP_DB\n        return audio\n\n    def __getitem__(self, idx):\n        file_id = self.names[idx]\n        audio, _ = librosa.load(os.path.join(self.data_path, f\"{file_id}.ogg\"), sr=None)\n        chunk_length = SR * self.seconds\n        chunks = []\n        start_seconds = []\n        \n        for i, start in enumerate(range(0, 600 - self.seconds // 2, self.seconds // 2)):\n            chunk = audio[int(chunk_length // 2 * i):int(chunk_length // 2 * i) + chunk_length]\n            chunks.append(chunk)\n            start_seconds.append(start)\n        chunks = torch.from_numpy(np.array(chunks)).float()\n        start_seconds = torch.from_numpy(np.array(start_seconds))\n\n        return chunks, file_id, start_seconds\n    \nCLASSES = CFG.target_columns\n\nfrom librosa.feature import melspectrogram\n\ndef as_mel(audio):\n    SR = 32000\n    F_MIN = 0.0\n    F_MAX = SR // 2\n    TOP_DB = 80\n    MIN_VALUE = -1\n    n_mels = 256\n    hop_length = 512\n    n_fft = 2048\n    \n    audio = melspectrogram(audio, sr=SR, n_mels=n_mels, hop_length=hop_length, n_fft=n_fft,\n                           fmax=F_MAX)\n    audio = librosa.power_to_db(audio, ref=np.max, top_db=TOP_DB) / TOP_DB\n    return audio","metadata":{"execution":{"iopub.status.busy":"2021-06-01T13:54:39.808075Z","iopub.execute_input":"2021-06-01T13:54:39.808448Z","iopub.status.idle":"2021-06-01T13:54:39.82226Z","shell.execute_reply.started":"2021-06-01T13:54:39.808415Z","shell.execute_reply":"2021-06-01T13:54:39.821293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## infer","metadata":{}},{"cell_type":"code","source":"\nTEST = (len(list(Path(\"../input/birdclef-2021/test_soundscapes/\").glob(\"*.ogg\"))) != 0)\nif TEST:\n    data_path = \"../input/birdclef-2021/test_soundscapes/\"\nelse:\n    data_path = \"../input/birdclef-2021/train_soundscapes/\"\n\n    \nTARGET_SR = 32000\nPERIOD    = 40\ntest_dataset = SoundscapesDataset(\n    data_path=data_path,\n    n_mels=conf[\"n_mels\"],\n    n_fft=conf[\"n_fft\"],\n    hop_length=conf[\"hop_length\"],\n    seconds = PERIOD\n)\n\nprint('Number of models: {}'.format(len(models)))\n\nweights = [0.08165454021559648,\n 0.21907731174089815,\n 0.2224014690785836,\n 0.05246388396627374,\n 0.283327943267436,\n 0.05529993267155051,\n 0.8195556605964778,\n 0.8012870998577118,\n 0.18139450344600117,\n 0.01733149684253748,\n 0.07826873890476346,\n 0.29824756907541783,\n 0.19770169253477266,\n 0.8314541626366747,\n 0.013507043322341372,\n 0.24902706130653013,\n 0.2852405486945102,\n 0.7589629054922609,\n 0.8169482342380096,\n 0.20799895247189154,\n 0.07652219249891853,\n 0.9510807537360569,\n 0.8421765003052626,\n 0.6524239243745986,\n 0.45931291527150997,\n 0.9521269680606435,\n 0.4172814307521808,\n 0.537796980088361,\n 0.07247043664860703]\n\nweights = [x / sum(weights) for x in weights]\n\ndata = {}\nwith torch.no_grad():\n    loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, num_workers=4, shuffle=False)\n    for audio, name, starts in tqdm(loader):\n        starts = starts.detach().cpu().numpy()\n        audio = audio[0]\n        prediction_arrays_even = []\n        prediction_arrays_odd = []\n\n        for frame in range(audio.size(0)):\n\n            probs = None\n            for m_ix, model in enumerate(models):\n                \n                if 'gpu' in names[m_ix]:\n                    mel = audio[frame].cuda()\n                    mel = torch.unsqueeze(mel, 0)\n                else:\n                    mel = torch.from_numpy(as_mel(audio[frame].numpy())).unsqueeze(0).unsqueeze(0).cuda()\n                \n                with torch.cuda.amp.autocast():\n                    output = model(mel)\n                output = output[\"framewise_logit\"]\n                output = torch.sigmoid(output).cpu().cpu().numpy()\n                output = weights[m_ix] * output\n\n                if probs is None:\n                    probs = output\n                else:\n                    probs += output\n\n            probs = probs[0]\n            if frame % 2 == 0:\n                prediction_arrays_even.append(probs)\n            else:\n                prediction_arrays_odd.append(probs)\n                \n        prediction_even = np.concatenate(prediction_arrays_even, axis=0)\n        prediction_odd = np.concatenate(prediction_arrays_odd, axis=0)\n        \n        offset = int(len(prediction_arrays_even[0]) / 2)\n        prediction_even[offset:offset + len(prediction_odd)] = (\n            prediction_even[offset:offset + len(prediction_odd)] + prediction_odd) / 2\n\n        for subframe, split in enumerate(np.array_split(prediction_even, 600 // 5, axis=0)):\n            species = []\n            if species:\n                birds = \" \".join(species)\n            audio_id, site, *_ = name[0].split(\"_\")\n            seconds = str(int((subframe + 1) * 5))\n            row_id = f\"{audio_id}_{site}_{seconds}\"\n            data[row_id] = split[PERIOD * 2:-PERIOD * 2].max(0)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:00:55.870387Z","iopub.execute_input":"2021-06-01T15:00:55.870717Z","iopub.status.idle":"2021-06-01T15:28:52.168335Z","shell.execute_reply.started":"2021-06-01T15:00:55.870687Z","shell.execute_reply":"2021-06-01T15:28:52.167448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## pp","metadata":{}},{"cell_type":"code","source":"\"\"\"\ntrain_soundscapes = pd.read_csv(\"../input/birdclef-2021/train_soundscape_labels.csv\").sort_values(by=\"row_id\").reset_index()\n\npred_dict = data\nnum_classes = len(CLASSES)\n\ngt = np.zeros([len(train_soundscapes), num_classes + 1])\npred_probs = np.zeros([len(train_soundscapes), num_classes])\ngt_ids = np.empty((len(train_soundscapes),), dtype=np.object)\n\nfor i, gt_row in train_soundscapes.iterrows():\n    gt_ids[i] = gt_row.row_id\n    pred_probs[i] = pred_dict[gt_row.row_id]\n    gt_row = gt_row.birds.split()\n    for g in gt_row:\n        if g == \"nocall\":\n            gt[i, num_classes] = 1\n        else:\n            gt[i, CLASSES.index(g)] = 1\n\nnocall_thresholds = np.linspace(0.25, 0.5, num=40)\ncall_thresholds = np.linspace(0.17, 0.35, num=40)\n# nocall_thresholds = np.linspace(0.3, 0.4, num=2)\n# call_thresholds = np.linspace(0.2, 0.3, num=2)\nbest_overall = 0\nbest_sum_overall = 0\nbest_calls_f1 = 0\nfor nct in nocall_thresholds:\n    for ct in call_thresholds:\n        # init nocalls\n        pred_nocall = np.zeros([len(train_soundscapes), ])\n        nct_preds = pred_probs > nct\n        nct_preds = nct_preds.sum(1) == 0\n\n        #set to nocalls and use different threshold for others\n        preds = np.zeros_like(gt)\n        preds[:, :num_classes] = pred_probs > ct\n        preds[:, num_classes] = nct_preds\n\n        gt_calls = gt[gt[:, num_classes] == 0]\n        pred_gt_calls = preds[gt[:, num_classes] == 0]\n        calls_score = f1_score(gt_calls, pred_gt_calls, average=\"samples\")\n        gt_nocalls = gt[gt[:, num_classes] == 1]\n        pred_gt_nocalls = preds[gt[:, num_classes] == 1]\n        nocalls_score = f1_score(gt_nocalls, pred_gt_nocalls, average=\"samples\")\n        f1_overall = 0.54 * nocalls_score + 0.46 * calls_score\n        if f1_overall > best_overall:\n            ## save predictions for submission\n            \n            saved_best_ct = ct\n            saved_best_nct = nct\n            best_overall = f1_overall\n            print(\"##########found better thresholds#########\")\n            print(f\"Call/Nocall Cls t: {nct:.04f} : Call T {ct:0.4f}\")\n            print(f\"F1 calls:{calls_score:.04f} | F1 nocalls:{nocalls_score:.04f} \")\n            print(f\"Overall {f1_overall :.04f}\")\n        sum_overall = f1_overall + calls_score\n        if sum_overall > best_sum_overall:\n            print(\"##########alternative thresholds#########\")\n            print(f\"Call/Nocall Cls t: {nct:.04f} : Call T {ct:0.4f}\")\n            print(f\"F1 calls:{calls_score:.04f} | F1 nocalls:{nocalls_score:.04f} \")\n            print(f\"Overall {f1_overall :.04f}\")\n            print(f\"sum of f1 and calls_score {sum_overall:.04f}\")\n            best_sum_overall = sum_overall\n        if calls_score > best_calls_f1:\n            print(\"##########found best calls f1#########\")\n            print(f\"Call/Nocall Cls t: {nct:.04f} : Call T {ct:0.4f}\")\n            print(f\"F1 calls:{calls_score:.04f} | F1 nocalls:{nocalls_score:.04f} \")\n            print(f\"Overall {f1_overall :.04f}\")\n            best_calls_f1 = calls_score\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:47:21.807682Z","iopub.execute_input":"2021-06-01T15:47:21.808046Z","iopub.status.idle":"2021-06-01T15:47:21.882752Z","shell.execute_reply.started":"2021-06-01T15:47:21.807967Z","shell.execute_reply":"2021-06-01T15:47:21.881509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_ids = []\nbirds   = []\n\n# nct = saved_best_nct\n# ct  = saved_best_ct\nnct = 0.3077\nct  = 0.2485   \n\nmasks = dict(np.load(\"../input/train-masks/masks.npz\"))\n\nfor i, (k, v) in enumerate(data.items()):\n    row_ids.append(k)\n\n    if np.sum(v > nct) == 0:\n        birds.append(\"nocall\")\n    else:\n        site = k.split(\"_\")[1]\n        mask = masks[site]\n        v = v * mask\n\n        pds = [CFG.target_columns[x[0]] for x in np.argwhere(v > ct).tolist()]\n        birds.append(\" \".join(sorted(list(pds))))","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:34:39.23292Z","iopub.execute_input":"2021-06-01T15:34:39.233253Z","iopub.status.idle":"2021-06-01T15:34:39.295389Z","shell.execute_reply.started":"2021-06-01T15:34:39.233221Z","shell.execute_reply":"2021-06-01T15:34:39.294625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n        \"row_id\": row_ids,\n        \"birds\": birds,\n    }).to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:34:42.331868Z","iopub.execute_input":"2021-06-01T15:34:42.33222Z","iopub.status.idle":"2021-06-01T15:34:42.472647Z","shell.execute_reply.started":"2021-06-01T15:34:42.33217Z","shell.execute_reply":"2021-06-01T15:34:42.471904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:29:20.197731Z","iopub.execute_input":"2021-05-30T20:29:20.198068Z","iopub.status.idle":"2021-05-30T20:29:20.218513Z","shell.execute_reply.started":"2021-05-30T20:29:20.198035Z","shell.execute_reply":"2021-05-30T20:29:20.21691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}