{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Birds call identification\n\n![](https://www.lovethegarden.com/sites/default/files/styles/header_image_xl/public/content/articles/UK_community-fun-facts-19-common-british-birds-you-can-find-your-garden_header.jpg?itok=MweJ55ij)","metadata":{}},{"cell_type":"code","source":"!pip install ../input/timm-pytorch-image-models/pytorch-image-models-master/\n!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:29:02.620278Z","iopub.execute_input":"2021-05-23T11:29:02.620678Z","iopub.status.idle":"2021-05-23T11:29:18.518199Z","shell.execute_reply.started":"2021-05-23T11:29:02.620588Z","shell.execute_reply":"2021-05-23T11:29:18.517319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import library","metadata":{}},{"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\nfrom tqdm import tqdm","metadata":{"papermill":{"duration":5.756505,"end_time":"2021-04-05T05:29:06.023023","exception":false,"start_time":"2021-04-05T05:29:00.266518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:22.952484Z","iopub.execute_input":"2021-05-23T11:29:22.952819Z","iopub.status.idle":"2021-05-23T11:29:28.424411Z","shell.execute_reply.started":"2021-05-23T11:29:22.952789Z","shell.execute_reply":"2021-05-23T11:29:28.42235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)","metadata":{"papermill":{"duration":0.134417,"end_time":"2021-04-05T05:29:06.284947","exception":false,"start_time":"2021-04-05T05:29:06.15053","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:33.1808Z","iopub.execute_input":"2021-05-23T11:29:33.181167Z","iopub.status.idle":"2021-05-23T11:29:33.190953Z","shell.execute_reply.started":"2021-05-23T11:29:33.181132Z","shell.execute_reply":"2021-05-23T11:29:33.189842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logger = get_logger(\"main.log\")\nset_seed(1213)","metadata":{"papermill":{"duration":0.137748,"end_time":"2021-04-05T05:29:06.556925","exception":false,"start_time":"2021-04-05T05:29:06.419177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:37.726258Z","iopub.execute_input":"2021-05-23T11:29:37.7266Z","iopub.status.idle":"2021-05-23T11:29:37.73684Z","shell.execute_reply.started":"2021-05-23T11:29:37.72657Z","shell.execute_reply":"2021-05-23T11:29:37.736094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Path directory and hyperparameter","metadata":{}},{"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    seed = 1213\n    epochs = 35\n    train = True\n    folds = [0]\n    img_size = 224\n    main_metric = \"epoch_f1_at_05\"\n    minimize_metric = False\n\n    ######################\n    # Data #\n    ######################\n    train_datadir = Path(\"../input/birdclef-2021/train_short_audio\")\n    train_csv = \"../input/birdclef-2021/train_metadata.csv\"\n    train_soundscape = \"../input/birdclef-2021/train_soundscape_labels.csv\"\n\n    ######################\n    # Dataset #\n    ######################\n    transforms = {\n        \"train\": [{\"name\": \"Normalize\"}],\n        \"valid\": [{\"name\": \"Normalize\"}],\n        \"test\": [{\"name\": \"Normalize\"}]\n    }\n    period = 20\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n    ######################\n    # Loaders #\n    ######################\n    loader_params = {\n        \"train\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        },\n        \"test\": {\n            \"batch_size\": 64,\n            \"num_workers\": 20,\n            \"shuffle\": False\n        }\n    }\n\n    ######################\n    # Split #\n    ######################\n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 1213\n    }\n\n    ######################\n    # Model #\n    ######################\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n    ######################\n    # Criterion #\n    ######################\n    loss_name = \"BCEFocal2WayLoss\"\n    loss_params: dict = {}\n\n    ######################\n    # Optimizer #\n    ######################\n    optimizer_name = \"Adam\"\n    base_optimizer = \"Adam\"\n    optimizer_params = {\n        \"lr\": 0.001\n    }\n    # For SAM optimizer\n    base_optimizer = \"Adam\"\n\n    ######################\n    # Scheduler #\n    ######################\n    scheduler_name = \"CosineAnnealingLR\"\n    scheduler_params = {\n        \"T_max\": 10\n    }","metadata":{"papermill":{"duration":0.152123,"end_time":"2021-04-05T05:29:06.828528","exception":false,"start_time":"2021-04-05T05:29:06.676405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:41.973592Z","iopub.execute_input":"2021-05-23T11:29:41.973906Z","iopub.status.idle":"2021-05-23T11:29:41.999925Z","shell.execute_reply.started":"2021-05-23T11:29:41.973875Z","shell.execute_reply":"2021-05-23T11:29:41.998969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_SR = 32000\nTEST = (len(list(Path(\"../input/birdclef-2021/test_soundscapes/\").glob(\"*.ogg\"))) != 0)\nif TEST:\n    DATADIR = Path(\"../input/birdclef-2021/test_soundscapes/\")\nelse:\n    DATADIR = Path(\"../input/birdclef-2021/train_soundscapes/\")","metadata":{"papermill":{"duration":0.133908,"end_time":"2021-04-05T05:29:07.081445","exception":false,"start_time":"2021-04-05T05:29:06.947537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:53.726916Z","iopub.execute_input":"2021-05-23T11:29:53.727297Z","iopub.status.idle":"2021-05-23T11:29:53.738994Z","shell.execute_reply.started":"2021-05-23T11:29:53.727263Z","shell.execute_reply":"2021-05-23T11:29:53.738086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_audios = list(DATADIR.glob(\"*.ogg\"))\nall_audio_ids = [\"_\".join(audio_id.name.split(\"_\")[:2]) for audio_id in all_audios]\nsubmission_df = pd.DataFrame({\n    \"row_id\": all_audio_ids\n})\nsubmission_df","metadata":{"papermill":{"duration":0.15113,"end_time":"2021-04-05T05:29:07.359792","exception":false,"start_time":"2021-04-05T05:29:07.208662","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:29:56.835218Z","iopub.execute_input":"2021-05-23T11:29:56.835545Z","iopub.status.idle":"2021-05-23T11:29:56.864782Z","shell.execute_reply.started":"2021-05-23T11:29:56.835514Z","shell.execute_reply":"2021-05-23T11:29:56.863973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes\n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(\n                self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\ndroprate=0.5 #0.5\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=CFG.n_fft, hop_length=CFG.hop_length,\n                                                 win_length=CFG.n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=CFG.sample_rate, n_fft=CFG.n_fft,\n                                                 n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax, ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        # (batch_size, 1, time_steps, freq_bins)\n        x = self.spectrogram_extractor(input)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=droprate, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=droprate, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","metadata":{"papermill":{"duration":0.16476,"end_time":"2021-04-05T05:29:07.645338","exception":false,"start_time":"2021-04-05T05:29:07.480578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:03.075402Z","iopub.execute_input":"2021-05-23T11:30:03.07573Z","iopub.status.idle":"2021-05-23T11:30:03.114093Z","shell.execute_reply.started":"2021-05-23T11:30:03.075699Z","shell.execute_reply":"2021-05-23T11:30:03.113237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model creation","metadata":{}},{"cell_type":"code","source":"class TestDataset(torchdata.Dataset):\n    def __init__(self, df: pd.DataFrame, clip: np.ndarray,\n                 waveform_transforms=None):\n        self.df = df\n        self.clip = clip\n        self.waveform_transforms=waveform_transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = 32000\n        sample = self.df.loc[idx, :]\n        row_id = sample.row_id\n\n        end_seconds = int(sample.seconds)\n        start_seconds = int(end_seconds - 5)\n\n        start_index = SR * start_seconds\n        end_index = SR * end_seconds\n\n        y = self.clip[start_index:end_index].astype(np.float32)\n\n        y = np.nan_to_num(y)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n\n        y = np.nan_to_num(y)\n\n        return y, row_id","metadata":{"papermill":{"duration":0.130478,"end_time":"2021-04-05T05:29:07.894491","exception":false,"start_time":"2021-04-05T05:29:07.764013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:20.870175Z","iopub.execute_input":"2021-05-23T11:30:20.870524Z","iopub.status.idle":"2021-05-23T11:30:20.879306Z","shell.execute_reply.started":"2021-05-23T11:30:20.870489Z","shell.execute_reply":"2021-05-23T11:30:20.8784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# images Masking","metadata":{}},{"cell_type":"code","source":"def get_transforms(phase: str):\n    transforms = CFG.transforms\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if globals().get(trns_name) is not None:\n                trns_cls = globals()[trns_name]\n                trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return Compose(trns_list)\n        else:\n            return None\n\n\ndef get_waveform_transforms(config: dict, phase: str):\n    return get_transforms(config, phase)\n\n\ndef get_spectrogram_transforms(config: dict, phase: str):\n    transforms = config.get('spectrogram_transforms')\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if hasattr(A, trns_name):\n                trns_cls = A.__getattribute__(trns_name)\n                trns_list.append(trns_cls(**trns_params))\n            else:\n                trns_cls = globals().get(trns_name)\n                if trns_cls is not None:\n                    trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return A.Compose(trns_list, p=1.0)\n        else:\n            return None\n\n\nclass Normalize:\n    def __call__(self, y: np.ndarray):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)\n\n\nclass NewNormalize:\n    def __call__(self, y: np.ndarray):\n        y_mm = y - y.mean()\n        return y_mm / y_mm.abs().max()\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass NoiseInjection(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_level=0.5, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.noise_level = (0.0, max_noise_level)\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        noise_level = np.random.uniform(*self.noise_level)\n        noise = np.random.randn(len(y))\n        augmented = (y + noise * noise_level).astype(y.dtype)\n        return augmented\n\n\nclass GaussianNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PitchShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_range=5, sr=32000):\n        super().__init__(always_apply, p)\n        self.max_range = max_range\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        n_steps = np.random.randint(-self.max_range, self.max_range)\n        augmented = librosa.effects.pitch_shift(y, self.sr, n_steps)\n        return augmented\n\n\nclass TimeStretch(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_rate=1, sr=32000):\n        super().__init__(always_apply, p)\n        self.max_rate = max_rate\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        rate = np.random.uniform(0, self.max_rate)\n        augmented = librosa.effects.time_stretch(y, rate)\n        return augmented\n\n\ndef _db2float(db: float, amplitude=True):\n    if amplitude:\n        return 10**(db / 20)\n    else:\n        return 10 ** (db / 10)\n\n\ndef volume_down(y: np.ndarray, db: float):\n    \"\"\"\n    Low level API for decreasing the volume\n    Parameters\n    ----------\n    y: numpy.ndarray\n        stereo / monaural input audio\n    db: float\n        how much decibel to decrease\n    Returns\n    -------\n    applied: numpy.ndarray\n        audio with decreased volume\n    \"\"\"\n    applied = y * _db2float(-db)\n    return applied\n\n\ndef volume_up(y: np.ndarray, db: float):\n    \"\"\"\n    Low level API for increasing the volume\n    Parameters\n    ----------\n    y: numpy.ndarray\n        stereo / monaural input audio\n    db: float\n        how much decibel to increase\n    Returns\n    -------\n    applied: numpy.ndarray\n        audio with increased volume\n    \"\"\"\n    applied = y * _db2float(db)\n    return applied\n\n\nclass RandomVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        if db >= 0:\n            return volume_up(y, db)\n        else:\n            return volume_down(y, db)\n\n\nclass OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        y = trns(y)\n        return y\n\n\nclass CosineVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n        dbs = _db2float(cosine * db)\n        return y * dbs\n\n\ndef drop_stripes(image: np.ndarray, dim: int, drop_width: int, stripes_num: int):\n    total_width = image.shape[dim]\n    lowest_value = image.min()\n    for _ in range(stripes_num):\n        distance = np.random.randint(low=0, high=drop_width, size=(1,))[0]\n        begin = np.random.randint(\n            low=0, high=total_width - distance, size=(1,))[0]\n\n        if dim == 0:\n            image[begin:begin + distance] = lowest_value\n        elif dim == 1:\n            image[:, begin + distance] = lowest_value\n        elif dim == 2:\n            image[:, :, begin + distance] = lowest_value\n    return image\n\n\nclass TimeFreqMasking(ImageOnlyTransform):\n    def __init__(self,\n                 time_drop_width: int,\n                 time_stripes_num: int,\n                 freq_drop_width: int,\n                 freq_stripes_num: int,\n                 always_apply=False,\n                 p=0.5):\n        super().__init__(always_apply, p)\n        self.time_drop_width = time_drop_width\n        self.time_stripes_num = time_stripes_num\n        self.freq_drop_width = freq_drop_width\n        self.freq_stripes_num = freq_stripes_num\n\n    def apply(self, img, **params):\n        img_ = img.copy()\n        if img.ndim == 2:\n            img_ = drop_stripes(\n                img_, dim=0, drop_width=self.freq_drop_width, stripes_num=self.freq_stripes_num)\n            img_ = drop_stripes(\n                img_, dim=1, drop_width=self.time_drop_width, stripes_num=self.time_stripes_num)\n        return img_","metadata":{"papermill":{"duration":0.169635,"end_time":"2021-04-05T05:29:08.182012","exception":false,"start_time":"2021-04-05T05:29:08.012377","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:31.115445Z","iopub.execute_input":"2021-05-23T11:30:31.115909Z","iopub.status.idle":"2021-05-23T11:30:31.18333Z","shell.execute_reply.started":"2021-05-23T11:30:31.11586Z","shell.execute_reply":"2021-05-23T11:30:31.182312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model inference","metadata":{}},{"cell_type":"code","source":"def prepare_model_for_inference(model, path: Path):\n    if not torch.cuda.is_available():\n        ckpt = torch.load(path, map_location=\"cpu\")\n    else:\n        ckpt = torch.load(path)\n    model.load_state_dict(ckpt[\"model_state_dict\"])\n    model.eval()\n    return model","metadata":{"papermill":{"duration":0.12811,"end_time":"2021-04-05T05:29:08.429548","exception":false,"start_time":"2021-04-05T05:29:08.301438","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:45.894172Z","iopub.execute_input":"2021-05-23T11:30:45.894495Z","iopub.status.idle":"2021-05-23T11:30:45.899852Z","shell.execute_reply.started":"2021-05-23T11:30:45.894459Z","shell.execute_reply":"2021-05-23T11:30:45.898664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame, \n                        clip: np.ndarray, \n                        model, \n                        threshold=0.5):\n\n    dataset = TestDataset(df=test_df, \n                          clip=clip,\n                          waveform_transforms=get_transforms(phase=\"test\"))\n    loader = torchdata.DataLoader(dataset, batch_size=1, shuffle=False)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    prediction_dict = {}\n    for image, row_id in tqdm(loader):\n        row_id = row_id[0]\n        image = image.to(device)\n\n        with torch.no_grad():\n            prediction = model(image)\n            proba = prediction[\"clipwise_output\"].detach().cpu().numpy().reshape(-1)\n\n        events = proba >= threshold\n        labels = np.argwhere(events).reshape(-1).tolist()\n\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: CFG.target_columns[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict","metadata":{"papermill":{"duration":0.133183,"end_time":"2021-04-05T05:29:08.680328","exception":false,"start_time":"2021-04-05T05:29:08.547145","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:50.012703Z","iopub.execute_input":"2021-05-23T11:30:50.013055Z","iopub.status.idle":"2021-05-23T11:30:50.024143Z","shell.execute_reply.started":"2021-05-23T11:30:50.013024Z","shell.execute_reply":"2021-05-23T11:30:50.023292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model predictions","metadata":{}},{"cell_type":"code","source":"def prediction(test_audios,\n               weights_path: Path,\n               threshold=0.5):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model = TimmSED(base_model_name=CFG.base_model_name,\n                    pretrained=False,\n                    num_classes=CFG.num_classes,\n                    in_channels=CFG.in_channels)\n    model = prepare_model_for_inference(model, weights_path).to(device)\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_path in test_audios:\n        with timer(f\"Loading {str(audio_path)}\", logger):\n            clip, _ = sf.read(audio_path)\n\n        seconds = []\n        row_ids = []\n        for second in range(5, 605, 5):\n            row_id = \"_\".join(audio_path.name.split(\"_\")[:2]) + f\"_{second}\"\n            seconds.append(second)\n            row_ids.append(row_id)\n            \n        test_df = pd.DataFrame({\n            \"row_id\": row_ids,\n            \"seconds\": seconds\n        })\n        with timer(f\"Prediction on {audio_path}\", logger):\n            prediction_dict = prediction_for_clip(test_df,\n                                                  clip=clip,\n                                                  model=model,\n                                                  threshold=threshold)\n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","metadata":{"papermill":{"duration":0.133587,"end_time":"2021-04-05T05:29:08.93102","exception":false,"start_time":"2021-04-05T05:29:08.797433","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:30:55.668314Z","iopub.execute_input":"2021-05-23T11:30:55.668639Z","iopub.status.idle":"2021-05-23T11:30:55.681154Z","shell.execute_reply.started":"2021-05-23T11:30:55.668609Z","shell.execute_reply":"2021-05-23T11:30:55.679357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"th=0.30 #0.5\nweights_path = Path(\"../input/birdclef2021-effnetb0-starter-weight/best.pth\")\nsubmission = prediction(test_audios=all_audios,\n                        weights_path=weights_path,\n                        threshold=th)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":57.547017,"end_time":"2021-04-05T05:30:06.596454","exception":false,"start_time":"2021-04-05T05:29:09.049437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:31:02.521086Z","iopub.execute_input":"2021-05-23T11:31:02.521411Z","iopub.status.idle":"2021-05-23T11:31:57.871625Z","shell.execute_reply.started":"2021-05-23T11:31:02.521379Z","shell.execute_reply":"2021-05-23T11:31:57.870836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.251644,"end_time":"2021-04-05T05:30:07.080742","exception":false,"start_time":"2021-04-05T05:30:06.829098","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-23T11:32:55.470704Z","iopub.execute_input":"2021-05-23T11:32:55.471088Z","iopub.status.idle":"2021-05-23T11:32:55.499048Z","shell.execute_reply.started":"2021-05-23T11:32:55.471046Z","shell.execute_reply":"2021-05-23T11:32:55.498128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style= 'color: red'> Upvote pls. </h1>","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}