{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n\nIn this notebook, I will show the way to train the model used here:\nhttps://www.kaggle.com/hidehisaarai1213/pytorch-inference-birdclef2021-starter\n\nNote that by default this notebook will only train the model one epoch, but [the weight](https://www.kaggle.com/hidehisaarai1213/birdclef2021-effnetb0-starter-weight) I used was obtained after 31epochs of training.\n"},{"metadata":{},"cell_type":"markdown","source":"## Dependencies"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/timm-pytorch-image-models/pytorch-image-models-master/\n!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install catalyst==20.12","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport math\nimport random\nimport warnings\n\nimport albumentations as A\nimport cv2\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\n\nfrom pathlib import Path\nfrom typing import List\n\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom catalyst.core import Callback, CallbackOrder, IRunner\nfrom catalyst.dl import Runner, SupervisedRunner\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom timm.models.layers import SelectAdaptivePool2d\nfrom torch.optim.optimizer import Optimizer\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    seed = 1213\n    epochs = 35\n    train = True\n    folds = [0]\n    img_size = 224\n    main_metric = \"epoch_f1_at_05\"\n    minimize_metric = False\n\n    ######################\n    # Data #\n    ######################\n    train_datadir = Path(\"../input/birdclef-2021/train_short_audio\")\n    train_csv = \"../input/birdclef-2021/train_metadata.csv\"\n    train_soundscape = \"../input/birdclef-2021/train_soundscape_labels.csv\"\n\n    ######################\n    # Dataset #\n    ######################\n    transforms = {\n        \"train\": [{\"name\": \"Normalize\"}],\n        \"valid\": [{\"name\": \"Normalize\"}]\n    }\n    period = 20\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n    ######################\n    # Loaders #\n    ######################\n    loader_params = {\n        \"train\": {\n            \"batch_size\": 32,\n            \"num_workers\": 4,\n            \"shuffle\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 4,\n            \"shuffle\": False\n        }\n    }\n\n    ######################\n    # Split #\n    ######################\n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 1213\n    }\n\n    ######################\n    # Model #\n    ######################\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n    ######################\n    # Criterion #\n    ######################\n    loss_name = \"BCEFocal2WayLoss\"\n    loss_params: dict = {}\n\n    ######################\n    # Optimizer #\n    ######################\n    optimizer_name = \"Adam\"\n    base_optimizer = \"Adam\"\n    optimizer_params = {\n        \"lr\": 0.001\n    }\n    # For SAM optimizer\n    base_optimizer = \"Adam\"\n\n    ######################\n    # Scheduler #\n    ######################\n    scheduler_name = \"CosineAnnealingLR\"\n    scheduler_params = {\n        \"T_max\": 10\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this notebook is by default run on debug mode (only train one epoch).\n# If you'd like to get the results on par with that of inference notebook, you'll need to train the model around 30 epochs\nDEBUG = True\nif DEBUG:\n    CFG.epochs = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\ndef get_device() -> torch.device:\n    return torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\ndef init_logger(log_file='train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset and Data Augmentations\n\nIn this section, I define dataset that crops 20 second chunk. The output of this dataset is a pair of waveform and corresponding label."},{"metadata":{"trusted":true},"cell_type":"code","source":"class WaveformDataset(torchdata.Dataset):\n    def __init__(self,\n                 df: pd.DataFrame,\n                 datadir: Path,\n                 img_size=224,\n                 waveform_transforms=None,\n                 period=20,\n                 validation=False):\n        self.df = df\n        self.datadir = datadir\n        self.img_size = img_size\n        self.waveform_transforms = waveform_transforms\n        self.period = period\n        self.validation = validation\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        sample = self.df.loc[idx, :]\n        wav_name = sample[\"filename\"]\n        ebird_code = sample[\"primary_label\"]\n\n        y, sr = sf.read(self.datadir / ebird_code / wav_name)\n\n        len_y = len(y)\n        effective_length = sr * self.period\n        if len_y < effective_length:\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            if not self.validation:\n                start = np.random.randint(effective_length - len_y)\n            else:\n                start = 0\n            new_y[start:start + len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            if not self.validation:\n                start = np.random.randint(len_y - effective_length)\n            else:\n                start = 0\n            y = y[start:start + effective_length].astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        y = np.nan_to_num(y)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n\n        y = np.nan_to_num(y)\n\n        labels = np.zeros(len(CFG.target_columns), dtype=float)\n        labels[CFG.target_columns.index(ebird_code)] = 1.0\n\n        return {\n            \"image\": y,\n            \"targets\": labels\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(phase: str):\n    transforms = CFG.transforms\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if globals().get(trns_name) is not None:\n                trns_cls = globals()[trns_name]\n                trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return Compose(trns_list)\n        else:\n            return None\n        \n        \nclass Normalize:\n    def __call__(self, y: np.ndarray):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Definition\n\nIn this notebook, I will use a model for SED task."},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes\n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(\n                self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=CFG.n_fft, hop_length=CFG.hop_length,\n                                                 win_length=CFG.n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=CFG.sample_rate, n_fft=CFG.n_fft,\n                                                 n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax, ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        # (batch_size, 1, time_steps, freq_bins)\n        x = self.spectrogram_extractor(input)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"__CRITERIONS__ = {\n    \"BCEFocalLoss\": BCEFocalLoss,\n    \"BCEFocal2WayLoss\": BCEFocal2WayLoss\n}\n\n\ndef get_criterion():\n    if hasattr(nn, CFG.loss_name):\n        return nn.__getattribute__(CFG.loss_name)(**CFG.loss_params)\n    elif __CRITERIONS__.get(CFG.loss_name) is not None:\n        return __CRITERIONS__[CFG.loss_name](**CFG.loss_params)\n    else:\n        raise NotImplementedError","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Utilities\n\nOptimizers, Schedulers, Callbacks and the Runner"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Custom optimizer\n__OPTIMIZERS__ = {}\n\n\ndef get_optimizer(model: nn.Module):\n    optimizer_name = CFG.optimizer_name\n    if optimizer_name == \"SAM\":\n        base_optimizer_name = CFG.base_optimizer\n        if __OPTIMIZERS__.get(base_optimizer_name) is not None:\n            base_optimizer = __OPTIMIZERS__[base_optimizer_name]\n        else:\n            base_optimizer = optim.__getattribute__(base_optimizer_name)\n        return SAM(model.parameters(), base_optimizer, **CFG.optimizer_params)\n\n    if __OPTIMIZERS__.get(optimizer_name) is not None:\n        return __OPTIMIZERS__[optimizer_name](model.parameters(),\n                                              **CFG.optimizer_params)\n    else:\n        return optim.__getattribute__(optimizer_name)(model.parameters(),\n                                                      **CFG.optimizer_params)\n\n\ndef get_scheduler(optimizer):\n    scheduler_name = CFG.scheduler_name\n\n    if scheduler_name is None:\n        return\n    else:\n        return optim.lr_scheduler.__getattribute__(scheduler_name)(\n            optimizer, **CFG.scheduler_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SchedulerCallback(Callback):\n    def __init__(self):\n        super().__init__(CallbackOrder.Scheduler)\n\n    def on_loader_end(self, state: IRunner):\n        lr = state.scheduler.get_last_lr()\n        state.epoch_metrics[\"lr\"] = lr[0]\n        if state.is_train_loader:\n            state.scheduler.step()\n\n\nclass SampleF1Callback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 prefix: str = \"f1\",\n                 threshold=0.5):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.prefix = prefix\n        self.threshold = threshold\n\n    def on_loader_start(self, state: IRunner):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: IRunner):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[\"clipwise_output\"].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output > self.threshold\n        score = metrics.f1_score(targ, y_pred, average=\"samples\")\n\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: IRunner):\n        y_pred = np.concatenate(self.prediction, axis=0) > self.threshold\n        y_true = np.concatenate(self.target, axis=0)\n        score = metrics.f1_score(y_true, y_pred, average=\"samples\")\n\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: IRunner):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: IRunner):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        try:\n            score = metrics.average_precision_score(\n                targ, clipwise_output, average=None)\n        except ValueError:\n            import pdb\n            pdb.set_trace()\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: IRunner):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = metrics.average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\ndef get_callbacks():\n    return [\n        SampleF1Callback(prefix=\"f1_at_05\", threshold=0.5),\n        SampleF1Callback(prefix=\"f1_at_03\", threshold=0.3),\n        SampleF1Callback(prefix=\"f1_at_07\", threshold=0.7),\n        mAPCallback()\n    ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_runner(device: torch.device):\n    return SupervisedRunner(device=device, input_key=\"image\", input_target_key=\"targets\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training!"},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n\nlogdir = Path(\"out\")\nlogdir.mkdir(exist_ok=True, parents=True)\nif (logdir / \"train.log\").exists():\n    os.remove(logdir / \"train.log\")\nlogger = init_logger(log_file=logdir / \"train.log\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# environment\nset_seed(CFG.seed)\ndevice = get_device()\n\n# validation\nsplitter = getattr(model_selection, CFG.split)(**CFG.split_params)\n\n# data\ntrain = pd.read_csv(CFG.train_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# main loop\nfor i, (trn_idx, val_idx) in enumerate(splitter.split(train, y=train[\"primary_label\"])):\n    if i not in CFG.folds:\n        continue\n    logger.info(\"=\" * 120)\n    logger.info(f\"Fold {i} Training\")\n    logger.info(\"=\" * 120)\n\n    trn_df = train.loc[trn_idx, :].reset_index(drop=True)\n    val_df = train.loc[val_idx, :].reset_index(drop=True)\n\n    loaders = {\n        phase: torchdata.DataLoader(\n            WaveformDataset(\n                df_,\n                CFG.train_datadir,\n                img_size=CFG.img_size,\n                waveform_transforms=get_transforms(phase),\n                period=CFG.period,\n                validation=(phase == \"valid\")\n            ),\n            **CFG.loader_params[phase])  # type: ignore\n        for phase, df_ in zip([\"train\", \"valid\"], [trn_df, val_df])\n    }\n\n    model = TimmSED(\n        base_model_name=CFG.base_model_name,\n        pretrained=CFG.pretrained,\n        num_classes=CFG.num_classes,\n        in_channels=CFG.in_channels)\n    criterion = get_criterion()\n    optimizer = get_optimizer(model)\n    scheduler = get_scheduler(optimizer)\n    callbacks = get_callbacks()\n    runner = get_runner(device)\n    runner.train(\n        model=model,\n        criterion=criterion,\n        loaders=loaders,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        num_epochs=CFG.epochs,\n        verbose=True,\n        logdir=logdir / f\"fold{i}\",\n        callbacks=callbacks,\n        main_metric=CFG.main_metric,\n        minimize_metric=CFG.minimize_metric)\n\n    del model, optimizer, scheduler\n    gc.collect()\n    torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}