{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hi. I wanted to run the [great notebook](https://www.kaggle.com/hidehisaarai1213/pytorch-training-birdclef2021-starter) by [Hidehisa Arai](https://www.kaggle.com/hidehisaarai1213) but the Kaggle environment doesn't provide sufficient resources to run it for longer time so I decided to shift the code to Collab Notebook. Also thanks to [kkiller](https://www.kaggle.com/kneroma) for his [notebook](https://www.kaggle.com/kneroma/clean-fast-simple-bird-identifier-training-colab/output) which provides good code for setting up things on collab. We can train different models like EfficientNet0, EfficientNet1 and so on for longer times and they have proven to be best when we look into the past competitions solutions. ","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/drive')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install --upgrade --force-reinstall --no-deps  kaggle > /dev/null\n! mkdir ~/.kaggle\n! cp \"/content/drive/My Drive/Kaggle/kaggle.json\" ~/.kaggle/\n! chmod 600 ~/.kaggle/kaggle.json","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport os\nif not os.path.exists(\"/content/input\"):\n  !mkdir input\n  !kaggle datasets download -d kozodoi/timm-pytorch-image-models\n  !unzip /content//timm-pytorch-image-models.zip -d input/\n  !rm -rf /content//timm-pytorch-image-models.zip","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not os.path.exists(\"/content/input/torchlibrosa\"):\n  !mkdir input/torchlibrosa\n  !kaggle datasets download -d kozistr/torchlibrosa\n  !unzip /content//torchlibrosa.zip -d input/torchlibrosa\n  !rm -rf /content//torchlibrosa.zip","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n!mkdir input/birdclef-2021\n!kaggle competitions download -c birdclef-2021\n!unzip /content//birdclef-2021.zip -d input/birdclef-2021\n!rm -rf /content//birdclef-2021.zip","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /content/input/pytorch-image-models-master\n!pip install /content/input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl\n!pip install catalyst==20.12\n!pip install git+https://github.com/albumentations-team/albumentations.git","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport math\nimport random\nimport warnings\n\nimport albumentations as A\nimport cv2\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\n\nfrom pathlib import Path\nfrom typing import List\n\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom catalyst.core import Callback, CallbackOrder, IRunner\nfrom catalyst.dl import Runner, SupervisedRunner\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom timm.models.layers import SelectAdaptivePool2d\nfrom torch.optim.optimizer import Optimizer\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    seed = 1213\n    epochs = 35\n    train = True\n    folds = [0]\n    img_size = 224\n    main_metric = \"epoch_f1_at_05\"\n    minimize_metric = False\n\n    ######################\n    # Data #\n    ######################\n    train_datadir = Path(\"/content/input/birdclef-2021/train_short_audio\")\n    train_csv = \"/content/input/birdclef-2021/train_metadata.csv\"\n    train_soundscape = \"/content/input/birdclef-2021/train_soundscape_labels.csv\"\n\n    ######################\n    # Dataset #\n    ######################\n    transforms = {\n        \"train\": [{\"name\": \"Normalize\"}],\n        \"valid\": [{\"name\": \"Normalize\"}]\n    }\n    period = 20\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n    ######################\n    # Loaders #\n    ######################\n    loader_params = {\n        \"train\": {\n            \"batch_size\": 32,\n            \"num_workers\": 4,\n            \"shuffle\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 4,\n            \"shuffle\": False\n        }\n    }\n\n    ######################\n    # Split #\n    ######################\n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 1213\n    }\n\n    ######################\n    # Model #\n    ######################\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n    ######################\n    # Criterion #\n    ######################\n    loss_name = \"BCEFocal2WayLoss\"\n    loss_params: dict = {}\n\n    ######################\n    # Optimizer #\n    ######################\n    optimizer_name = \"Adam\"\n    base_optimizer = \"Adam\"\n    optimizer_params = {\n        \"lr\": 0.001\n    }\n    # For SAM optimizer\n    base_optimizer = \"Adam\"\n\n    ######################\n    # Scheduler #\n    ######################\n    scheduler_name = \"CosineAnnealingLR\"\n    scheduler_params = {\n        \"T_max\": 10\n    }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nif DEBUG:\n    CFG.epochs = 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\ndef get_device() -> torch.device:\n    return torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\ndef init_logger(log_file='train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WaveformDataset(torchdata.Dataset):\n    def __init__(self,\n                 df: pd.DataFrame,\n                 datadir: Path,\n                 img_size=224,\n                 waveform_transforms=None,\n                 period=20,\n                 validation=False):\n        self.df = df\n        self.datadir = datadir\n        self.img_size = img_size\n        self.waveform_transforms = waveform_transforms\n        self.period = period\n        self.validation = validation\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        sample = self.df.loc[idx, :]\n        wav_name = sample[\"filename\"]\n        ebird_code = sample[\"primary_label\"]\n\n        y, sr = sf.read(self.datadir / ebird_code / wav_name)\n\n        len_y = len(y)\n        effective_length = sr * self.period\n        if len_y < effective_length:\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            if not self.validation:\n                start = np.random.randint(effective_length - len_y)\n            else:\n                start = 0\n            new_y[start:start + len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            if not self.validation:\n                start = np.random.randint(len_y - effective_length)\n            else:\n                start = 0\n            y = y[start:start + effective_length].astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        y = np.nan_to_num(y)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n\n        y = np.nan_to_num(y)\n\n        labels = np.zeros(len(CFG.target_columns), dtype=float)\n        labels[CFG.target_columns.index(ebird_code)] = 1.0\n\n        return {\n            \"image\": y,\n            \"targets\": labels\n        }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(phase: str):\n    transforms = CFG.transforms\n    if transforms is None:\n        return None\n    else:\n        if transforms[phase] is None:\n            return None\n        trns_list = []\n        for trns_conf in transforms[phase]:\n            trns_name = trns_conf[\"name\"]\n            trns_params = {} if trns_conf.get(\"params\") is None else \\\n                trns_conf[\"params\"]\n            if globals().get(trns_name) is not None:\n                trns_cls = globals()[trns_name]\n                trns_list.append(trns_cls(**trns_params))\n\n        if len(trns_list) > 0:\n            return Compose(trns_list)\n        else:\n            return None\n        \n        \nclass Normalize:\n    def __call__(self, y: np.ndarray):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)\n\n\nclass Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n    \ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes\n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(\n                self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\ndef gem(x: torch.Tensor, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return self.__class__.__name__ + f\"(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=CFG.n_fft, hop_length=CFG.hop_length,\n                                                 win_length=CFG.n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=CFG.sample_rate, n_fft=CFG.n_fft,\n                                                 n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax, ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, input):\n        # (batch_size, 1, time_steps, freq_bins)\n        x = self.spectrogram_extractor(input)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)\n        x = self.encoder(x)\n\n        # (batch_size, channels, frames)\n        x = torch.mean(x, dim=2)\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output,\n            \"segmentwise_output\": segmentwise_output,\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output\n        }\n\n        return output_dict","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"__CRITERIONS__ = {\n    \"BCEFocalLoss\": BCEFocalLoss,\n    \"BCEFocal2WayLoss\": BCEFocal2WayLoss\n}\n\n\ndef get_criterion():\n    if hasattr(nn, CFG.loss_name):\n        return nn.__getattribute__(CFG.loss_name)(**CFG.loss_params)\n    elif __CRITERIONS__.get(CFG.loss_name) is not None:\n        return __CRITERIONS__[CFG.loss_name](**CFG.loss_params)\n    else:\n        raise NotImplementedError\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom optimizer\n__OPTIMIZERS__ = {}\n\n\ndef get_optimizer(model: nn.Module):\n    optimizer_name = CFG.optimizer_name\n    if optimizer_name == \"SAM\":\n        base_optimizer_name = CFG.base_optimizer\n        if __OPTIMIZERS__.get(base_optimizer_name) is not None:\n            base_optimizer = __OPTIMIZERS__[base_optimizer_name]\n        else:\n            base_optimizer = optim.__getattribute__(base_optimizer_name)\n        return SAM(model.parameters(), base_optimizer, **CFG.optimizer_params)\n\n    if __OPTIMIZERS__.get(optimizer_name) is not None:\n        return __OPTIMIZERS__[optimizer_name](model.parameters(),\n                                              **CFG.optimizer_params)\n    else:\n        return optim.__getattribute__(optimizer_name)(model.parameters(),\n                                                      **CFG.optimizer_params)\n\n\ndef get_scheduler(optimizer):\n    scheduler_name = CFG.scheduler_name\n\n    if scheduler_name is None:\n        return\n    else:\n        return optim.lr_scheduler.__getattribute__(scheduler_name)(\n            optimizer, **CFG.scheduler_params)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SchedulerCallback(Callback):\n    def __init__(self):\n        super().__init__(CallbackOrder.Scheduler)\n\n    def on_loader_end(self, state: IRunner):\n        lr = state.scheduler.get_last_lr()\n        state.epoch_metrics[\"lr\"] = lr[0]\n        if state.is_train_loader:\n            state.scheduler.step()\n\n\nclass SampleF1Callback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 prefix: str = \"f1\",\n                 threshold=0.5):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.prefix = prefix\n        self.threshold = threshold\n\n    def on_loader_start(self, state: IRunner):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: IRunner):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[\"clipwise_output\"].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output > self.threshold\n        score = metrics.f1_score(targ, y_pred, average=\"samples\")\n\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: IRunner):\n        y_pred = np.concatenate(self.prediction, axis=0) > self.threshold\n        y_true = np.concatenate(self.target, axis=0)\n        score = metrics.f1_score(y_true, y_pred, average=\"samples\")\n\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = \"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: IRunner):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: IRunner):\n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        try:\n            score = metrics.average_precision_score(\n                targ, clipwise_output, average=None)\n        except ValueError:\n            import pdb\n            pdb.set_trace()\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: IRunner):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = metrics.average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\ndef get_callbacks():\n    return [\n        SampleF1Callback(prefix=\"f1_at_05\", threshold=0.5),\n        SampleF1Callback(prefix=\"f1_at_03\", threshold=0.3),\n        SampleF1Callback(prefix=\"f1_at_07\", threshold=0.7),\n        mAPCallback()\n    ]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_runner(device: torch.device):\n    return SupervisedRunner(device=device, input_key=\"image\", input_target_key=\"targets\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n\nlogdir = Path(\"out\")\nlogdir.mkdir(exist_ok=True, parents=True)\nif (logdir / \"train.log\").exists():\n    os.remove(logdir / \"train.log\")\nlogger = init_logger(log_file=logdir / \"train.log\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# environment\nset_seed(CFG.seed)\ndevice = get_device()\n\n# validation\nsplitter = getattr(model_selection, CFG.split)(**CFG.split_params)\n\n# data\ntrain = pd.read_csv(CFG.train_csv)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# main loop\nfor i, (trn_idx, val_idx) in enumerate(splitter.split(train, y=train[\"primary_label\"])):\n    if i not in CFG.folds:\n        continue\n    logger.info(\"=\" * 120)\n    logger.info(f\"Fold {i} Training\")\n    logger.info(\"=\" * 120)\n\n    trn_df = train.loc[trn_idx, :].reset_index(drop=True)\n    val_df = train.loc[val_idx, :].reset_index(drop=True)\n\n    loaders = {\n        phase: torchdata.DataLoader(\n            WaveformDataset(\n                df_,\n                CFG.train_datadir,\n                img_size=CFG.img_size,\n                waveform_transforms=get_transforms(phase),\n                period=CFG.period,\n                validation=(phase == \"valid\")\n            ),\n            **CFG.loader_params[phase])  # type: ignore\n        for phase, df_ in zip([\"train\", \"valid\"], [trn_df, val_df])\n    }\n\n    model = TimmSED(\n        base_model_name=CFG.base_model_name,\n        pretrained=CFG.pretrained,\n        num_classes=CFG.num_classes,\n        in_channels=CFG.in_channels)\n    criterion = get_criterion()\n    optimizer = get_optimizer(model)\n    scheduler = get_scheduler(optimizer)\n    callbacks = get_callbacks()\n    runner = get_runner(device)\n    runner.train(\n        model=model,\n        criterion=criterion,\n        loaders=loaders,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        num_epochs=CFG.epochs,\n        verbose=True,\n        logdir=logdir / f\"fold{i}\",\n        callbacks=callbacks,\n        main_metric=CFG.main_metric,\n        minimize_metric=CFG.minimize_metric)\n\n    del model, optimizer, scheduler\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}