{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install colorednoise > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold 0 test run","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport gc\nimport os\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport random\nimport time\nimport warnings\n\nimport librosa\nimport colorednoise as cn\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as torchdata\n\nfrom contextlib import contextmanager\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\nfrom typing import Optional\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error, roc_auc_score\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\nfrom tqdm import tqdm\n\nimport albumentations as A\nimport albumentations.pytorch.transforms as T\n\nimport matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\nfrom torch.cuda.amp import autocast, GradScaler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\nall_path = glob.glob('../input/birdclef2022-audio-to-numpy-1-4/train_np/*/*.npy')\\\n+ glob.glob('../input/birdclef2022-audio-to-numpy-2-4/train_np/*/*.npy')\\\n+ glob.glob('../input/birdclef2022-audio-to-numpy-3-4/train_np/*/*.npy')\\\n+ glob.glob('../input/birdclef2022-audio-to-numpy-4-4/train_np/*/*.npy')\n\nlen(all_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\n\n\ntrain = pd.read_csv('../input/birdclef-2022/train_metadata.csv')\n\n\ntrain['new_target'] = train['primary_label'] + ' ' + train['secondary_labels'].map(lambda x: ' '.join(ast.literal_eval(x)))\ntrain['len_new_target'] = train['new_target'].map(lambda x: len(x.split()))\n# train['len_new_target'].value_counts()\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_df = pd.DataFrame(all_path, columns=['file_path'])\npath_df['filename'] = path_df['file_path'].map(lambda x: x.split('/')[-2]+'/'+x.split('/')[-1][:-4])\npath_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, path_df, on='filename')\nprint(train.shape)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nfor n, (trn_index, val_index) in enumerate(Fold.split(train, train['primary_label'])):\n    train.loc[val_index, 'kfold'] = int(n)\ntrain['kfold'] = train['kfold'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_folds.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    EXP_ID = 'EX005'\n    seed = 71\n    epochs = 23\n    cutmix_and_mixup_epochs = 18\n    folds = [0] # [0, 1, 2, 3, 4]\n    N_FOLDS = 5\n    LR = 1e-3\n    ETA_MIN = 1e-6\n    WEIGHT_DECAY = 1e-6\n    train_bs = 16 # 32\n    valid_bs = 32 # 64\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    EARLY_STOPPING = True\n    DEBUG = False # True\n    EVALUATION = 'AUC'\n    apex = True\n\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 152\n    in_channels = 3\n    target_columns = 'afrsil1 akekee akepa1 akiapo akikik amewig aniani apapan arcter \\\n                      barpet bcnher belkin1 bkbplo bknsti bkwpet blkfra blknod bongul \\\n                      brant brnboo brnnod brnowl brtcur bubsan buffle bulpet burpar buwtea \\\n                      cacgoo1 calqua cangoo canvas caster1 categr chbsan chemun chukar cintea \\\n                      comgal1 commyn compea comsan comwax coopet crehon dunlin elepai ercfra eurwig \\\n                      fragul gadwal gamqua glwgul gnwtea golphe grbher3 grefri gresca gryfra gwfgoo \\\n                      hawama hawcoo hawcre hawgoo hawhaw hawpet1 hoomer houfin houspa hudgod iiwi incter1 \\\n                      jabwar japqua kalphe kauama laugul layalb lcspet leasan leater1 lessca lesyel lobdow lotjae \\\n                      madpet magpet1 mallar3 masboo mauala maupar merlin mitpar moudov norcar norhar2 normoc norpin \\\n                      norsho nutman oahama omao osprey pagplo palila parjae pecsan peflov perfal pibgre pomjae puaioh \\\n                      reccar redava redjun redpha1 refboo rempar rettro ribgul rinduc rinphe rocpig rorpar rudtur ruff \\\n                      saffin sander semplo sheowl shtsan skylar snogoo sooshe sooter1 sopsku1 sora spodov sposan \\\n                      towsol wantat1 warwhe1 wesmea wessan wetshe whfibi whiter whttro wiltur yebcar yefcan zebdov'.split()\n\n    img_size = 224 # 128\n    main_metric = \"epoch_f1_at_03\"\n\n    period = 5\n    n_mels = 224 # 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    melspectrogram_parameters = {\n        \"n_mels\": 224, # 128,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n    \n    \nclass AudioParams:\n    \"\"\"\n    Parameters used for the audio data\n    \"\"\"\n    sr = CFG.sample_rate\n    duration = CFG.period\n    # Melspectrogram\n    n_mels = CFG.n_mels\n    fmin = CFG.fmin\n    fmax = CFG.fmax","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray, sr):\n        for trns in self.transforms:\n            y = trns(y, sr)\n        return y\n\n\nclass AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray, sr):\n        if self.always_apply:\n            return self.apply(y, sr=sr)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y, sr=sr)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray, **params):\n        raise NotImplementedError\n\n\nclass OneOf(Compose):\n    # https://github.com/albumentations-team/albumentations/blob/master/albumentations/core/composition.py\n    def __init__(self, transforms, p=0.5):\n        super().__init__(transforms)\n        self.p = p\n        transforms_ps = [t.p for t in transforms]\n        s = sum(transforms_ps)\n        self.transforms_ps = [t / s for t in transforms_ps]\n\n    def __call__(self, y: np.ndarray, sr):\n        data = y\n        if self.transforms_ps and (random.random() < self.p):\n            random_state = np.random.RandomState(random.randint(0, 2 ** 32 - 1))\n            t = random_state.choice(self.transforms, p=self.transforms_ps)\n            data = t(y, sr)\n        return data\n\n\nclass Normalize(AudioTransform):\n    def __init__(self, always_apply=False, p=1):\n        super().__init__(always_apply, p)\n\n    def apply(self, y: np.ndarray, **params):\n        max_vol = np.abs(y).max()\n        y_vol = y * 1 / max_vol\n        return np.asfortranarray(y_vol)\n\n\nclass NewNormalize(AudioTransform):\n    def __init__(self, always_apply=False, p=1):\n        super().__init__(always_apply, p)\n\n    def apply(self, y: np.ndarray, **params):\n        y_mm = y - y.mean()\n        return y_mm / y_mm.abs().max()\n\n\nclass NoiseInjection(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_level=0.5):\n        super().__init__(always_apply, p)\n\n        self.noise_level = (0.0, max_noise_level)\n\n    def apply(self, y: np.ndarray, **params):\n        noise_level = np.random.uniform(*self.noise_level)\n        noise = np.random.randn(len(y))\n        augmented = (y + noise * noise_level).astype(y.dtype)\n        return augmented\n\n\nclass GaussianNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PinkNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\n\nclass PitchShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_range=5):\n        super().__init__(always_apply, p)\n        self.max_range = max_range\n\n    def apply(self, y: np.ndarray, sr, **params):\n        n_steps = np.random.randint(-self.max_range, self.max_range)\n        augmented = librosa.effects.pitch_shift(y, sr, n_steps)\n        return augmented\n\n\nclass TimeStretch(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_rate=1):\n        super().__init__(always_apply, p)\n        self.max_rate = max_rate\n\n    def apply(self, y: np.ndarray, **params):\n        rate = np.random.uniform(0, self.max_rate)\n        augmented = librosa.effects.time_stretch(y, rate)\n        return augmented\n\n\ndef _db2float(db: float, amplitude=True):\n    if amplitude:\n        return 10 ** (db / 20)\n    else:\n        return 10 ** (db / 10)\n\n\ndef volume_down(y: np.ndarray, db: float):\n    \"\"\"\n    Low level API for decreasing the volume\n    Parameters\n    ----------\n    y: numpy.ndarray\n        stereo / monaural input audio\n    db: float\n        how much decibel to decrease\n    Returns\n    -------\n    applied: numpy.ndarray\n        audio with decreased volume\n    \"\"\"\n    applied = y * _db2float(-db)\n    return applied\n\n\ndef volume_up(y: np.ndarray, db: float):\n    \"\"\"\n    Low level API for increasing the volume\n    Parameters\n    ----------\n    y: numpy.ndarray\n        stereo / monaural input audio\n    db: float\n        how much decibel to increase\n    Returns\n    -------\n    applied: numpy.ndarray\n        audio with increased volume\n    \"\"\"\n    applied = y * _db2float(db)\n    return applied\n\n\nclass RandomVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        if db >= 0:\n            return volume_up(y, db)\n        else:\n            return volume_down(y, db)\n\n\nclass CosineVolume(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, limit=10):\n        super().__init__(always_apply, p)\n        self.limit = limit\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.limit, self.limit)\n        cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n        dbs = _db2float(cosine * db)\n        return y * dbs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nOUTPUT_DIR = f'./'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n   \n    \ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nset_seed(CFG.seed)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_loss(y_true, y_pred):\n    return metrics.roc_auc_score(np.array(y_true), np.array(y_pred))\n\n\n# ====================================================\n# Training helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \n\nclass MetricMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = []\n        self.y_pred = []\n    \n    def update(self, y_true, y_pred):\n        self.y_true.extend(y_true.cpu().detach().numpy().tolist())\n        self.y_pred.extend(y_pred[\"clipwise_output\"].cpu().detach().numpy().tolist())\n\n    @property\n    def avg(self):\n        self.f1_03 = metrics.f1_score(np.array(self.y_true), np.array(self.y_pred) > 0.3, average=\"micro\")\n        self.f1_05 = metrics.f1_score(np.array(self.y_true), np.array(self.y_pred) > 0.5, average=\"micro\")\n        \n        return {\n            \"f1_at_03\" : self.f1_03,\n            \"f1_at_05\" : self.f1_05,\n        }\n    \n    \n# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_melspec(y, params):\n    \"\"\"\n    Computes a mel-spectrogram and puts it at decibel scale\n    Arguments:\n        y {np array} -- signal\n        params {AudioParams} -- Parameters to use for the spectrogram. Expected to have the attributes sr, n_mels, f_min, f_max\n    Returns:\n        np array -- Mel-spectrogram\n    \"\"\"\n    melspec = librosa.feature.melspectrogram(\n        y=y, sr=params.sr, n_mels=params.n_mels, fmin=params.fmin, fmax=params.fmax,\n    )\n\n    melspec = librosa.power_to_db(melspec).astype(np.float32)\n    return melspec\n\n\ndef crop_or_pad(y, length, sr, train=True, probs=None):\n    \"\"\"\n    Crops an array to a chosen length\n    Arguments:\n        y {1D np array} -- Array to crop\n        length {int} -- Length of the crop\n        sr {int} -- Sampling rate\n    Keyword Arguments:\n        train {bool} -- Whether we are at train time. If so, crop randomly, else return the beginning of y (default: {True})\n        probs {None or numpy array} -- Probabilities to use to chose where to crop (default: {None})\n    Returns:\n        1D np array -- Cropped array\n    \"\"\"\n    if len(y) <= length:\n        y = np.concatenate([y, np.zeros(length - len(y))])\n    else:\n        if not train:\n            start = 0\n        elif probs is None:\n            start = np.random.randint(len(y) - length)\n        else:\n            start = (\n                    np.random.choice(np.arange(len(probs)), p=probs) + np.random.random()\n            )\n            start = int(sr * (start))\n\n        y = y[start: start + length]\n\n    return y.astype(np.float32)\n\n\ndef mono_to_color(X, eps=1e-6, mean=None, std=None):\n    \"\"\"\n    Converts a one channel array to a 3 channel one in [0, 255]\n    Arguments:\n        X {numpy array [H x W]} -- 2D array to convert\n    Keyword Arguments:\n        eps {float} -- To avoid dividing by 0 (default: {1e-6})\n        mean {None or np array} -- Mean for normalization (default: {None})\n        std {None or np array} -- Std for normalization (default: {None})\n    Returns:\n        numpy array [3 x H x W] -- RGB numpy array\n    \"\"\"\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\nmean = (0.485, 0.456, 0.406) # RGB\nstd = (0.229, 0.224, 0.225) # RGB\n\nalbu_transforms = {\n    'train' : A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=16),\n                A.CoarseDropout(max_holes=4),\n            ], p=0.5),\n            A.Normalize(mean, std),\n    ]),\n    'valid' : A.Compose([\n            A.Normalize(mean, std),\n    ]),\n}\n\n\nclass WaveformDataset(torch.utils.data.Dataset):\n    def __init__(self,\n                 df: pd.DataFrame,\n                 mode='train'):\n        self.df = df\n        self.mode = mode\n\n        if mode == 'train':\n            self.wave_transforms = Compose(\n                [\n                    OneOf(\n                        [\n                            NoiseInjection(p=1, max_noise_level=0.04),\n                            GaussianNoise(p=1, min_snr=5, max_snr=20),\n                            PinkNoise(p=1, min_snr=5, max_snr=20),\n                        ],\n                        p=0.2,\n                    ),\n                    RandomVolume(p=0.2, limit=4),\n                    Normalize(p=1),\n                ]\n            )\n        else:\n            self.wave_transforms = Compose(\n                [\n                    Normalize(p=1),\n                ]\n            )\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        SR = 32000\n        sample = self.df.loc[idx, :]\n        \n        wav_path = sample[\"file_path\"]\n        labels = sample[\"new_target\"]\n\n        y = np.load(wav_path)\n\n        # SEC = int(len(y)/2/SR)\n        # if SEC > 0:\n        #     start = np.random.randint(SEC)\n        #     end = start+AudioParams.duration\n        if len(y) > 0:\n            y = y[:AudioParams.duration*SR]\n\n            if self.wave_transforms:\n                y = self.wave_transforms(y, sr=SR)\n\n        y = np.concatenate([y, y, y])[:AudioParams.duration * AudioParams.sr] \n        y = crop_or_pad(y, AudioParams.duration * AudioParams.sr, sr=AudioParams.sr, train=True, probs=None)\n        image = compute_melspec(y, AudioParams)\n        image = mono_to_color(image)\n        image = image.astype(np.uint8)\n        \n        # image = np.load(wav_path) # (224, 313, 3)\n        image = albu_transforms[self.mode](image=image)['image']\n        image = image.T\n        \n        targets = np.zeros(len(CFG.target_columns), dtype=float)\n        for ebird_code in labels.split():\n            targets[CFG.target_columns.index(ebird_code)] = 1.0\n\n        return {\n            \"image\": image,\n            \"targets\": targets,\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output\n\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n\n\nclass TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64//2, time_stripes_num=2,\n                                               freq_drop_width=8//2, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n\n    def forward(self, input_data):\n        x = input_data # (batch_size, 3, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            if random.random() < 0.25:\n                x = self.spec_augmenter(x)\n\n        x = x.transpose(2, 3)\n\n        x = self.encoder(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output,\n            'logit': logit,\n            'framewise_logit': framewise_logit,\n        }\n\n        return output_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\n\ndef cutmix(data, targets, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n\n    lam = np.random.beta(alpha, alpha)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n\n    new_targets = [targets, shuffled_targets, lam]\n    return data, new_targets\n\ndef mixup(data, targets, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n\n    lam = np.random.beta(alpha, alpha)\n    new_data = data * lam + shuffled_data * (1 - lam)\n    new_targets = [targets, shuffled_targets, lam]\n    return new_data, new_targets\n\n\ndef cutmix_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\ndef mixup_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\n\ndef loss_fn(logits, targets):\n    loss_fct = BCEFocal2WayLoss()\n    loss = loss_fct(logits, targets)\n    return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(model, data_loader, device, optimizer, scheduler):\n    model.train()\n    scaler = GradScaler(enabled=CFG.apex)\n    losses = AverageMeter()\n    scores = MetricMeter()\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    for data in tk0:\n        optimizer.zero_grad()\n        inputs = data['image'].to(device)\n        targets = data['targets'].to(device)\n        with autocast(enabled=CFG.apex):\n            outputs = model(inputs)\n            loss = loss_fn(outputs, targets)\n        \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        scheduler.step()\n        losses.update(loss.item(), inputs.size(0))\n        scores.update(targets, outputs)\n        tk0.set_postfix(loss=losses.avg)\n    return scores.avg, losses.avg\n\n\ndef train_mixup_cutmix_fn(model, data_loader, device, optimizer, scheduler):\n    model.train()\n    scaler = GradScaler(enabled=CFG.apex)\n    losses = AverageMeter()\n    scores = MetricMeter()\n    tk0 = tqdm(data_loader, total=len(data_loader))\n\n    for data in tk0:\n        optimizer.zero_grad()\n        inputs = data['image'].to(device)\n        targets = data['targets'].to(device)\n\n        if np.random.rand()<0.5:\n            inputs, new_targets = mixup(inputs, targets, 0.4)\n            with autocast(enabled=CFG.apex):\n                outputs = model(inputs)\n                loss = mixup_criterion(outputs, new_targets) \n        else:\n            inputs, new_targets = cutmix(inputs, targets, 0.4)\n            with autocast(enabled=CFG.apex):\n                outputs = model(inputs)\n                loss = cutmix_criterion(outputs, new_targets)\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        scheduler.step()\n        losses.update(loss.item(), inputs.size(0))\n        scores.update(new_targets[0], outputs)\n        tk0.set_postfix(loss=losses.avg)\n    return scores.avg, losses.avg\n\n\ndef valid_fn(model, data_loader, device):\n    model.eval()\n    losses = AverageMeter()\n    scores = MetricMeter()\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    valid_preds = []\n    with torch.no_grad():\n        for data in tk0:\n            inputs = data['image'].to(device)\n            targets = data['targets'].to(device)\n            outputs = model(inputs)\n            loss = loss_fn(outputs, targets)\n            losses.update(loss.item(), inputs.size(0))\n            scores.update(targets, outputs)\n            tk0.set_postfix(loss=losses.avg)\n    return scores.avg, losses.avg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_fn(model, data_loader, device):\n    model.eval()\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    final_output = []\n    final_target = []\n    with torch.no_grad():\n        for b_idx, data in enumerate(tk0):\n            inputs = data['image'].to(device)\n            targets = data['targets'].to(device).detach().cpu().numpy().tolist()\n            output = model(inputs)\n            output = output[\"clipwise_output\"].cpu().detach().cpu().numpy().tolist()\n            final_output.extend(output)\n            final_target.extend(targets)\n    return final_output, final_target\n\n\ndef calc_cv(model_paths):\n    df = pd.read_csv('train_folds.csv')\n    y_true = []\n    y_pred = []\n    for fold, model_path in enumerate(model_paths):\n        model = TimmSED(\n            base_model_name=CFG.base_model_name,\n            pretrained=CFG.pretrained,\n            num_classes=CFG.num_classes,\n            in_channels=CFG.in_channels)\n\n        model.to(device)\n        model.load_state_dict(torch.load(model_path))\n        model.eval()\n\n        val_df = df[df.kfold == fold].reset_index(drop=True)\n        dataset = WaveformDataset(df=val_df, mode='valid')\n        dataloader = torch.utils.data.DataLoader(\n            dataset, batch_size=CFG.valid_bs, num_workers=0, pin_memory=True, shuffle=False\n        )\n\n        final_output, final_target = inference_fn(model, dataloader, device)\n        y_pred.extend(final_output)\n        y_true.extend(final_target)\n        torch.cuda.empty_cache()\n\n        f1_03 = metrics.f1_score(np.array(y_true), np.array(y_pred) > 0.3, average=\"micro\")\n        print(f'micro f1_0.3 {f1_03}')\n\n    f1_03 = metrics.f1_score(np.array(y_true), np.array(y_pred) > 0.3, average=\"micro\")\n    f1_05 = metrics.f1_score(np.array(y_true), np.array(y_pred) > 0.5, average=\"micro\")\n\n    print(f'overall micro f1_0.3 {f1_03}')\n    print(f'overall micro f1_0.5 {f1_05}')\n    return","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# main loop\nfor fold in range(5):\n    if fold not in CFG.folds:\n        continue\n    print(\"=\" * 100)\n    print(f\"Fold {fold} Training\")\n    print(\"=\" * 100)\n\n    trn_df = train[train.kfold != fold].reset_index(drop=True)\n    val_df = train[train.kfold == fold].reset_index(drop=True)\n\n    train_dataset = WaveformDataset(df=trn_df, mode='train')\n    train_dataloader = torch.utils.data.DataLoader(\n        train_dataset, batch_size=CFG.train_bs, num_workers=0, pin_memory=True, shuffle=True\n    )\n    \n    valid_dataset = WaveformDataset(df=val_df, mode='valid')\n    valid_dataloader = torch.utils.data.DataLoader(\n        valid_dataset, batch_size=CFG.valid_bs, num_workers=0, pin_memory=True, shuffle=False\n    )\n\n    model = TimmSED(\n        base_model_name=CFG.base_model_name,\n        pretrained=CFG.pretrained,\n        num_classes=CFG.num_classes,\n        in_channels=CFG.in_channels)\n\n    optimizer = transformers.AdamW(model.parameters(), lr=CFG.LR, weight_decay=CFG.WEIGHT_DECAY)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, eta_min=CFG.ETA_MIN, T_max=500)\n\n    model = model.to(device)\n\n    min_loss = 999\n    best_score = -np.inf\n\n    for epoch in range(CFG.epochs):\n        print(\"Starting {} epoch...\".format(epoch+1))\n\n        start_time = time.time()\n\n        if epoch < CFG.cutmix_and_mixup_epochs:\n            train_avg, train_loss = train_mixup_cutmix_fn(model, train_dataloader, device, optimizer, scheduler)\n        else: \n            train_avg, train_loss = train_fn(model, train_dataloader, device, optimizer, scheduler)\n\n        valid_avg, valid_loss = valid_fn(model, valid_dataloader, device)\n\n        elapsed = time.time() - start_time\n\n        print(f'Epoch {epoch+1} - avg_train_loss: {train_loss:.5f}  avg_val_loss: {valid_loss:.5f}  time: {elapsed:.0f}s')\n        print(f\"Epoch {epoch+1} - train_f1_at_03:{train_avg['f1_at_03']:0.5f}  valid_f1_at_03:{valid_avg['f1_at_03']:0.5f}\")\n        print(f\"Epoch {epoch+1} - train_f1_at_05:{train_avg['f1_at_05']:0.5f}  valid_f1_at_05:{valid_avg['f1_at_05']:0.5f}\")\n\n        if valid_avg['f1_at_03'] > best_score:\n            print(f\">>>>>>>> Model Improved From {best_score} ----> {valid_avg['f1_at_03']}\")\n            print(f\"other scores here... {valid_avg['f1_at_03']}, {valid_avg['f1_at_05']}\")\n            torch.save(model.state_dict(), f'fold-{fold}.bin')\n            best_score = valid_avg['f1_at_03']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_paths = [f'fold-{i}.bin' for i in CFG.folds]\n\ncalc_cv(model_paths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}