{"cells":[{"metadata":{},"cell_type":"markdown","source":"* V1: EfficientNet BaseLine: mix precision, label smoothing, mixup, same data augmentation(time mask, freque**ncy mask etc.)\n* V3: PANN ResNet38 BaseLine (BugFix):\n    https://github.com/qiuqiangkong/audioset_tagging_cnn\n* V4: PANN ResNet38: Without Mixup\n* V5: PANN ResNet38：Tmax=10\n* v7: PANN CNN14_DecesionLevelAttetnion， Based on: https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection\n* v8: PANN CNN14_DecesionLevelAttetnion，with mixup, and label smoothing.\n* v9: Change metric from f1score(macro) to f1score(micro), Epoch:100\n* V12: Create SimpleBalanceSampler ResNet38.\n* V14: ResNet38 without SimpleBalanceSampler\n* V15: OOF, Based on: https://www.kaggle.com/shonenkov/competition-metrics \n* V16：V15 BugFix\n* V17：Wavegram_Logmel_Cnn14 + Attention head","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install ../input/bird-panns/torchlibrosa-master/torchlibrosa-master/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\nimport matplotlib.pyplot as plt\nimport warnings\nfrom datetime import datetime\nfrom glob import glob\nfrom joblib import delayed, Parallel\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\nimport librosa.display \nfrom fastprogress import progress_bar\nfrom scipy.io import wavfile\nfrom librosa.core import resample, to_mono\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\nfrom torch.utils.data.sampler import Sampler\n\n#from efficientnet_pytorch import EfficientNet\n#from apex import amp\nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    period= 5\n    num_fold = 5\n    num_class = 264\n    num_workers = 2\n    \n    ####################\n    #training parameters\n    ####################\n    verbose = True\n    verbose_step = 1\n    folder = 'panns'\n    use_mixup = True\n    use_amp = False\n    accumulate_steps = 1\n    batch_size = 32\n    lr = 1e-3\n    n_epochs = 50\n    weight_decay = 0\n    \n    step_scheduler = False\n    validation_scheduler = True\n    SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR\n    scheduler_params = dict(\n        T_max=n_epochs\n    )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\nset_seed()\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n    INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"\n\ntrain = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\n\nif not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split Train/Val DataSet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(config.num_fold)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", \n                                 columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_fold = 0\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Labels Dectionary","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast\nBIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ebird_codes =  train_all.query(\"fold != @use_fold\")[\"ebird_code\"].values.tolist()\nall_targets = []\nfor i in range(len(ebird_codes)):\n    ebird_code = ebird_codes[i]\n    labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n    labels[BIRD_CODE[ebird_code]] = 1\n    all_targets.append(labels)\n    \nall_targets = np.array(all_targets)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define DataLoader","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class SimpleBalanceClassSampler(Sampler):\n\n    def __init__(self, targets, classes_num):\n\n        self.targets = targets\n        self.classes_num = classes_num\n        \n        self.samples_num_per_class = np.sum(self.targets, axis=0)\n        self.max_num = np.max(self.samples_num_per_class)\n        \n        self.indexes_per_class = []\n        # Training indexes of all sound classes. E.g.: \n        # [[0, 11, 12, ...], [3, 4, 15, 16, ...], [7, 8, ...], ...]\n        for k in range(self.classes_num):\n            self.indexes_per_class.append(\n                np.where(self.targets[:, k] == 1)[0])\n        \n        self.length = self.classes_num * self.max_num\n\n    def __iter__(self):\n        \n        all_indexs = []\n        \n        for k in range(self.classes_num):\n            if len(self.indexes_per_class[k]) == self.max_num:\n                all_indexs.append(self.indexes_per_class[k])\n            else:\n                gap = self.max_num - len(self.indexes_per_class[k])\n                random_choice = np.random.choice(self.indexes_per_class[k], int(gap), replace=True)\n                all_indexs.append(np.array(list(random_choice) + list(self.indexes_per_class[k])))\n                \n        l = np.stack(all_indexs).T\n        l = l.reshape(-1)\n        random.shuffle(l)\n        return iter(l)\n\n    def __len__(self):\n        return int(self.length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class birddataset(data.Dataset):\n    \"\"\"\n    Based On\n    https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast/data?\n    \"\"\"\n    \n    def __init__(self, file_list, test=False, label_smooth=True):\n        \n        self.file_list = file_list\n        self.label_smooth = label_smooth\n        self.test = test\n \n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self, idx: int):\n        \n        wav_path, ebird_code = self.file_list[idx]\n        wave, sr = sf.read(wav_path)\n        effective_length = sr * config.period\n        if len(wave) < effective_length:\n            new_wave = np.zeros(effective_length, dtype=wave.dtype)\n            start = np.random.randint(effective_length - len(wave))\n            new_wave[start:start + len(wave)] = wave\n            wave = new_wave.astype(np.float32)\n        elif len(wave) > effective_length:\n            start = np.random.randint(len(wave) - effective_length)\n            wave = wave[start:start + effective_length].astype(np.float32)\n        else:\n            wave= wave.astype(np.float32)\n        \n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n        if self.label_smooth and self.test == False:\n            labels = self.make_label_smooth(labels, num_classes=config.num_class)\n        \n        return {\"waveform\": wave, \"targets\": labels}\n    \n    def make_label_smooth(self, labels, num_classes, epsilon=0.1):\n        b = np.ones(num_classes) * (1 / num_classes)\n        return (1 - epsilon) * labels + epsilon * b ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Model","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return np.array(mixup_lambdas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def do_mixup(x, mixup_lambda):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes \n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n \n    if hasattr(layer, 'bias'):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n    \ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def _resnet_conv3x3(in_planes, out_planes):\n    #3x3 convolution with padding\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,\n                     padding=1, groups=1, bias=False, dilation=1)\n\n\ndef _resnet_conv1x1(in_planes, out_planes):\n    #1x1 convolution\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"class _ResnetBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in _ResnetBasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n\n        self.stride = stride\n\n        self.conv1 = _resnet_conv3x3(inplanes, planes)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = _resnet_conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        nn.init.constant_(self.bn2.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            out = F.avg_pool2d(x, kernel_size=(2, 2))\n        else:\n            out = x\n\n        out = self.conv1(out)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"class ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.conv2 = nn.Conv2d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"class _ResNet(nn.Module):\n    def __init__(self, block, layers, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(_ResNet, self).__init__()\n\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n\n        self.layer1 = self._make_layer(block, 64, layers[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            if stride == 1:\n                downsample = nn.Sequential(\n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[0])\n                init_bn(downsample[1])\n            elif stride == 2:\n                downsample = nn.Sequential(\n                    nn.AvgPool2d(kernel_size=2), \n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[1])\n                init_bn(downsample[2])\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class ResNet38(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        \n        super(ResNet38, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        # self.conv_block2 = ConvBlock(in_channels=64, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"class ConvPreWavBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvPreWavBlock, self).__init__()\n        \n        self.conv1 = nn.Conv1d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1,\n                              padding=1, bias=False)\n                              \n        self.conv2 = nn.Conv1d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1, dilation=2, \n                              padding=2, bias=False)\n                              \n        self.bn1 = nn.BatchNorm1d(out_channels)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        x = F.max_pool1d(x, kernel_size=pool_size)\n        \n        return x\n    \nclass Wavegram_Logmel_Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(Wavegram_Logmel_Cnn14, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        self.pre_conv0 = nn.Conv1d(in_channels=1, out_channels=64, kernel_size=11, stride=5, padding=5, bias=False)\n        self.pre_bn0 = nn.BatchNorm1d(64)\n        self.pre_block1 = ConvPreWavBlock(64, 64)\n        self.pre_block2 = ConvPreWavBlock(64, 128)\n        self.pre_block3 = ConvPreWavBlock(128, 128)\n        self.pre_block4 = ConvBlock(in_channels=4, out_channels=64)\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=128, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.pre_conv0)\n        init_bn(self.pre_bn0)\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        # Wavegram\n        a1 = F.relu_(self.pre_bn0(self.pre_conv0(input[:, None, :])))\n        a1 = self.pre_block1(a1, pool_size=4)\n        a1 = self.pre_block2(a1, pool_size=4)\n        a1 = self.pre_block3(a1, pool_size=4)\n        a1 = a1.reshape((a1.shape[0], -1, 32, a1.shape[-1])).transpose(2, 3)\n        a1 = self.pre_block4(a1, pool_size=(2, 1))\n\n        # Log mel spectrogram\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n            a1 = do_mixup(a1, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n\n        # Concatenate Wavegram and Log mel spectrogram along the channel dimension\n        x = torch.cat((x, a1), dim=1)\n\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x_ = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x_, dim=2)\n        x2 = torch.mean(x_, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        output_dict = {'frames_num': frames_num, 'embedding': x_, }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n        \nclass PANNsCNN14Att(nn.Module):\n    \n    def __init__(self, sample_rate: int, window_size: int, hop_size: int,\n                 mel_bins: int, fmin: int, fmax: int, classes_num: int):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 32  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(\n            n_fft=window_size,\n            hop_length=hop_size,\n            win_length=window_size,\n            window=window,\n            center=center,\n            pad_mode=pad_mode,\n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(\n            sr=sample_rate,\n            n_fft=window_size,\n            n_mels=mel_bins,\n            fmin=fmin,\n            fmax=fmax,\n            ref=ref,\n            amin=amin,\n            top_db=top_db,\n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(\n            time_drop_width=64,\n            time_stripes_num=2,\n            freq_drop_width=8,\n            freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def cnn_feature_extractor(self, x):\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        return x\n    \n    def preprocess(self, input, mixup_lambda=None):\n        # t1 = time.time()\n        x = self.spectrogram_extractor(input)  # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)  # (batch_size, 1, time_steps, mel_bins)\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        return x, frames_num\n        \n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)\n        # Output shape (batch size, channels, time, frequency)\n        x = self.cnn_feature_extractor(x)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n        print(x.size())\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        print(x.size())\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Transfer_Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        super(Transfer_Cnn14, self).__init__()\n        audioset_classes_num = 527\n        \n        self.base = Wavegram_Logmel_Cnn14(sample_rate, window_size, \n                                          hop_size, mel_bins, fmin,\n                                          fmax, audioset_classes_num)\n\n        # Transfer to another task layer\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.att_block = AttBlock(2048, classes_num, activation='sigmoid')\n        self.interpolate_ratio = 32\n        self.init_weight()\n    def init_weight(self):\n        init_layer(self.fc1)\n        \n    def load_from_pretrain(self, pretrained_checkpoint_path):\n        checkpoint = torch.load(pretrained_checkpoint_path)\n        self.base.load_state_dict(checkpoint['model'])\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"Input: (batch_size, data_length)\n        \"\"\"\n        base_output = self.base(input, mixup_lambda)\n        x = base_output['embedding']\n        frames_num = base_output['frames_num']\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(num_classes, pretrained=False):\n    \n    model_config = {\n        \"sample_rate\": 32000,\n        \"window_size\": 1024,\n        \"hop_size\": 320,\n        \"mel_bins\": 64,\n        \"fmin\": 50,\n        \"fmax\": 14000,\n        }\n\n    model_config[\"classes_num\"] = num_classes\n    model = Transfer_Cnn14(**model_config)\n    if pretrained:\n        model.load_from_pretrain(\"../input/bird-panns/Wavegram_Logmel_Cnn14_mAP0.439.pth\")\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Trainer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clip_bce(output_dict, target_dict):\n    \"\"\"Binary crossentropy loss.\n    \"\"\"\n    return F.binary_cross_entropy(\n        output_dict['clipwise_output'], target_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    \n    def __init__(self, model, device, config):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n        self.best_score = 0.0\n        self.early_stop_count = 0\n        \n        self.model = model\n        self.device = device\n        self.use_amp = config.use_amp\n        self.accumulate_steps = config.accumulate_steps\n        self.model.to(self.device)\n        self.use_mixup = config.use_mixup\n        self.threshold = 0.5\n        \n        if self.use_mixup:\n            self.mixup_augmenter = Mixup(mixup_alpha=1.)\n        self.loss_fn = PANNsLoss()\n        self.optimizer = torch.optim.Adam(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n        \n        if self.use_amp:\n            self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level=\"O1\",verbosity=0)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        \n        best_epoch = 0\n        for e in range(self.config.n_epochs):\n            if self.early_stop_count == 10:\n                self.log(f'early stopping: Epoch: {self.epoch}')\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            summary_loss, f1_score, acc_score = self.validation(self.model, validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, val_f1_score: {f1_score:.5f}, val_acc_score: {acc_score:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n                    os.remove(path)\n            else:\n                self.early_stop_count += 1\n            if f1_score > self.best_score:\n                self.best_score = f1_score\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-acc-checkpoint.bin')\n\n            if self.config.validation_scheduler:\n                self.scheduler.step()\n\n            self.epoch += 1\n            \n            \n    def validation(self, model, val_loader):\n        model.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        pred_results  = []\n        origin_labels = []\n        \n        for step, batch_data_dict in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                batch_size = batch_data_dict['waveform'].shape[0]\n                batch_output_dict = self.model(torch.FloatTensor(batch_data_dict['waveform']).cuda(), None)\n                batch_target_dict = {'target': torch.FloatTensor(batch_data_dict['targets']).cuda()}\n                loss = clip_bce(batch_output_dict, batch_target_dict['target'])\n                \n                proba = batch_output_dict['clipwise_output'].detach().cpu().numpy()\n                y_pred = proba.argmax(axis=1)\n                y_true = batch_data_dict['targets'].argmax(axis=1)\n                \n                pred_results.append(y_pred)\n                origin_labels.append(y_true)\n                \n                summary_loss.update(loss.detach().item(), batch_size)\n            \n        pred_results = np.concatenate(pred_results)    \n        origin_labels = np.concatenate(origin_labels)\n        val_f1_score = f1_score(origin_labels, pred_results, average='macro')\n        val_accuracy_score = accuracy_score(origin_labels, pred_results)\n\n        return summary_loss, val_f1_score, val_accuracy_score\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        self.optimizer.zero_grad() #very important\n        for step, batch_data_dict in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            batch_size = batch_data_dict['waveform'].shape[0]\n            \n            if self.use_mixup:\n                batch_data_dict['mixup_lambda'] = self.mixup_augmenter.get_lambda(len(batch_data_dict['waveform']))\n                batch_size = batch_size//2\n                batch_output_dict = self.model(torch.FloatTensor(batch_data_dict['waveform']).cuda(), \n                                               torch.FloatTensor(batch_data_dict['mixup_lambda']).cuda())\n                batch_target_dict = {'target': torch.FloatTensor(do_mixup(batch_data_dict['targets'], \n                                                                          batch_data_dict['mixup_lambda'][:, np.newaxis])).cuda()}\n            else:\n                batch_output_dict = self.model(torch.FloatTensor(batch_data_dict['waveform']).cuda(), None)\n                batch_target_dict = {'target': torch.FloatTensor(batch_data_dict['targets']).cuda()}\n            \n            loss = clip_bce(batch_output_dict, batch_target_dict['target'])\n            if self.use_amp:\n                with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n                    scaled_loss.backward()\n            else:\n                loss.backward()\n            if (step+1) % self.accumulate_steps == 0: # Wait for several backward steps\n                self.optimizer.step()                 # Now we can do an optimizer step\n                self.optimizer.zero_grad()\n\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n            \n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({'model_state_dict': self.model.state_dict()}, path)\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(list_data_dict):\n    \"\"\"Collate data.\n    Args:\n      list_data_dict, e.g., [{'waveform': (clip_samples,), ...}, \n                             {'waveform': (clip_samples,), ...},\n                             ...]\n    Returns:\n      np_data_dict, dict, e.g.,\n          {'waveform': (batch_size, clip_samples), ...}\n    \"\"\"\n    np_data_dict = {}\n    \n    for key in list_data_dict[0].keys():\n        #[print(data_dict[key].shape) for data_dict in list_data_dict]\n        np_data_dict[key] = np.array([data_dict[key] for data_dict in list_data_dict])\n    \n    return np_data_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training(net, config):\n    device = torch.device('cuda:0')\n    \n    train_dataset = birddataset(train_file_list, label_smooth=False, test=False)\n    validation_dataset = birddataset(val_file_list, label_smooth=False, test=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=config.batch_size, #*2 if config.use_mixup else config.batch_size,\n        sampler=SimpleBalanceClassSampler(all_targets, config.num_class),\n        pin_memory=False,\n        drop_last=True,\n        collate_fn=collate_fn,\n        num_workers=config.num_workers,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=config.batch_size,\n        num_workers=config.num_workers,\n        shuffle=False,\n        collate_fn = collate_fn,\n        pin_memory=False,\n    )\n\n    fitter = Fitter(model=net, device=device, config=config)\n    fitter.fit(train_loader, val_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Start Training","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"net = get_model(num_classes=config.num_class, pretrained=True)\nrun_training(net, config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## OOF(Simple)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}\nval_true = train_all.query(\"fold == @use_fold\")[\"ebird_code\"].values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def row_wise_f1_score_micro(y_true, y_pred):\n    \"\"\" author @shonenkov \"\"\"\n    F1 = []\n    for preds, trues in zip(y_pred, y_true):\n        TP, FN, FP = 0, 0, 0\n        preds = preds.split()\n        trues = trues.split()\n        for true in trues:\n            if true in preds:\n                TP += 1\n            else:\n                FN += 1\n        for pred in preds:\n            if pred not in trues:\n                FP += 1\n        F1.append(2*TP / (2*TP + FN + FP))\n    return np.mean(F1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_trained_model(weight_path=None):\n    \n    model_config = {\n        \"sample_rate\": 32000,\n        \"window_size\": 1024,\n        \"hop_size\": 320,\n        \"mel_bins\": 64,\n        \"fmin\": 50,\n        \"fmax\": 14000,\n        \"classes_num\":264\n        }\n\n    model = Transfer_Cnn14(**model_config)\n    if weight_path:\n        print(\"load pretrain weight: {}\".format(weight_path))\n        weights = torch.load(weight_path, map_location=torch.device('cpu'))\n        model.load_state_dict(weights['model_state_dict'])\n    model.cuda()\n    model.eval()\n    \n    return model\n\nbest_weight_path = glob(f'{config.folder}/best-checkpoint-*epoch.bin')[-1]\ntrained_net = get_trained_model(best_weight_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(data.Dataset):\n    def __init__(self, file_list):\n        \n        self.file_list = file_list \n        \n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self, idx: int):\n        \n        SR = 32000\n        wav_path, ebird_code = self.file_list[idx]\n        y, sr = sf.read(wav_path)\n        effective_length = sr * config.period\n        \n        if len(y) > effective_length:\n            y = y.astype(np.float32)\n            len_y = len(y)\n            start = 0\n            end = SR * 5\n            y_all = []\n            while len_y > start:\n                y_batch = y[start:end].astype(np.float32)\n                if len(y_batch) != (SR * 5):\n                    y_pad = np.zeros(5 * SR, dtype=np.float32)\n                    y_pad[:len(y_batch)] = y_batch\n                    y_all.append(y_pad)\n                    break\n                start = end\n                end = end + SR * 5\n                y_all.append(y_batch)\n            y_all = np.asarray(y_all)\n            return y_all\n        elif len(y) < effective_length:\n            new_wave = np.zeros(effective_length, dtype=y.dtype)\n            start = np.random.randint(effective_length - len(y))\n            new_wave[start:start + len(y)] = y\n            y = new_wave.astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction(val_file_list, model, threshold):\n    \n    SR = 32000\n    period = config.period\n    dataset = TestDataset(val_file_list)\n    loader = data.DataLoader(dataset, batch_size=1, shuffle=False)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    #j = 0\n    predictions = []\n    for wave in progress_bar(loader):\n\n        if wave.size()[1] == SR*period: \n            wave = wave.to(device).float()\n\n            with torch.no_grad():\n                prediction = model(wave)\n                event_proba = prediction['clipwise_output'].detach().cpu().numpy().reshape(-1)\n                \n            events = proba >= threshold\n            labels = np.argwhere(events).reshape(-1).tolist()\n\n        else:\n            # to avoid prediction on large batch\n            wave = wave.squeeze(0)\n            batch_size = 16\n            whole_size = wave.size(0)\n            if whole_size % batch_size == 0:\n                n_iter = whole_size // batch_size\n            else:\n                n_iter = whole_size // batch_size + 1\n                \n            all_events = set()\n            for batch_i in range(n_iter):\n                batch = wave[batch_i * batch_size:(batch_i + 1) * batch_size]\n                if batch.ndim == 3:\n                    batch = batch.unsqueeze(0)\n\n                batch = batch.to(device)\n                with torch.no_grad():\n                    prediction = model(batch)\n                    proba = prediction['clipwise_output'].detach().cpu().numpy()\n                    \n                events = proba >= threshold\n                for i in range(len(events)):\n                    event = events[i, :]\n                    labels = np.argwhere(event).reshape(-1).tolist()\n                    for label in labels:\n                        all_events.add(label)\n                        \n            labels = list(all_events)\n        if len(labels) == 0:\n            predictions.append(\"nocall\")\n        else:\n            labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))\n            label_string = \" \".join(labels_str_list)\n            predictions.append(label_string)\n\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_optimization(val_file_list, val_true, model):\n    \n    best_score = 0.0\n    best_threshold = 0.0\n    for threshold in np.arange(0.1, 1.0, 0.1):\n        val_predict = prediction(val_file_list, model, threshold)\n        score = row_wise_f1_score_micro(val_true, val_predict)\n        \n        if score >= best_score:\n            best_score = score\n            best_threshold = threshold\n\n    print(best_score, best_threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"threshold_optimization(val_file_list, val_true, model=trained_net)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}