{"cells":[{"metadata":{"papermill":{"duration":0.008815,"end_time":"2020-08-23T10:38:30.714007","exception":false,"start_time":"2020-08-23T10:38:30.705192","status":"completed"},"tags":[]},"cell_type":"markdown","source":"* The training process is here:\nhttps://www.kaggle.com/chanhu/training-bird-simple-baseline\n\n* This notebook is copied from Tawara, please upvote his work:\nhttps://www.kaggle.com/ttahara/inference-birdsong-baseline-resnest50-fast\n\n* V8:ResNet38 without Mixup LB:0.535\n* V9:ResNet38(With Mixup) LB: 0.561\n* V12：PANN CNN14_DecesionLevelAttetnion Inference('framewise_output') LB:0.291\n* V13：PANN CNN14_DecesionLevelAttetnion Inference('clipwise_output') LB:461\n* V14：PANN CNN14_DecesionLevelAttetnion Inference epoch 100('clipwise_output') LB:0.28\n* V15：ResNet38 Inference with SimpleClassBalanceSampler(best loss) LB:0.558\n* V17：ResNet38 Inference with SimpleClassBalanceSampler(best f1 score) LB:0.556\n* V18: ResNet38 Inference with SimpleClassBalanceSampler(best loss) LB:0.562\n* V20: ResNet38 Inference Change threshold from 0.5-> 0.6 LB:0.563\n* V21：Wavegram_Logmel_Cnn14, threshold:0.6 LB:0.564\n* V22: Wavegram_Logmel_Cnn14 + AttentionHead LB:0.542\n* V26：Wavegram_Logmel_Cnn14 3fold... LB:0.566\n* V27：Wavegram_Logmel_Cnn14 5fold\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:38:30.734549Z","iopub.status.busy":"2020-08-23T10:38:30.733725Z","iopub.status.idle":"2020-08-23T10:39:00.920125Z","shell.execute_reply":"2020-08-23T10:39:00.918584Z"},"papermill":{"duration":30.198673,"end_time":"2020-08-23T10:39:00.920294","exception":false,"start_time":"2020-08-23T10:38:30.721621","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install ../input/bird-panns/torchlibrosa-master/torchlibrosa-master/","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-08-23T10:39:00.950677Z","iopub.status.busy":"2020-08-23T10:39:00.949858Z","iopub.status.idle":"2020-08-23T10:39:05.065948Z","shell.execute_reply":"2020-08-23T10:39:05.065207Z"},"papermill":{"duration":4.134883,"end_time":"2020-08-23T10:39:05.066077","exception":false,"start_time":"2020-08-23T10:39:00.931194","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport math\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nimport yaml\nfrom joblib import delayed, Parallel\nfrom glob import glob\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\n\nimport numpy as np\nimport pandas as pd\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU\nfrom torch.nn.modules.utils import _pair\nimport torch.utils.data as data\nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation\n#from efficientnet_pytorch import EfficientNet\n\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.091262Z","iopub.status.busy":"2020-08-23T10:39:05.090447Z","iopub.status.idle":"2020-08-23T10:39:05.094847Z","shell.execute_reply":"2020-08-23T10:39:05.094358Z"},"papermill":{"duration":0.020585,"end_time":"2020-08-23T10:39:05.094949","exception":false,"start_time":"2020-08-23T10:39:05.074364","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n#     torch.backends.cudnn.deterministic = True  # type: ignore\n#     torch.backends.cudnn.benchmark = True  # type: ignore\n    \n\n@contextmanager\ndef timer(name: str) -> None:\n    \"\"\"Timer Util\"\"\"\n    t0 = time.time()\n    print(\"[{}] start\".format(name))\n    yield\n    print(\"[{}] done in {:.0f} s\".format(name, time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.115244Z","iopub.status.busy":"2020-08-23T10:39:05.114458Z","iopub.status.idle":"2020-08-23T10:39:05.120124Z","shell.execute_reply":"2020-08-23T10:39:05.119468Z"},"papermill":{"duration":0.017206,"end_time":"2020-08-23T10:39:05.120237","exception":false,"start_time":"2020-08-23T10:39:05.103031","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# logger = get_logger(\"main.log\")\nset_seed(42)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.14072Z","iopub.status.busy":"2020-08-23T10:39:05.140066Z","iopub.status.idle":"2020-08-23T10:39:05.144686Z","shell.execute_reply":"2020-08-23T10:39:05.144159Z"},"papermill":{"duration":0.016764,"end_time":"2020-08-23T10:39:05.144779","exception":false,"start_time":"2020-08-23T10:39:05.128015","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"SR = 32000\nROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\n# TRAIN_RESAMPLED_AUDIO_DIRS = [\n#   INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n# ]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.166649Z","iopub.status.busy":"2020-08-23T10:39:05.166055Z","iopub.status.idle":"2020-08-23T10:39:05.440292Z","shell.execute_reply":"2020-08-23T10:39:05.439293Z"},"papermill":{"duration":0.287998,"end_time":"2020-08-23T10:39:05.440418","exception":false,"start_time":"2020-08-23T10:39:05.15242","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(RAW_DATA / \"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.462007Z","iopub.status.busy":"2020-08-23T10:39:05.461243Z","iopub.status.idle":"2020-08-23T10:39:05.471522Z","shell.execute_reply":"2020-08-23T10:39:05.470625Z"},"papermill":{"duration":0.023162,"end_time":"2020-08-23T10:39:05.471655","exception":false,"start_time":"2020-08-23T10:39:05.448493","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.493922Z","iopub.status.busy":"2020-08-23T10:39:05.493009Z","iopub.status.idle":"2020-08-23T10:39:05.668458Z","shell.execute_reply":"2020-08-23T10:39:05.667341Z"},"papermill":{"duration":0.188261,"end_time":"2020-08-23T10:39:05.668603","exception":false,"start_time":"2020-08-23T10:39:05.480342","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\nsub.to_csv(\"submission.csv\", index=False)  # this will be overwritten if everything goes well","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-08-23T10:39:05.707948Z","iopub.status.busy":"2020-08-23T10:39:05.697411Z","iopub.status.idle":"2020-08-23T10:39:05.72841Z","shell.execute_reply":"2020-08-23T10:39:05.727889Z"},"papermill":{"duration":0.052079,"end_time":"2020-08-23T10:39:05.72851","exception":false,"start_time":"2020-08-23T10:39:05.676431","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.759188Z","iopub.status.busy":"2020-08-23T10:39:05.756976Z","iopub.status.idle":"2020-08-23T10:39:05.761868Z","shell.execute_reply":"2020-08-23T10:39:05.761318Z"},"papermill":{"duration":0.025539,"end_time":"2020-08-23T10:39:05.76198","exception":false,"start_time":"2020-08-23T10:39:05.736441","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class TestDataset(data.Dataset):\n    def __init__(self, df: pd.DataFrame, clip: np.ndarray):\n        \n        self.df = df\n        self.clip = clip\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = 32000\n        sample = self.df.loc[idx, :]\n        site = sample.site\n        row_id = sample.row_id\n        \n        if site == \"site_3\":\n            y = self.clip.astype(np.float32)\n            len_y = len(y)\n            start = 0\n            end = SR * 5\n            y_all = []\n            while len_y > start:\n                y_batch = y[start:end].astype(np.float32)\n                if len(y_batch) != (SR * 5):\n                    y_pad = np.zeros(5 * SR, dtype=np.float32)\n                    y_pad[:len(y_batch)] = y_batch\n                    y_all.append(y_pad)\n                    break\n                start = end\n                end = end + SR * 5\n                y_all.append(y_batch)\n            y_all = np.asarray(y_all)\n            return y_all, row_id, site\n        else:\n            end_seconds = int(sample.seconds)\n            start_seconds = int(end_seconds - 5)\n            \n            start_index = SR * start_seconds\n            end_index = SR * end_seconds\n            \n            y = self.clip[start_index:end_index].astype(np.float32)\n\n        return y, row_id, site","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-08-23T10:39:05.806027Z","iopub.status.busy":"2020-08-23T10:39:05.790185Z","iopub.status.idle":"2020-08-23T10:39:05.860805Z","shell.execute_reply":"2020-08-23T10:39:05.860167Z"},"papermill":{"duration":0.09151,"end_time":"2020-08-23T10:39:05.860939","exception":false,"start_time":"2020-08-23T10:39:05.769429","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n \n    if hasattr(layer, 'bias'):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n    \ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)\ndef _resnet_conv3x3(in_planes, out_planes):\n    #3x3 convolution with padding\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,\n                     padding=1, groups=1, bias=False, dilation=1)\n\n\ndef _resnet_conv1x1(in_planes, out_planes):\n    #1x1 convolution\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.conv2 = nn.Conv2d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x\nclass _ResnetBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(_ResnetBasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in _ResnetBasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n\n        self.stride = stride\n\n        self.conv1 = _resnet_conv3x3(inplanes, planes)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = _resnet_conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.conv1)\n        init_bn(self.bn1)\n        init_layer(self.conv2)\n        init_bn(self.bn2)\n        nn.init.constant_(self.bn2.weight, 0)\n\n    def forward(self, x):\n        identity = x\n\n        if self.stride == 2:\n            out = F.avg_pool2d(x, kernel_size=(2, 2))\n        else:\n            out = x\n\n        out = self.conv1(out)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = F.dropout(out, p=0.1, training=self.training)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        if self.downsample is not None:\n            identity = self.downsample(identity)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\nclass _ResNet(nn.Module):\n    def __init__(self, block, layers, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(_ResNet, self).__init__()\n\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n\n        self.layer1 = self._make_layer(block, 64, layers[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            if stride == 1:\n                downsample = nn.Sequential(\n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[0])\n                init_bn(downsample[1])\n            elif stride == 2:\n                downsample = nn.Sequential(\n                    nn.AvgPool2d(kernel_size=2), \n                    _resnet_conv1x1(self.inplanes, planes * block.expansion),\n                    norm_layer(planes * block.expansion),\n                )\n                init_layer(downsample[1])\n                init_bn(downsample[2])\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        return x\nclass ResNet38(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        \n        super(ResNet38, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        # self.conv_block2 = ConvBlock(in_channels=64, out_channels=64)\n\n        self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)\n\n        self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n\n\n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.resnet(x)\n        x = F.avg_pool2d(x, kernel_size=(2, 2))\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training, inplace=True)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-08-23T10:39:05.90504Z","iopub.status.busy":"2020-08-23T10:39:05.88941Z","iopub.status.idle":"2020-08-23T10:39:05.928341Z","shell.execute_reply":"2020-08-23T10:39:05.927784Z"},"papermill":{"duration":0.059626,"end_time":"2020-08-23T10:39:05.928436","exception":false,"start_time":"2020-08-23T10:39:05.86881","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class ConvPreWavBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvPreWavBlock, self).__init__()\n        \n        self.conv1 = nn.Conv1d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1,\n                              padding=1, bias=False)\n                              \n        self.conv2 = nn.Conv1d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=3, stride=1, dilation=2, \n                              padding=2, bias=False)\n                              \n        self.bn1 = nn.BatchNorm1d(out_channels)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        x = F.max_pool1d(x, kernel_size=pool_size)\n        \n        return x\n    \nclass Wavegram_Logmel_Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(Wavegram_Logmel_Cnn14, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        self.pre_conv0 = nn.Conv1d(in_channels=1, out_channels=64, kernel_size=11, stride=5, padding=5, bias=False)\n        self.pre_bn0 = nn.BatchNorm1d(64)\n        self.pre_block1 = ConvPreWavBlock(64, 64)\n        self.pre_block2 = ConvPreWavBlock(64, 128)\n        self.pre_block3 = ConvPreWavBlock(128, 128)\n        self.pre_block4 = ConvBlock(in_channels=4, out_channels=64)\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=128, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.pre_conv0)\n        init_bn(self.pre_bn0)\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n\n        # Wavegram\n        a1 = F.relu_(self.pre_bn0(self.pre_conv0(input[:, None, :])))\n        a1 = self.pre_block1(a1, pool_size=4)\n        a1 = self.pre_block2(a1, pool_size=4)\n        a1 = self.pre_block3(a1, pool_size=4)\n        a1 = a1.reshape((a1.shape[0], -1, 32, a1.shape[-1])).transpose(2, 3)\n        a1 = self.pre_block4(a1, pool_size=(2, 1))\n\n        # Log mel spectrogram\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n            a1 = do_mixup(a1, mixup_lambda)\n        \n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n\n        # Concatenate Wavegram and Log mel spectrogram along the channel dimension\n        x = torch.cat((x, a1), dim=1)\n\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:05.955115Z","iopub.status.busy":"2020-08-23T10:39:05.953284Z","iopub.status.idle":"2020-08-23T10:39:05.95587Z","shell.execute_reply":"2020-08-23T10:39:05.956325Z"},"papermill":{"duration":0.020073,"end_time":"2020-08-23T10:39:05.956435","exception":false,"start_time":"2020-08-23T10:39:05.936362","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_model(weight_path=None):\n    \n    model_config = {\n        \"sample_rate\": 32000,\n        \"window_size\": 1024,\n        \"hop_size\": 320,\n        \"mel_bins\": 64,\n        \"fmin\": 50,\n        \"fmax\": 14000,\n        \"classes_num\":264\n        }\n\n    model = Wavegram_Logmel_Cnn14(**model_config)\n    if weight_path:\n        print(\"load pretrain weight: {}\".format(weight_path))\n        weights = torch.load(weight_path, map_location=torch.device('cpu'))\n        model.load_state_dict(weights['model_state_dict'])\n    model.cuda()\n    model.eval()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.007217,"end_time":"2020-08-23T10:39:05.971337","exception":false,"start_time":"2020-08-23T10:39:05.96412","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Prediction loop"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-08-23T10:39:06.005474Z","iopub.status.busy":"2020-08-23T10:39:05.999999Z","iopub.status.idle":"2020-08-23T10:39:06.011007Z","shell.execute_reply":"2020-08-23T10:39:06.010515Z"},"papermill":{"duration":0.032111,"end_time":"2020-08-23T10:39:06.011105","exception":false,"start_time":"2020-08-23T10:39:05.978994","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prediction_for_clip(test_df, \n                        clip, \n                        models, \n                        threshold=0.5):\n\n    dataset = TestDataset(df=test_df, clip=clip)\n    loader = data.DataLoader(dataset, batch_size=1, shuffle=False)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    \n    prediction_dict = {}\n    for image, row_id, site in progress_bar(loader):\n        site = site[0]\n        row_id = row_id[0]\n        image = image.to(device).float()\n        if site in {\"site_1\", \"site_2\"}:\n            image = image.to(device).float()\n\n            with torch.no_grad():\n                proba = 0.0\n                for model in models:\n                    model.eval()\n                    prediction = model(image)\n                    proba += prediction['clipwise_output'].detach().cpu().numpy().reshape(-1)\n                proba /= len(models)\n                #print(proba.shape)\n\n            events = proba >= threshold\n            labels = np.argwhere(events).reshape(-1).tolist()\n\n        else:\n            # to avoid prediction on large batch\n            image = image.squeeze(0)\n            batch_size = 16\n            whole_size = image.size(0)\n            if whole_size % batch_size == 0:\n                n_iter = whole_size // batch_size\n            else:\n                n_iter = whole_size // batch_size + 1\n                \n            all_events = set()\n            for batch_i in range(n_iter):\n                batch = image[batch_i * batch_size:(batch_i + 1) * batch_size]\n                if batch.ndim == 3:\n                    batch = batch.unsqueeze(0)\n\n                batch = batch.to(device)\n                with torch.no_grad():\n                    proba = 0.0\n                    for model in models:\n                        model.eval()\n                        prediction = model(image)\n                        proba += prediction['clipwise_output'].detach().cpu().numpy()\n                    proba /= len(models)\n                #print(proba.shape)\n                    \n                events = proba >= threshold\n                #print(len(events))\n                for i in range(len(events)):\n                    event = events[i, :]\n                    labels = np.argwhere(event).reshape(-1).tolist()\n                    for label in labels:\n                        all_events.add(label)\n                        \n            labels = list(all_events)\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:06.039048Z","iopub.status.busy":"2020-08-23T10:39:06.038123Z","iopub.status.idle":"2020-08-23T10:39:06.040322Z","shell.execute_reply":"2020-08-23T10:39:06.04087Z"},"papermill":{"duration":0.02217,"end_time":"2020-08-23T10:39:06.040989","exception":false,"start_time":"2020-08-23T10:39:06.018819","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prediction(test_df,\n               test_audio,\n               weight_paths,\n               target_sr,\n               threshold=0.5):\n    \n    models = []\n    for weight_path in weight_paths:\n        models.append(get_model(weight_path))\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=target_sr,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_dict = prediction_for_clip(test_df_for_audio_id,\n                                                  clip=clip,\n                                                  models=models,\n                                                  threshold=threshold)\n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.007197,"end_time":"2020-08-23T10:39:06.055913","exception":false,"start_time":"2020-08-23T10:39:06.048716","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Prediction"},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-08-23T10:39:06.0769Z","iopub.status.busy":"2020-08-23T10:39:06.076272Z","iopub.status.idle":"2020-08-23T10:39:30.716057Z","shell.execute_reply":"2020-08-23T10:39:30.716592Z"},"papermill":{"duration":24.652942,"end_time":"2020-08-23T10:39:30.716743","exception":false,"start_time":"2020-08-23T10:39:06.063801","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = prediction(test_df=test,\n                           test_audio=TEST_AUDIO_DIR,\n                           weight_paths=glob(\"../input/pann-best-weights/*.bin\"),\n                           target_sr=32000,\n                           threshold=0.6)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:30.752996Z","iopub.status.busy":"2020-08-23T10:39:30.752298Z","iopub.status.idle":"2020-08-23T10:39:30.771273Z","shell.execute_reply":"2020-08-23T10:39:30.771755Z"},"papermill":{"duration":0.0427,"end_time":"2020-08-23T10:39:30.771919","exception":false,"start_time":"2020-08-23T10:39:30.729219","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-08-23T10:39:30.806097Z","iopub.status.busy":"2020-08-23T10:39:30.805066Z","iopub.status.idle":"2020-08-23T10:39:30.809553Z","shell.execute_reply":"2020-08-23T10:39:30.810059Z"},"papermill":{"duration":0.02511,"end_time":"2020-08-23T10:39:30.810179","exception":false,"start_time":"2020-08-23T10:39:30.785069","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission['birds'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.013182,"end_time":"2020-08-23T10:39:30.836469","exception":false,"start_time":"2020-08-23T10:39:30.823287","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## EOF"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}