{"cells":[{"metadata":{"id":"i-fnmf2m5T6h"},"cell_type":"markdown","source":"# Birdsong Pytorch Baseline: EnvNet V2 (Training)","execution_count":null},{"metadata":{"id":"stP2-qZq5T6j"},"cell_type":"markdown","source":"## import libraries","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"Ruc1WOfZ5T6p","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\n%timeit\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time, random, sys, os\nimport sklearn.metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\n\nfrom torch.optim.lr_scheduler import _LRScheduler, ReduceLROnPlateau, CosineAnnealingLR\nfrom torch.utils.data.dataset import Dataset\nfrom math import cos, pi\nimport librosa\nfrom scipy.io import wavfile\nimport torch.nn.functional as F\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","execution_count":null,"outputs":[]},{"metadata":{"id":"JI1-nFZf5T67"},"cell_type":"markdown","source":"## settings","execution_count":null},{"metadata":{"id":"KsEWo6Kjkudd","trusted":true},"cell_type":"code","source":"# set parameters\nNUM_FOLD = 5\nNUM_CLASS = 264\nSEED = 42\nNUM_EPOCH =10*5\nNUM_CYCLE = 10*5\nBATCH_SIZE = 32\nLR = [1e-1, 1e-8]\nFOLD_LIST = [1]\nCROP_LENGTH = 1000000\nFEATURE_PATH = '../input/birdsong-recognition/train_audio'\nOUTPUT_DIR = \"./\"\n\ncudnn.benchmark = True\nstarttime = time.time()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"id":"XIqFG9nG5T7L","trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"id":"8gxIeeSK5T7j"},"cell_type":"markdown","source":"## Training Utility","execution_count":null},{"metadata":{"id":"4TglSMQncuW1","trusted":true},"cell_type":"code","source":"def compute_gain(sound, fs, min_db=-80.0, mode='RMSE'):\n    if fs <= 32000:\n        n_fft = 2048\n    elif fs <= 44100:\n        n_fft = 4096\n    else:\n        raise Exception('Invalid fs {}'.format(fs))\n    stride = n_fft // 2\n\n    gain = []\n    for i in range(0, len(sound) - n_fft + 1, stride):\n        if mode == 'RMSE':\n            g = np.mean(sound[i: i + n_fft] ** 2)\n        elif mode == 'A_weighting':\n            spec = np.fft.rfft(np.hanning(n_fft + 1)[:-1] * sound[i: i + n_fft])\n            power_spec = np.abs(spec) ** 2\n            a_weighted_spec = power_spec * np.power(10, a_weight(fs, n_fft) / 10)\n            g = np.sum(a_weighted_spec)\n        else:\n            raise Exception('Invalid mode {}'.format(mode))\n        gain.append(g)\n\n    gain = np.array(gain)\n    gain = np.maximum(gain, np.power(10, min_db / 10))\n    gain_db = 10 * np.log10(gain)\n\n    return gain_db\n\n\ndef mix(sound1, sound2, r, fs):\n    gain1 = np.max(compute_gain(sound1, fs))  # Decibel\n    gain2 = np.max(compute_gain(sound2, fs))\n    t = 1.0 / (1 + np.power(10, (gain1 - gain2) / 20.) * (1 - r) / r)\n    sound = ((sound1 * t + sound2 * (1 - t)) / np.sqrt(t ** 2 + (1 - t) ** 2))\n    sound = sound.astype(np.float32)\n\n    return sound\n\n\nclass WaveDataset(Dataset):\n    def __init__(self, X, y,\n                 crop=-1, crop_mode='original', padding=0,\n                 mixup=False, scaling=-1, gain=-1,\n                 fs=44100,\n                 ):\n        self.X = X\n        self.y = y\n        self.crop = crop\n        self.crop_mode = crop_mode\n        self.padding = padding\n        self.mixup = mixup\n        self.scaling = scaling\n        self.gain = gain\n        self.fs = fs\n\n    def preprocess(self, sound):\n        for f in self.preprocess_funcs:\n            sound = f(sound)\n\n        return sound\n\n    def do_padding(self, snd):\n        snd_new = np.pad(snd, self.padding, 'constant')\n        return snd_new\n\n    def do_crop(self, snd):\n        if self.crop_mode=='random':\n            shift = np.random.randint(0, snd.shape[0] - self.crop)\n            snd_new = snd[shift:shift + self.crop]\n        else:\n            snd_new = snd\n        return snd_new\n\n    def do_gain(self, snd):\n        snd_new = snd * np.power(10, random.uniform(-self.gain, self.gain) / 20.0)\n        return snd_new\n\n    def do_scaling(self, snd, interpolate='Nearest'):\n        scale = np.power(self.scaling, random.uniform(-1, 1))\n        output_size = int(len(snd) * scale)\n        ref = np.arange(output_size) / scale\n        if interpolate == 'Linear':\n            ref1 = ref.astype(np.int32)\n            ref2 = np.minimum(ref1+1, len(snd)-1)\n            r = ref - ref1\n            snd_new = snd[ref1] * (1-r) + snd[ref2] * r\n        elif interpolate == 'Nearest':\n            snd_new = snd[ref.astype(np.int32)]\n        else:\n            raise Exception('Invalid interpolation mode {}'.format(interpolate))\n\n        return snd_new\n\n    def do_mixup(self, snd, label, alpha=1):\n        idx2 = np.random.randint(0, len(self.X))\n        snd2, _ = librosa.core.load(os.path.join(self.X, self.y['ebird_code'][idx2], self.y['filename'][idx2]), res_type=\"kaiser_fast\")\n        label2 = np.zeros(264).astype(np.float32)\n        label2[BIRD_CODE[self.y['ebird_code'][idx2]]] = 1.0\n        if self.scaling!=-1:\n            snd2 = self.do_scaling(snd2)\n        snd2 = self.do_padding(snd2)\n        snd2 = self.do_crop(snd2)\n\n        rate = np.random.beta(alpha, alpha)\n        snd_new = mix(snd, snd, rate, self.fs)\n        label_new = label * rate + label2 * (1 - rate)\n        return snd_new, label_new\n\n    def __getitem__(self, index):\n        snd, _ = librosa.core.load(os.path.join(self.X, self.y['ebird_code'][index], self.y['filename'][index]), res_type=\"kaiser_fast\")\n        # print(snd.shape)\n        label = np.zeros(264).astype(np.float32)\n        label[BIRD_CODE[self.y['ebird_code'][index]]] = 1.0\n        if self.scaling!=-1:\n            snd = self.do_scaling(snd)\n        snd = self.do_padding(snd)\n        snd = self.do_crop(snd)\n        if self.mixup:\n            snd, label = self.do_mixup(snd, label)\n        if self.gain!=-1:\n            snd = self.do_gain(snd)\n        snd = snd.reshape([1, 1, -1]).astype(np.float32) / 32768.0\n        return snd, label\n\n    def __len__(self):\n        return len(self.X)","execution_count":null,"outputs":[]},{"metadata":{"id":"O6TkA6ZTc75O","trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\n# All-in-one calculation of per-class lwlrap.\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class","execution_count":null,"outputs":[]},{"metadata":{"id":"XwUaiSSCcJP-","trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef cycle(iterable):\n    \"\"\"\n    convert dataloader to iterator\n    :param iterable:\n    :return:\n    \"\"\"\n    while True:\n        for x in iterable:\n            yield x\n\n\nclass CosineLR(_LRScheduler):\n    \"\"\"cosine annealing.\n    \"\"\"\n    def __init__(self, optimizer, step_size_min=1e-5, t0=100, tmult=2, curr_epoch=-1, last_epoch=-1):\n        self.step_size_min = step_size_min\n        self.t0 = t0\n        self.tmult = tmult\n        self.epochs_since_restart = curr_epoch\n        super(CosineLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        self.epochs_since_restart += 1\n\n        if self.epochs_since_restart > self.t0:\n            self.t0 *= self.tmult\n            self.epochs_since_restart = 0\n\n        lrs = [self.step_size_min + (\n                0.5 * (base_lr - self.step_size_min) * (1 + cos(self.epochs_since_restart * pi / self.t0)))\n               for base_lr in self.base_lrs]\n\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"id":"kqwdrllxlu1E"},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"id":"d9xjRTh6G3u1","trusted":true},"cell_type":"code","source":"class ConvBnRelu(nn.Module):\n    def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0, dilation=1, groups=1):\n        super(ConvBnRelu, self).__init__()\n        self.conv_bn_relu = nn.Sequential(\n            nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding, dilation, groups, False),\n            nn.BatchNorm2d(out_channel),\n            nn.ReLU(True))\n\n    def forward(self, x):\n        return self.conv_bn_relu(x)\n\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size()[0], -1)\n\n\nclass EnvNetv2(nn.Module):\n    def __init__(self, num_classes=1):\n        super(EnvNetv2, self).__init__()\n        self.conv1 = ConvBnRelu(1, 32, (1, 64), stride=(1, 2))\n        self.conv2 = ConvBnRelu(32, 64, (1, 16), stride=(1, 2))\n        self.conv3 = ConvBnRelu(1, 32, (8, 8))\n        self.conv4 = ConvBnRelu(32, 32, (8, 8))\n        self.conv5 = ConvBnRelu(32, 64, (1, 4))\n        self.conv6 = ConvBnRelu(64, 64, (1, 4))\n        self.conv7 = ConvBnRelu(64, 128, (1, 2))\n        self.conv8 = ConvBnRelu(128, 128, (1, 2))\n        self.conv9 = ConvBnRelu(128, 256, (1, 2))\n        self.conv10 = ConvBnRelu(256, 256, (1, 2))\n        self.maxpool1 = nn.MaxPool2d((1, 64), stride=(1, 64))\n        self.maxpool2 = nn.MaxPool2d((5, 3), stride=(5, 3))\n        self.maxpool3 = nn.MaxPool2d((1, 2), stride=(1, 2))\n        self.gmp = nn.AdaptiveMaxPool2d((10, 1))\n        self.flatten = Flatten()\n        self.last_linear1 = nn.Sequential(\n            nn.Linear(256 * 10, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, num_classes),\n        )\n        self.last_linear2 = nn.Sequential(\n            nn.Linear(256 * 10, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, num_classes),\n        )\n\n    def forward(self, input):\n        h = self.conv1(input)\n        h = self.conv2(h)\n        h = self.maxpool1(h)\n        h = h.transpose(1, 2)\n        h = self.conv3(h)\n        h = self.conv4(h)\n        h = self.maxpool2(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.maxpool3(h)\n        h = self.conv7(h)\n        h = self.conv8(h)\n        h = self.maxpool3(h)\n        h = self.conv9(h)\n        h = self.conv10(h)\n        h = self.gmp(h)\n        h = self.flatten(h)\n        h = self.last_linear1(h)\n        return h","execution_count":null,"outputs":[]},{"metadata":{"id":"araY0dLhmDSU"},"cell_type":"markdown","source":"## Train function","execution_count":null},{"metadata":{"id":"QSKtMCTKmJIC","trusted":true},"cell_type":"code","source":"def train(train_loaders, model, optimizer, scheduler, epoch):\n    train_loader = train_loaders\n    kl_avr = AverageMeter()\n    bce_avr = AverageMeter()\n    lsigmoid = nn.LogSigmoid().cuda()\n    lsoftmax = nn.LogSoftmax(dim=1).cuda()\n    softmax = nn.Softmax(dim=1).cuda()\n    criterion_kl = nn.KLDivLoss().cuda()\n    criterion_bce = nn.BCEWithLogitsLoss().cuda()\n\n    # switch to train mode\n    model.train()\n\n    # training\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(train_loader):\n        # get batches\n        input = torch.autograd.Variable(input.cuda())\n        target = torch.autograd.Variable(target.cuda())\n\n        # compute output\n        output = model(input)\n        kl = criterion_kl(lsoftmax(output), target)\n        bce = criterion_bce(output, target)\n        loss = bce\n        pred = softmax(output)\n        pred = pred.data.cpu().numpy()\n\n        # backprop\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step(metrics=loss)  # metrics=loss\n\n        # record log\n        kl_avr.update(kl.data, input.size(0))\n        bce_avr.update(bce.data, input.size(0))\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n\n    # calc metric\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true, preds)\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n\n    return kl_avr.avg.item(), lwlrap, bce_avr.avg.item()\n\n\ndef validate(val_loader, model):\n    kl_avr = AverageMeter()\n    bce_avr = AverageMeter()\n    lsoftmax = nn.LogSoftmax(dim=1).cuda()\n    softmax = torch.nn.Softmax(dim=1).cuda()\n    criterion_kl = nn.KLDivLoss().cuda()\n    criterion_bce = nn.BCEWithLogitsLoss().cuda()\n\n    # switch to eval mode\n    model.eval()\n\n    # validate\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(val_loader):\n        # get batches\n        input = torch.autograd.Variable(input.cuda())\n        target = torch.autograd.Variable(target.cuda())\n\n        # compute output\n        with torch.no_grad():\n            output = model(input)\n            kl = criterion_kl(lsoftmax(output), target)\n            bce = criterion_bce(output, target)\n            pred = softmax(output)\n            pred = pred.data.cpu().numpy()\n\n        # record log\n        kl_avr.update(kl.data, input.size(0))\n        bce_avr.update(bce.data, input.size(0))\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n\n    # calc metric\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true, preds)\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n\n    return kl_avr.avg.item(), lwlrap, bce_avr.avg.item()","execution_count":null,"outputs":[]},{"metadata":{"id":"SB-h9iOHmPe5"},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"id":"q9ho6I9TeLxX","trusted":true},"cell_type":"code","source":"# load table data\ndf_train = pd.read_csv(\"../input/birdsong-recognition/train.csv\")\n#     + pd.read_csv(\"../input/xeno-canto-bird-recordings-extended-a-m/train_extended.csv\")\n#     + pd.read_csv(\"../input/xeno-canto-bird-recordings-extended-n-z/train_extended.csv\")\ndf_test = pd.read_csv(\"../input/birdcall-check/test.csv\")\nsub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\n\n# fold splitting\n#folds = list(KFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED).split(np.arange(len(df_train))))\nfolds = list(StratifiedKFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED).split(df_train, df_train[\"ebird_code\"]))\n\n# Training\nlog_columns = ['epoch', 'kl', 'bce', 'lwlrap', 'val_kl', 'val_bce', 'val_lwlrap', 'time']\nfor fold, (ids_train_split, ids_valid_split) in enumerate(folds):\n    if fold+1 not in FOLD_LIST: continue\n    print(\"fold: {}\".format(fold + 1))\n    train_log = pd.DataFrame(columns=log_columns)\n\n    # build model\n    model = EnvNetv2(NUM_CLASS).cuda()\n\n    # prepare data loaders\n    df_train_fold = df_train.iloc[ids_train_split].reset_index(drop=True)\n    dataset_train = WaveDataset(FEATURE_PATH, df_train_fold,\n                                crop=CROP_LENGTH, crop_mode='random', padding=CROP_LENGTH//2,\n                                mixup=True, scaling=1.25, gain=6\n                                )\n    train_loader = DataLoader(dataset_train, batch_size=BATCH_SIZE,\n                                shuffle=True, num_workers=1, pin_memory=True,\n                                )\n\n    df_valid = df_train.iloc[ids_valid_split].reset_index(drop=True)\n    dataset_valid = WaveDataset(FEATURE_PATH, df_valid, padding=CROP_LENGTH//2)\n    valid_loader = DataLoader(dataset_valid, batch_size=1,\n                                shuffle=False, num_workers=1, pin_memory=True,\n                                )\n\n    # set optimizer and loss\n    optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=LR[0], momentum = 0.9, nesterov = True)\n    # optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=LR[0])\n    # scheduler = CosineLR(optimizer, step_size_min=LR[1], t0=len(train_loader) * NUM_CYCLE, tmult=1)\n    scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2, threshold=0.0001, min_lr=0.00000001)\n    # scheduler = CosineAnnealingLR(optimizer, T_max=10)\n\n    # training\n    best_val = 999\n    for epoch in range(NUM_EPOCH):\n        # train for one epoch\n        kl, lwlrap, bce = train(train_loader, model, optimizer, scheduler, epoch)\n\n        # evaluate on validation set\n        val_kl, val_lwlrap, val_bce = validate(valid_loader, model)\n\n        # print log\n        endtime = time.time() - starttime\n        print(\"Epoch: {}/{} \".format(epoch + 1, NUM_EPOCH)\n                + \"KL: {:.4f} \".format(kl)\n                + \"BCE: {:.4f} \".format(bce)\n                + \"LwLRAP: {:.4f} \".format(lwlrap)\n                + \"Valid KL: {:.4f} \".format(val_kl)\n                + \"Valid BCE: {:.4f} \".format(val_bce)\n                + \"Valid LWLRAP: {:.4f} \".format(val_lwlrap)\n                + \"sec: {:.1f}\".format(endtime)\n                )\n\n        # save log and weights\n        train_log_epoch = pd.DataFrame(\n            [[epoch+1, kl, bce, lwlrap, val_kl, val_bce, val_lwlrap, endtime]], columns=log_columns)\n        train_log = pd.concat([train_log, train_log_epoch])\n        train_log.to_csv(\"{}/train_log_fold{}.csv\".format(OUTPUT_DIR, fold+1), index=False)\n        if (epoch+1)%NUM_CYCLE==0:\n            torch.save(model.state_dict(), \"{}/weight_fold_{}_epoch_{}.pth\".format(OUTPUT_DIR, fold+1, epoch+1))\n        if best_val > val_bce:\n            torch.save(model.state_dict(), \"{}/weight_fold_{}_epoch_best.pth\".format(OUTPUT_DIR, fold+1))\n            best_val = val_bce","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}