{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install git+https://github.com/rwightman/pytorch-image-models.git","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:09.602774Z","iopub.execute_input":"2021-05-28T15:32:09.603157Z","iopub.status.idle":"2021-05-28T15:32:20.458465Z","shell.execute_reply.started":"2021-05-28T15:32:09.603080Z","shell.execute_reply":"2021-05-28T15:32:20.457548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport math\nimport random\nimport warnings\nfrom tqdm import tqdm\n\nimport albumentations as A\nimport cv2\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport timm\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\n\nfrom timm.models.layers import SelectAdaptivePool2d\nfrom torch.optim.optimizer import Optimizer","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:20.461286Z","iopub.execute_input":"2021-05-28T15:32:20.461660Z","iopub.status.idle":"2021-05-28T15:32:25.285820Z","shell.execute_reply.started":"2021-05-28T15:32:20.461624Z","shell.execute_reply":"2021-05-28T15:32:25.283268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    ######################\n    # Globals #\n    ######################\n    device = torch.device('cuda')\n    seed = 1213\n    epochs = 10\n    img_size = 224\n    LR = 0.001\n    lr_min = 1e-5\n \n    ######################\n    # Data #\n    ######################\n    train_csv = \"../input/birdclef-utils/train.csv\"\n    train_npy_dir = '../input/kkiller-birdclef-2021/audio_images/'\n\n    ######################\n    # Dataset #\n    ######################\n    duration = 20\n    n_mels = 128\n    sr = 32000\n\n    target_columns = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']\n\n    ######################\n    # Loaders #\n    ######################\n    train_batch_size = 32\n    num_workers =  4\n    valid_batch_size = 64\n\n    ######################\n    # Model #\n    ######################\n    base_model_name = \"densenet121\"\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 397\n    in_channels = 1\n\n    ######################\n    # Criterion #\n    ######################\n    loss = \"BCE\"","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.288384Z","iopub.execute_input":"2021-05-28T15:32:25.289024Z","iopub.status.idle":"2021-05-28T15:32:25.323106Z","shell.execute_reply.started":"2021-05-28T15:32:25.288973Z","shell.execute_reply":"2021-05-28T15:32:25.322102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"class Swish_func(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n    \n\nclass Swish(nn.Module):\n    def __init__(self, **kwargs):\n        super().__init__()\n        pass\n    def forward(self, input_tensor):\n        return Swish_func.apply(input_tensor)\n\n\nclass Mish_func(torch.autograd.Function):\n    \"\"\"from: https://github.com/tyunist/memory_efficient_mish_swish/blob/master/mish.py\"\"\"\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.tanh(F.softplus(i))\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        v = 1. + i.exp()\n        h = v.log() \n        grad_gh = 1./h.cosh().pow_(2) \n        grad_hx = i.sigmoid()\n        grad_gx = grad_gh *  grad_hx  \n        grad_f =  torch.tanh(F.softplus(i)) + i * grad_gx \n        return grad_output * grad_f \n\n\nclass Mish(nn.Module):\n    def __init__(self, **kwargs):\n        super().__init__()\n        pass\n    def forward(self, input_tensor):\n        return Mish_func.apply(input_tensor)\n\n\ndef replace_activations(model, existing_layer, new_layer):\n    for name, module in reversed(model._modules.items()):\n        if len(list(module.children())) > 0:\n            model._modules[name] = replace_activations(module, existing_layer, new_layer)\n        if type(module) == existing_layer:\n            layer_old = module\n            layer_new = new_layer\n            model._modules[name] = layer_new\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.325877Z","iopub.execute_input":"2021-05-28T15:32:25.326512Z","iopub.status.idle":"2021-05-28T15:32:25.351610Z","shell.execute_reply.started":"2021-05-28T15:32:25.326462Z","shell.execute_reply":"2021-05-28T15:32:25.350537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def centralized_gradient(x, use_gc=True, gc_conv_only=False):\n    '''credit - https://github.com/Yonghongwei/Gradient-Centralization '''\n    if use_gc:\n        if gc_conv_only:\n            if len(list(x.size())) > 3:\n                x.add_(-x.mean(dim=tuple(range(1, len(list(x.size())))), keepdim=True))\n        else:\n            if len(list(x.size())) > 1:\n                x.add_(-x.mean(dim=tuple(range(1, len(list(x.size())))), keepdim=True))\n    return x\n\n\nclass Ranger(Optimizer):\n\n    def __init__(self, params, lr=1e-3,                       # lr\n                 alpha=0.5, k=5, N_sma_threshhold=5,           # Ranger options\n                 betas=(.95, 0.999), eps=1e-5, weight_decay=0,  # Adam options\n                 # Gradient centralization on or off, applied to conv layers only or conv + fc layers\n                 use_gc=True, gc_conv_only=False, gc_loc=True\n                 ):\n\n        # parameter checks\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(f'Invalid slow update rate: {alpha}')\n        if not 1 <= k:\n            raise ValueError(f'Invalid lookahead steps: {k}')\n        if not lr > 0:\n            raise ValueError(f'Invalid Learning Rate: {lr}')\n        if not eps > 0:\n            raise ValueError(f'Invalid eps: {eps}')\n\n        # parameter comments:\n        # beta1 (momentum) of .95 seems to work better than .90...\n        # N_sma_threshold of 5 seems better in testing than 4.\n        # In both cases, worth testing on your dataset (.90 vs .95, 4 vs 5) to make sure which works best for you.\n\n        # prep defaults and init torch.optim base\n        defaults = dict(lr=lr, alpha=alpha, k=k, step_counter=0, betas=betas,\n                        N_sma_threshhold=N_sma_threshhold, eps=eps, weight_decay=weight_decay)\n        super().__init__(params, defaults)\n\n        # adjustable threshold\n        self.N_sma_threshhold = N_sma_threshhold\n\n        # look ahead params\n\n        self.alpha = alpha\n        self.k = k\n\n        # radam buffer for state\n        self.radam_buffer = [[None, None, None] for ind in range(10)]\n\n        # gc on or off\n        self.gc_loc = gc_loc\n        self.use_gc = use_gc\n        self.gc_conv_only = gc_conv_only\n        # level of gradient centralization\n        #self.gc_gradient_threshold = 3 if gc_conv_only else 1\n\n        print(\n            f\"Ranger optimizer loaded. \\nGradient Centralization usage = {self.use_gc}\")\n        if (self.use_gc and self.gc_conv_only == False):\n            print(f\"GC applied to both conv and fc layers\")\n        elif (self.use_gc and self.gc_conv_only == True):\n            print(f\"GC applied to conv layers only\")\n\n    def __setstate__(self, state):\n        print(\"set state called\")\n        super(Ranger, self).__setstate__(state)\n\n    def step(self, closure=None):\n        loss = None\n        # note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.\n        # Uncomment if you need to use the actual closure...\n\n        # if closure is not None:\n        #loss = closure()\n\n        # Evaluate averages and grad, update param tensors\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n\n                if grad.is_sparse:\n                    raise RuntimeError(\n                        'Ranger optimizer does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]  # get state dict for this param\n\n                if len(state) == 0:  # if first time to run...init dictionary with our desired entries\n                    # if self.first_run_check==0:\n                    # self.first_run_check=1\n                    #print(\"Initializing slow buffer...should not see this at load from saved model!\")\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n\n                    # look ahead weight storage now in state dict\n                    state['slow_buffer'] = torch.empty_like(p.data)\n                    state['slow_buffer'].copy_(p.data)\n\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(\n                        p_data_fp32)\n\n                # begin computations\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                # GC operation for Conv layers and FC layers\n                # if grad.dim() > self.gc_gradient_threshold:\n                #    grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True))\n                if self.gc_loc:\n                    grad = centralized_gradient(grad, use_gc=self.use_gc, gc_conv_only=self.gc_conv_only)\n\n                state['step'] += 1\n\n                # compute variance mov avg\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n\n                # compute mean moving avg\n                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n\n                buffered = self.radam_buffer[int(state['step'] % 10)]\n\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * \\\n                        state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n                    if N_sma > self.N_sma_threshhold:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (\n                            N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                # if group['weight_decay'] != 0:\n                #    p_data_fp32.add_(-group['weight_decay']\n                #                     * group['lr'], p_data_fp32)\n\n                # apply lr\n                if N_sma > self.N_sma_threshhold:\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    G_grad = exp_avg / denom\n                else:\n                    G_grad = exp_avg\n\n                if group['weight_decay'] != 0:\n                    G_grad.add_(p_data_fp32, alpha=group['weight_decay'])\n                # GC operation\n                if self.gc_loc == False:\n                    G_grad = centralized_gradient(G_grad, use_gc=self.use_gc, gc_conv_only=self.gc_conv_only)\n\n                p_data_fp32.add_(G_grad, alpha=-step_size * group['lr'])\n                p.data.copy_(p_data_fp32)\n\n                # integrated look ahead...\n                # we do it at the param level instead of group level\n                if state['step'] % group['k'] == 0:\n                    # get access to slow param tensor\n                    slow_p = state['slow_buffer']\n                    # (fast weights - slow weights) * alpha\n                    slow_p.add_(p.data - slow_p, alpha=self.alpha)\n                    # copy interpolated weights to RAdam param tensor\n                    p.data.copy_(slow_p)\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.353139Z","iopub.execute_input":"2021-05-28T15:32:25.353815Z","iopub.status.idle":"2021-05-28T15:32:25.396678Z","shell.execute_reply.started":"2021-05-28T15:32:25.353779Z","shell.execute_reply":"2021-05-28T15:32:25.395899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations","metadata":{}},{"cell_type":"code","source":"class TimeMask(object):\n    \n    def __init__(self, T = 40, num_masks = 1, replace_with_zero = False):\n        \n        self.T = T\n        self.num_masks = num_masks\n        self.replace_with_zero = replace_with_zero\n        \n    def __call__(self,spec):\n        \n        cloned = spec.clone()\n        len_spectro = cloned.shape[2]\n        \n        for i in range(0, self.num_masks):\n            t = random.randrange(0, self.T)\n            t_zero = random.randrange(0, len_spectro - t)\n            \n            if (t_zero == t_zero + t):\n                return cloned\n            \n            mask_end = random.randrange(t_zero, t_zero + t)\n            if(self.replace_with_zero):\n                cloned[0][:,t_zero:mask_end] = 0\n            else:\n                cloned[0][:,t_zero:mask_end] = cloned.mean()\n                \n        return cloned\n\nclass FreqMask(object):\n    \n    def __init__(self, F = 10, num_masks = 1, replace_with_zero = False):\n        self.F = F\n        self.num_masks = num_masks\n        self.replace_with_zero = replace_with_zero\n        \n    def __call__(self,spec):\n        cloned = spec.clone()\n        num_mel_channels = cloned.shape[1]\n\n        for i in range(0, self.num_masks):\n            f = random.randrange(0,self.F)\n            f_zero = random.randrange(0, num_mel_channels - f)\n\n            if f_zero == f_zero + f:\n                return cloned\n\n            mask_end = random.randrange(f_zero, f_zero + f)\n            if self.replace_with_zero:\n                cloned[0][f_zero:mask_end] = 0\n            else:\n                cloned[0][f_zero:mask_end] = cloned.mean()\n\n        return cloned\n\n\nclass TimeWarp(object):\n    \n    def __init__(self,W = 5):\n        self.W = W\n        \n    def __call__(self, spec):\n        num_rows = spec.shape[1]\n        spec_len = spec.shape[2]\n        device = spec.device\n        y = num_rows//2\n        horizontal_line_at_ctr = spec[0][y]\n        assert len(horizontal_line_at_ctr) == spec_len\n\n        point_to_warp = horizontal_line_at_ctr[random.randrange(self.W, spec_len - self.W)]\n        assert isinstance(point_to_warp, torch.Tensor)\n\n        dist_to_warp = random.randrange(-self.W, self.W)\n        src_pts, dest_pts = (torch.tensor([[[y, point_to_warp]]], device=device), \n                             torch.tensor([[[y, point_to_warp + dist_to_warp]]], device=device))\n        warped_spectro, dense_flows = sparse_image_warp(spec, src_pts, dest_pts)\n        return warped_spectro.squeeze(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.398197Z","iopub.execute_input":"2021-05-28T15:32:25.398762Z","iopub.status.idle":"2021-05-28T15:32:25.420053Z","shell.execute_reply.started":"2021-05-28T15:32:25.398724Z","shell.execute_reply":"2021-05-28T15:32:25.419029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sparse_image_warp(img_tensor,\n                      source_control_point_locations,\n                      dest_control_point_locations,\n                      interpolation_order=2,\n                      regularization_weight=0.0,\n                      num_boundaries_points=0):\n    control_point_flows = (dest_control_point_locations - source_control_point_locations)\n\n    batch_size, image_height, image_width = img_tensor.shape\n    grid_locations = get_grid_locations(image_height, image_width)\n    flattened_grid_locations = torch.tensor(flatten_grid_locations(grid_locations, image_height, image_width))\n\n    flattened_flows = interpolate_spline(\n        dest_control_point_locations,\n        control_point_flows,\n        flattened_grid_locations,\n        interpolation_order,\n        regularization_weight)\n\n    dense_flows = create_dense_flows(flattened_flows, batch_size, image_height, image_width)\n\n    warped_image = dense_image_warp(img_tensor, dense_flows)\n\n    return warped_image, dense_flows\n\n\ndef get_grid_locations(image_height, image_width):\n    \"\"\"Wrapper for np.meshgrid.\"\"\"\n\n    y_range = np.linspace(0, image_height - 1, image_height)\n    x_range = np.linspace(0, image_width - 1, image_width)\n    y_grid, x_grid = np.meshgrid(y_range, x_range, indexing='ij')\n    return np.stack((y_grid, x_grid), -1)\n\n\ndef flatten_grid_locations(grid_locations, image_height, image_width):\n    return np.reshape(grid_locations, [image_height * image_width, 2])\n\n\ndef create_dense_flows(flattened_flows, batch_size, image_height, image_width):\n    # possibly .view\n    return torch.reshape(flattened_flows, [batch_size, image_height, image_width, 2])\n\n\ndef interpolate_spline(train_points, train_values, query_points, order, regularization_weight=0.0, ):\n    # First, fit the spline to the observed data.\n    w, v = solve_interpolation(train_points, train_values, order, regularization_weight)\n    # Then, evaluate the spline at the query locations.\n    query_values = apply_interpolation(query_points, train_points, w, v, order)\n\n    return query_values\n\n\ndef solve_interpolation(train_points, train_values, order, regularization_weight):\n    b, n, d = train_points.shape\n    k = train_values.shape[-1]\n\n    # First, rename variables so that the notation (c, f, w, v, A, B, etc.)\n    # follows https://en.wikipedia.org/wiki/Polyharmonic_spline.\n    # To account for python style guidelines we use\n    # matrix_a for A and matrix_b for B.\n\n    c = train_points\n    f = train_values.float()\n\n    matrix_a = phi(cross_squared_distance_matrix(c, c), order).unsqueeze(0)  # [b, n, n]\n    #     if regularization_weight > 0:\n    #         batch_identity_matrix = array_ops.expand_dims(\n    #           linalg_ops.eye(n, dtype=c.dtype), 0)\n    #         matrix_a += regularization_weight * batch_identity_matrix\n\n    # Append ones to the feature values for the bias term in the linear model.\n    ones = torch.ones(1, dtype=train_points.dtype).view([-1, 1, 1])\n    matrix_b = torch.cat((c, ones), 2).float()  # [b, n, d + 1]\n\n    # [b, n + d + 1, n]\n    left_block = torch.cat((matrix_a, torch.transpose(matrix_b, 2, 1)), 1)\n\n    num_b_cols = matrix_b.shape[2]  # d + 1\n\n    # In Tensorflow, zeros are used here. Pytorch gesv fails with zeros for some reason we don't understand.\n    # So instead we use very tiny randn values (variance of one, zero mean) on one side of our multiplication.\n    lhs_zeros = torch.randn((b, num_b_cols, num_b_cols)) / 1e10\n    right_block = torch.cat((matrix_b, lhs_zeros),\n                            1)  # [b, n + d + 1, d + 1]\n    lhs = torch.cat((left_block, right_block),\n                    2)  # [b, n + d + 1, n + d + 1]\n\n    rhs_zeros = torch.zeros((b, d + 1, k), dtype=train_points.dtype).float()\n    rhs = torch.cat((f, rhs_zeros), 1)  # [b, n + d + 1, k]\n\n    # Then, solve the linear system and unpack the results.\n    X, LU = torch.solve(rhs, lhs)\n    w = X[:, :n, :]\n    v = X[:, n:, :]\n\n    return w, v\n\n\ndef cross_squared_distance_matrix(x, y):\n    \"\"\"Pairwise squared distance between two (batch) matrices' rows (2nd dim).\n        Computes the pairwise distances between rows of x and rows of y\n        Args:\n        x: [batch_size, n, d] float `Tensor`\n        y: [batch_size, m, d] float `Tensor`\n        Returns:\n        squared_dists: [batch_size, n, m] float `Tensor`, where\n        squared_dists[b,i,j] = ||x[b,i,:] - y[b,j,:]||^2\n    \"\"\"\n    x_norm_squared = torch.sum(torch.mul(x, x))\n    y_norm_squared = torch.sum(torch.mul(y, y))\n\n    x_y_transpose = torch.matmul(x.squeeze(0), y.squeeze(0).transpose(0, 1))\n\n    # squared_dists[b,i,j] = ||x_bi - y_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj\n    squared_dists = x_norm_squared - 2 * x_y_transpose + y_norm_squared\n\n    return squared_dists.float()\n\n\ndef phi(r, order):\n    \"\"\"Coordinate-wise nonlinearity used to define the order of the interpolation.\n    See https://en.wikipedia.org/wiki/Polyharmonic_spline for the definition.\n    Args:\n    r: input op\n    order: interpolation order\n    Returns:\n    phi_k evaluated coordinate-wise on r, for k = r\n    \"\"\"\n    EPSILON = torch.tensor(1e-10)\n    # using EPSILON prevents log(0), sqrt0), etc.\n    # sqrt(0) is well-defined, but its gradient is not\n    if order == 1:\n        r = torch.max(r, EPSILON)\n        r = torch.sqrt(r)\n        return r\n    elif order == 2:\n        return 0.5 * r * torch.log(torch.max(r, EPSILON))\n    elif order == 4:\n        return 0.5 * torch.square(r) * torch.log(torch.max(r, EPSILON))\n    elif order % 2 == 0:\n        r = torch.max(r, EPSILON)\n        return 0.5 * torch.pow(r, 0.5 * order) * torch.log(r)\n    else:\n        r = torch.max(r, EPSILON)\n        return torch.pow(r, 0.5 * order)\n\n\ndef apply_interpolation(query_points, train_points, w, v, order):\n    \"\"\"Apply polyharmonic interpolation model to data.\n    Given coefficients w and v for the interpolation model, we evaluate\n    interpolated function values at query_points.\n    Args:\n    query_points: `[b, m, d]` x values to evaluate the interpolation at\n    train_points: `[b, n, d]` x values that act as the interpolation centers\n                    ( the c variables in the wikipedia article)\n    w: `[b, n, k]` weights on each interpolation center\n    v: `[b, d, k]` weights on each input dimension\n    order: order of the interpolation\n    Returns:\n    Polyharmonic interpolation evaluated at points defined in query_points.\n    \"\"\"\n    query_points = query_points.unsqueeze(0)\n    # First, compute the contribution from the rbf term.\n    pairwise_dists = cross_squared_distance_matrix(query_points.float(), train_points.float())\n    phi_pairwise_dists = phi(pairwise_dists, order)\n\n    rbf_term = torch.matmul(phi_pairwise_dists, w)\n\n    # Then, compute the contribution from the linear term.\n    # Pad query_points with ones, for the bias term in the linear model.\n    ones = torch.ones_like(query_points[..., :1])\n    query_points_pad = torch.cat((\n        query_points,\n        ones\n    ), 2).float()\n    linear_term = torch.matmul(query_points_pad, v)\n\n    return rbf_term + linear_term\n\n\ndef dense_image_warp(image, flow):\n    \"\"\"Image warping using per-pixel flow vectors.\n    Apply a non-linear warp to the image, where the warp is specified by a dense\n    flow field of offset vectors that define the correspondences of pixel values\n    in the output image back to locations in the  source image. Specifically, the\n    pixel value at output[b, j, i, c] is\n    images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c].\n    The locations specified by this formula do not necessarily map to an int\n    index. Therefore, the pixel value is obtained by bilinear\n    interpolation of the 4 nearest pixels around\n    (b, j - flow[b, j, i, 0], i - flow[b, j, i, 1]). For locations outside\n    of the image, we use the nearest pixel values at the image boundary.\n    Args:\n    image: 4-D float `Tensor` with shape `[batch, height, width, channels]`.\n    flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`.\n    name: A name for the operation (optional).\n    Note that image and flow can be of type tf.half, tf.float32, or tf.float64,\n    and do not necessarily have to be the same type.\n    Returns:\n    A 4-D float `Tensor` with shape`[batch, height, width, channels]`\n    and same type as input image.\n    Raises:\n    ValueError: if height < 2 or width < 2 or the inputs have the wrong number\n    of dimensions.\n    \"\"\"\n    image = image.unsqueeze(3)  # add a single channel dimension to image tensor\n    batch_size, height, width, channels = image.shape\n\n    # The flow is defined on the image grid. Turn the flow into a list of query\n    # points in the grid space.\n    grid_x, grid_y = torch.meshgrid(\n        torch.arange(width), torch.arange(height))\n\n    stacked_grid = torch.stack((grid_y, grid_x), dim=2).float()\n\n    batched_grid = stacked_grid.unsqueeze(-1).permute(3, 1, 0, 2)\n\n    query_points_on_grid = batched_grid - flow\n    query_points_flattened = torch.reshape(query_points_on_grid,\n                                           [batch_size, height * width, 2])\n    # Compute values at the query points, then reshape the result back to the\n    # image grid.\n    interpolated = interpolate_bilinear(image, query_points_flattened)\n    interpolated = torch.reshape(interpolated,\n                                 [batch_size, height, width, channels])\n    return interpolated\n\n\ndef interpolate_bilinear(grid,\n                         query_points,\n                         name='interpolate_bilinear',\n                         indexing='ij'):\n    \"\"\"Similar to Matlab's interp2 function.\n    Finds values for query points on a grid using bilinear interpolation.\n    Args:\n    grid: a 4-D float `Tensor` of shape `[batch, height, width, channels]`.\n    query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 2]`.\n    name: a name for the operation (optional).\n    indexing: whether the query points are specified as row and column (ij),\n      or Cartesian coordinates (xy).\n    Returns:\n    values: a 3-D `Tensor` with shape `[batch, N, channels]`\n    Raises:\n    ValueError: if the indexing mode is invalid, or if the shape of the inputs\n      invalid.\n    \"\"\"\n    if indexing != 'ij' and indexing != 'xy':\n        raise ValueError('Indexing mode must be \\'ij\\' or \\'xy\\'')\n\n    shape = grid.shape\n    if len(shape) != 4:\n        msg = 'Grid must be 4 dimensional. Received size: '\n        raise ValueError(msg + str(grid.shape))\n\n    batch_size, height, width, channels = grid.shape\n\n    shape = [batch_size, height, width, channels]\n    query_type = query_points.dtype\n    grid_type = grid.dtype\n\n    num_queries = query_points.shape[1]\n\n    alphas = []\n    floors = []\n    ceils = []\n    index_order = [0, 1] if indexing == 'ij' else [1, 0]\n    unstacked_query_points = query_points.unbind(2)\n\n    for dim in index_order:\n        queries = unstacked_query_points[dim]\n\n        size_in_indexing_dimension = shape[dim + 1]\n\n        # max_floor is size_in_indexing_dimension - 2 so that max_floor + 1\n        # is still a valid index into the grid.\n        max_floor = torch.tensor(size_in_indexing_dimension - 2, dtype=query_type)\n        min_floor = torch.tensor(0.0, dtype=query_type)\n        maxx = torch.max(min_floor, torch.floor(queries))\n        floor = torch.min(maxx, max_floor)\n        int_floor = floor.long()\n        floors.append(int_floor)\n        ceil = int_floor + 1\n        ceils.append(ceil)\n\n        # alpha has the same type as the grid, as we will directly use alpha\n        # when taking linear combinations of pixel values from the image.\n        alpha = (queries - floor).clone().detach()\n        min_alpha = torch.tensor(0.0, dtype=grid_type)\n        max_alpha = torch.tensor(1.0, dtype=grid_type)\n        alpha = torch.min(torch.max(min_alpha, alpha), max_alpha)\n\n        # Expand alpha to [b, n, 1] so we can use broadcasting\n        # (since the alpha values don't depend on the channel).\n        alpha = torch.unsqueeze(alpha, 2)\n        alphas.append(alpha)\n\n    flattened_grid = torch.reshape(\n        grid, [batch_size * height * width, channels])\n    batch_offsets = torch.reshape(\n        torch.arange(batch_size) * height * width, [batch_size, 1])\n\n    # This wraps array_ops.gather. We reshape the image data such that the\n    # batch, y, and x coordinates are pulled into the first dimension.\n    # Then we gather. Finally, we reshape the output back. It's possible this\n    # code would be made simpler by using array_ops.gather_nd.\n    def gather(y_coords, x_coords, name):\n        linear_coordinates = batch_offsets + y_coords * width + x_coords\n        gathered_values = torch.gather(flattened_grid.t(), 1, linear_coordinates)\n        return torch.reshape(gathered_values,\n                             [batch_size, num_queries, channels])\n\n    # grab the pixel values in the 4 corners around each query point\n    top_left = gather(floors[0], floors[1], 'top_left')\n    top_right = gather(floors[0], ceils[1], 'top_right')\n    bottom_left = gather(ceils[0], floors[1], 'bottom_left')\n    bottom_right = gather(ceils[0], ceils[1], 'bottom_right')\n\n    interp_top = alphas[1] * (top_right - top_left) + top_left\n    interp_bottom = alphas[1] * (bottom_right - bottom_left) + bottom_left\n    interp = alphas[0] * (interp_bottom - interp_top) + interp_top\n\n    return interp","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.422036Z","iopub.execute_input":"2021-05-28T15:32:25.422766Z","iopub.status.idle":"2021-05-28T15:32:25.480825Z","shell.execute_reply.started":"2021-05-28T15:32:25.422730Z","shell.execute_reply":"2021-05-28T15:32:25.479963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class BirdClefDataset(torch.utils.data.Dataset):\n\n    def __init__(self, df, sr = CFG.sr, duration = CFG.duration,augmentations = None):\n\n        self.df = df\n        self.sr = sr \n        self.duration = duration\n        self.augmentations = augmentations\n\n    def __len__(self):\n        return len(self.df)\n\n    @staticmethod\n    def normalize(image):\n        image = image / 255.0\n        image = image.unsqueeze(0)\n        return image\n\n    def __getitem__(self, idx):\n        '''\n        Please note that the numpy dataset should have shape (num_segments,n_mels,time_steps)\n        '''\n\n        row = self.df.iloc[idx]\n        impath = CFG.train_npy_dir + f\"{row.primary_label}/{row.filename}.npy\"\n\n        image = np.load(str(impath))\n        \n        ########## RANDOM SAMPLING ################\n        image = image[np.random.choice(len(image))]\n        ############################################\n        \n        image = torch.tensor(image).float()\n        image = self.normalize(image)\n        image = image.permute(0,2,1)\n\n        return {\n            'images' : image,\n            'labels' : torch.tensor(row[21:]).float()\n        }","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.483853Z","iopub.execute_input":"2021-05-28T15:32:25.484428Z","iopub.status.idle":"2021-05-28T15:32:25.495006Z","shell.execute_reply.started":"2021-05-28T15:32:25.484392Z","shell.execute_reply":"2021-05-28T15:32:25.494182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Debugging\ndf = pd.read_csv('../input/birdclef-utils/train.csv')\nd = BirdClefDataset(df)\nfor i in d:\n    print(i['images'].size())\n    print(i['labels'])\n    break\n    \ndel df,d,i","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:25.496779Z","iopub.execute_input":"2021-05-28T15:32:25.497414Z","iopub.status.idle":"2021-05-28T15:32:27.856797Z","shell.execute_reply.started":"2021-05-28T15:32:25.497373Z","shell.execute_reply":"2021-05-28T15:32:27.855876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SED Model\n\nComponents :-\n* Preprocessor : For converting Audio into Images , Already implemented it in another notebook . Preprocessor should take in audio and output numpy files of shape (num_segments,n_mels,frames(time_steps))\n* Encoder : CNN Architecture\n* Decoder or Attention Block : Final Output","metadata":{}},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:27.858073Z","iopub.execute_input":"2021-05-28T15:32:27.858560Z","iopub.status.idle":"2021-05-28T15:32:27.867867Z","shell.execute_reply.started":"2021-05-28T15:32:27.858520Z","shell.execute_reply":"2021-05-28T15:32:27.867044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:27.870201Z","iopub.execute_input":"2021-05-28T15:32:27.870454Z","iopub.status.idle":"2021-05-28T15:32:27.879398Z","shell.execute_reply.started":"2021-05-28T15:32:27.870424Z","shell.execute_reply":"2021-05-28T15:32:27.878694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n    \n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:27.880970Z","iopub.execute_input":"2021-05-28T15:32:27.881375Z","iopub.status.idle":"2021-05-28T15:32:27.891197Z","shell.execute_reply.started":"2021-05-28T15:32:27.881339Z","shell.execute_reply":"2021-05-28T15:32:27.890235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimmSED(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False, num_classes=24, in_channels=1):\n        super().__init__()\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        base_model = timm.create_model(\n            base_model_name, pretrained=pretrained, in_chans=in_channels)\n        layers = list(base_model.children())[:-2]\n        self.encoder = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            in_features = base_model.fc.in_features\n        else:\n            in_features = base_model.classifier.in_features\n        self.fc1 = nn.Linear(in_features, in_features, bias=True)\n        self.att_block = AttBlockV2(\n            in_features, num_classes, activation=\"sigmoid\")\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n\n    def forward(self, x):\n        # input = (batch_size,channels,time_steps,n_mels)\n        frames_num = x.shape[2]\n        \n        x = x.transpose(1, 3) #(bs,n_mels,time_steps,channels) \n        x = self.bn0(x) # (Applying batch norm on frequencey)\n        x = x.transpose(1, 3) #(bs,channels,time_steps,n_mels)\n\n        x = x.transpose(2, 3) # (bs,channels,n_mels,time_steps) or (batch_size, channels, freq, frames)\n        x = self.encoder(x) # (bs,channels,freq,frames)\n\n        x = torch.mean(x, dim=2) # (batch_size, channels, frames) Merging across frequency\n\n        # channel smoothing\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) # (batch_size, channels, frames) Pooling\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)  # (batch_size, channels, frames) Pooling\n        x = x1 + x2  # (batch_size, channels, frames) Adding pooled outputs\n\n        x = F.dropout(x, p=0.5, training=self.training)  # (batch_size, channels, frames) Applying Dropout\n        x = x.transpose(1, 2)  # (batch_size, frames, channels)\n        x = F.relu_(self.fc1(x)) # Applying fc on channels (projecting channels into different vec space)\n        x = x.transpose(1, 2)  # (batch_size, frames, channels)\n        x = F.dropout(x, p=0.5, training=self.training)  # (batch_size, frames, channels)\n        \n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            \"framewise_output\": framewise_output, ## Output_size = (batch_size,time_steps(num_frames),num_classes)\n            \"segmentwise_output\": segmentwise_output,  ## Output_size = (batch_size,frame,num_classes)\n            \"logit\": logit,\n            \"framewise_logit\": framewise_logit,\n            \"clipwise_output\": clipwise_output  ### Output_size = (batch_size,num_classes) at clip level\n        }\n\n        return output_dict","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:27.892440Z","iopub.execute_input":"2021-05-28T15:32:27.892792Z","iopub.status.idle":"2021-05-28T15:32:27.909275Z","shell.execute_reply.started":"2021-05-28T15:32:27.892757Z","shell.execute_reply":"2021-05-28T15:32:27.908538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Debugger","metadata":{}},{"cell_type":"code","source":"model = TimmSED('efficientnet_b0')\nt1 = torch.ones((1,1,1251,128))\nprint(model(t1)['segmentwise_output'].size())\nprint(model(t1)['framewise_output'].size())\nprint(model(t1)['clipwise_output'].size())\n\ndel model,t1","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:27.910513Z","iopub.execute_input":"2021-05-28T15:32:27.910896Z","iopub.status.idle":"2021-05-28T15:32:29.098460Z","shell.execute_reply.started":"2021-05-28T15:32:27.910861Z","shell.execute_reply":"2021-05-28T15:32:29.097543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Losses\n\n* BCE Focal Loss\n* BCE Normal Loss\n\nChoose anyone","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.099937Z","iopub.execute_input":"2021-05-28T15:32:29.100530Z","iopub.status.idle":"2021-05-28T15:32:29.109852Z","shell.execute_reply.started":"2021-05-28T15:32:29.100469Z","shell.execute_reply":"2021-05-28T15:32:29.109002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SEDLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.112613Z","iopub.execute_input":"2021-05-28T15:32:29.112902Z","iopub.status.idle":"2021-05-28T15:32:29.121240Z","shell.execute_reply.started":"2021-05-28T15:32:29.112866Z","shell.execute_reply":"2021-05-28T15:32:29.120335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_criterion(name='BCE'):\n    if name == 'BCE':\n        loss = SEDLoss()\n    else:\n        loss = BCEFocal2WayLoss()\n        \n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.122572Z","iopub.execute_input":"2021-05-28T15:32:29.123221Z","iopub.status.idle":"2021-05-28T15:32:29.130032Z","shell.execute_reply.started":"2021-05-28T15:32:29.123183Z","shell.execute_reply":"2021-05-28T15:32:29.129206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine","metadata":{}},{"cell_type":"code","source":"def train_fn(model,criterion, data_loader, optimizer, scheduler, i):\n    model.train()\n    fin_loss = 0.0\n    \n    tk = tqdm(data_loader, desc = \"Epoch\" + \" [TRAIN] \" + str(i+1))\n    for t,data in enumerate(tk):\n        for k,v in data.items():\n            data[k] = v.to(CFG.device)\n        optimizer.zero_grad()\n        output = model(data['images'])\n        loss = criterion(output,data['labels'])\n        loss.backward()\n        optimizer.step() \n        fin_loss += loss.item() \n    \n        tk.set_postfix({'loss' : '%.6f' %float(fin_loss/(t+1)), 'LR' : optimizer.param_groups[0]['lr']})\n        \n        \n    scheduler.step()\n    return None\n\n\ndef eval_fn(model, criterion, data_loader, i):\n    model.eval()\n    fin_loss = 0.0\n\n    tk = tqdm(data_loader, desc = \"Epoch\" + \" [VALID] \" + str(i+1))\n\n    with torch.no_grad():\n        for t,data in enumerate(tk):\n            for k,v in data.items():\n                data[k] = v.to(CFG.device)\n            output = model(data['images'])\n            loss = criterion(output,data['labels'])\n            fin_loss += loss.item() \n\n            tk.set_postfix({'loss' : '%.6f' %float(fin_loss/(t+1))})\n\n    return fin_loss/len(data_loader)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.131259Z","iopub.execute_input":"2021-05-28T15:32:29.131802Z","iopub.status.idle":"2021-05-28T15:32:29.143374Z","shell.execute_reply.started":"2021-05-28T15:32:29.131766Z","shell.execute_reply":"2021-05-28T15:32:29.142579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"def run_training():\n    \n    df = pd.read_csv(CFG.train_csv)\n\n    for fold in range(0,1):\n        print(f'On fold : {fold}')\n        df_train = df[df['fold'] != fold]\n        df_valid = df[df['fold'] == fold]\n\n        trainset = BirdClefDataset(df_train)  \n        validset = BirdClefDataset(df_valid)  \n\n        trainloader = DataLoader(\n            trainset,\n            batch_size = CFG.train_batch_size,\n            shuffle = True, \n            num_workers = CFG.num_workers,\n            pin_memory = True\n        )\n\n        validloader = DataLoader(\n            validset,\n            batch_size = CFG.valid_batch_size,\n            shuffle = True,\n            num_workers = CFG.num_workers,\n            pin_memory = True\n        )\n\n        model = TimmSED(\n            base_model_name=CFG.base_model_name,\n            pretrained=CFG.pretrained,\n            num_classes = CFG.num_classes,\n            in_channels = CFG.in_channels  \n        )\n        #model = replace_activations(model, torch.nn.ReLU, Mish())\n        model = model.to(CFG.device)\n        \n        criterion = get_criterion(CFG.loss)\n        optimizer = Ranger(model.parameters(), lr = CFG.LR)\n        scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.epochs, eta_min=CFG.lr_min)\n\n        best_loss = np.inf\n        for i in range(CFG.epochs):\n            train_score = train_fn(model,criterion, trainloader, optimizer, scheduler, i)\n            val_loss = eval_fn(model,criterion, validloader, i)\n\n            if val_loss < best_loss:\n                best_loss = val_loss\n                torch.save(model.state_dict(), f\"{CFG.base_model_name}_sed_model_fold-{fold}.pt\")","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.144499Z","iopub.execute_input":"2021-05-28T15:32:29.144975Z","iopub.status.idle":"2021-05-28T15:32:29.155766Z","shell.execute_reply.started":"2021-05-28T15:32:29.144940Z","shell.execute_reply":"2021-05-28T15:32:29.154931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_training()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T15:32:29.158701Z","iopub.execute_input":"2021-05-28T15:32:29.159004Z","iopub.status.idle":"2021-05-28T15:34:06.168293Z","shell.execute_reply.started":"2021-05-28T15:32:29.158957Z","shell.execute_reply":"2021-05-28T15:34:06.165265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}