{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport sys\n\nimport librosa\n#import audioread\nimport soundfile\nimport torch\nimport random\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedKFold\n\nimport fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import SaveModelCallback, ReduceLROnPlateauCallback\nfrom torch.utils.data import Dataset, DataLoader\nfrom radam import *\nfrom mish_activation import *\nimport glob\n#from torchlibrosa.augmentation import SpecAugmentation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = ['../input/birdsong-resampled-train-audio-00/','../input/birdsong-resampled-train-audio-01/',\n         '../input/birdsong-resampled-train-audio-02/','../input/birdsong-resampled-train-audio-03/',\n         '../input/birdsong-resampled-train-audio-04/']\nLABELS = '../input/birdsong-recognition/train.csv'\nNUM_WORKERS = 12\nnfolds = 4\nSEED = 2020\nOUT = 'model0'\nbs = 64#48\n\nclass config:\n    sampling_rate = 32000\n    duration = 5#20.03\n    samples = int(sampling_rate*duration)\n    top_db = 60 # Noise filtering, default = 60\n    \n    # Frequencies kept in spectrograms\n    fmin = 50\n    fmax =  14000\n\n    # Spectrogram parameters\n    n_mels = 128 # = spec_height\n    n_fft = 1024\n    hop_length = 313\n    \ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nos.makedirs(OUT, exist_ok=True)\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = []\nfor p in PATH:\n    files += glob.glob(p + '/*/*.wav')\nfiles = {os.path.basename(f):f for f in files}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(LABELS)\nlabel_map = {p:i for i,p in enumerate(sorted(df.ebird_code.unique()))}\ndf['label'] = df.ebird_code.map(label_map)\n\nsplits = StratifiedKFold(n_splits=nfolds, random_state=SEED, shuffle=True)\ndf['split'] = 0\nfor i,s in enumerate(list(splits.split(df,df.label))):\n    df.loc[s[1],'split'] = i\ndf['filename_w'] = [f[:-3] + 'wav' for f in df.filename]\ndf = df.loc[df.filename_w.isin(files.keys())].reset_index()\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean,std = -40.0,12.0 #quick estimation\n\ndef img2tensor(img,dtype:np.dtype=np.float32):\n    if img.ndim==2 : img = np.expand_dims(img,2)\n    img = np.transpose(img,(2,0,1))\n    return torch.from_numpy(img.astype(dtype, copy=False))\n\nclass BirdDataset(Dataset):\n    def __init__(self, df, fold=0, train=True, tfms=None):\n        self.df = df.copy()\n        self.df = self.df.loc[self.df.split != fold] if train else self.df.loc[self.df.split == fold]\n        self.df = self.df.reset_index(drop=True)\n        self.train = train\n        self.tfms = tfms\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        label,fname = self.df.iloc[idx][['label','filename_w']]\n        #sample according to length\n        #if self.train:\n        #    tmp_df = self.df.loc[self.df.label == label, 'duration']\n        #    duration = np.sqrt(tmp_df.values)\n        #    idx = np.random.choice(tmp_df.index, 1, p=duration/duration.sum())\n        #    fname = self.df.iloc[idx]['filename_w'].item()\n        #fname = os.path.join(PATH,fname)\n        fname = files[fname]\n        \n        l = soundfile.info(fname).frames\n        while l == 0: #there are corrupted files\n            idx = np.random.randint(len(self.df))\n            label,fname = self.df.iloc[idx][['label','filename_w']]\n            fname = os.path.join(PATH,fname)\n            l = soundfile.info(fname).frames   \n                \n        effective_length = config.samples\n        if l <= effective_length:\n            wave, sr = soundfile.read(fname)\n            new_wave = np.zeros(effective_length, dtype=wave.dtype)\n            start = np.random.randint(effective_length - len(wave)) if effective_length > len(wave) else 0\n            new_wave[start:start + l] = wave\n            wave = new_wave.astype(np.float32)\n        else:\n            start = np.random.randint(l - effective_length) if l > effective_length else 0\n            wave, sr = soundfile.read(fname,start=start,stop=start+effective_length)\n        wave= wave.astype(np.float32)\n\n        #norm wave and random rescale\n        #wave = wave*(0.025/max(wave.std(),0.01))\n        #if self.train: wave = wave*max(np.random.normal(1.0, 0.2),0.3)\n            \n        mel = librosa.feature.melspectrogram(wave, \n                    sr=config.sampling_rate,\n                    n_mels=config.n_mels,\n                    hop_length=config.hop_length,\n                    n_fft=config.n_fft,\n                    fmin=config.fmin,\n                    fmax=config.fmax)\n        logmel = librosa.power_to_db(mel).astype(np.float32)\n        return (img2tensor(logmel) - mean)/std, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttnBlock(nn.Module):\n    def __init__(self, n=512, nheads=8, dim_feedforward=512):\n        super().__init__()\n        self.attn = nn.MultiheadAttention(n,nheads)\n        self.norm = nn.LayerNorm(n)\n        self.drop = nn.Dropout(0.2)\n        \n    def forward(self, x):\n        shape = x.shape\n        x = x.view(shape[0],shape[1],-1).permute(2,0,1)\n        x = self.norm(self.drop(self.attn(x,x,x)[0]) + x)\n        x = x.permute(1,2,0).reshape(shape)\n        return x    \n\nclass Model(nn.Module):\n    def __init__(self, n=len(label_map), arch='resnext50_32x4d_ssl', \n                 path='facebookresearch/semi-supervised-ImageNet1K-models', ps=0.5):\n        super().__init__()\n        m = torch.hub.load(path, arch)\n        nc = list(m.children())[-1].in_features\n        self.enc = nn.Sequential(*list(m.children())[:-2])\n        \n        shape = self.enc[0].weight.shape\n        w = self.enc[0].weight.sum(1).unsqueeze(1)\n        self.enc[0] = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n        self.enc[0].weight = nn.Parameter(w)\n\n        nh = 768\n        self.head = nn.Sequential(nn.Conv2d(nc,nh,(config.n_mels//32,1)),AttnBlock(nh),AttnBlock(nh),\n                                  nn.Conv2d(nh,n,1))\n        \n    def forward(self, x):\n        x = self.head(self.enc(x))\n        #bs,n,1,len//32\n        return torch.logsumexp(x,-1).squeeze() - torch.Tensor([x.shape[-1]]).to(x.device).log()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OneHot(Callback):\n    def __init__(self, nunique=len(label_map)):\n        super().__init__()\n        self.nunique = nunique\n        \n    def on_batch_begin(self, last_target, **kwargs):\n        last_target = F.one_hot(last_target.long(), self.nunique).float()\n        return {'last_target': last_target}\n    \n#correct implementation of focal loss for soft labels\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n        \n    def forward(self, input, target, reduction='mean'):\n        n = input.shape[-1]\n        input = input.view(-1).float()\n        target = target.view(-1).float()\n        loss = -target*F.logsigmoid(input)*torch.exp(self.gamma*F.logsigmoid(-input)) -\\\n           (1.0 - target)*F.logsigmoid(-input)*torch.exp(self.gamma*F.logsigmoid(input))\n        \n        return n*loss.mean() if reduction=='mean' else loss\n    \ndef acc_m(x,y):\n    return (x.argmax(-1) == y.argmax(-1)).float().mean()\n\nclass FBetaMax(Callback):\n    def __init__(self, beta=1):\n        super().__init__()\n        self.beta = beta\n        self.preds = []\n        self.targets = []\n        \n    def on_epoch_begin(self, **kwargs):\n        self.preds = []\n        self.targets = []\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        self.preds.append(last_output.cpu())\n        self.targets.append(last_target.cpu())\n    \n    def on_epoch_end(self, last_metrics, **kwargs):\n        p = torch.cat(self.preds,0)\n        t = torch.cat(self.targets,0)\n        th1,th2 = p.min(), p.max()\n        nth = 1000\n        metric = torch.stack([fbeta(p,t,thresh=th1+(th2-th1)*i/(nth-1),\n                                    beta=self.beta,sigmoid=False) for i in range(nth)]).max()\n\n        return add_metrics(last_metrics, metric)\n    \nclass AccMax(Callback):\n    def __init__(self, beta=1):\n        super().__init__()\n        self.beta = beta\n        self.preds = []\n        self.targets = []\n        \n    def on_epoch_begin(self, **kwargs):\n        self.preds = []\n        self.targets = []\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        self.preds.append(last_output.cpu())\n        self.targets.append(last_target.cpu())\n        \n    def metric(self,p,t,th):\n        return ((p > th).long() == t.long()).min(-1)[0].float().mean()\n    \n    def on_epoch_end(self, last_metrics, **kwargs):\n        p = torch.cat(self.preds,0)\n        t = torch.cat(self.targets,0)\n        th1,th2 = p.min(), p.max()\n        nth = 1000\n        metric = torch.stack([self.metric(p,t,th1+(th2-th1)*i/(nth-1)) for i in range(nth)]).max()\n\n        return add_metrics(last_metrics, metric)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(x, y, alpha=0.4):\n    gamma = np.random.beta(alpha, alpha)\n    gamma = max(1-gamma, gamma)\n    shuffle = torch.randperm(x.shape[0]).to(x.device)\n    x = gamma*x + (1-gamma)*x[shuffle]\n    y = gamma*y + (1-gamma)*y[shuffle]\n    return x, y\n\ndef cutmix(x, ys, alpha=0.4):\n    gamma = np.random.beta(alpha, alpha)\n    gamma = max(1-gamma, gamma)\n    shuffle = torch.randperm(x.shape[0]).to(x.device)\n    ys_shuffle = [y[shuffle] for y in ys]\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), gamma)\n    x[..., bbx1:bbx2, bby1:bby2] = x[shuffle][..., bbx1:bbx2, bby1:bby2]\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    y = lam*y + (1-lam)*y[shuffle]\n    return x, y\n  \nclass Mixup(LearnerCallback):\n    def __init__(self, learn, alpha=0.4):\n        super().__init__(learn)\n        self.alpha = alpha\n        #self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n        #    freq_drop_width=8, freq_stripes_num=0)\n        \n    def on_batch_begin(self, last_input, last_target, train, **kwargs):\n        if not train: return\n        bs = last_input.shape[0]\n        #last_input = self.spec_augmenter(last_input)\n        last_input, last_target = mixup(last_input, last_target, self.alpha)\n        return {'last_input': last_input, 'last_target': last_target}\n    \nclass Cutmix(LearnerCallback):\n    def __init__(self, learn, alpha=1.0):\n        super().__init__(learn)\n        self.alpha = alpha\n        \n    def on_batch_begin(self, last_input, last_target, train, **kwargs):\n        if not train: return\n        bs = last_input.shape[0]\n        last_input, last_target = cutmix(last_input, last_target, self.alpha)\n        return {'last_input': last_input, 'last_target': last_target}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#5s\nfname = 'model0'\nfor fold in range(1):\n    ds_t = BirdDataset(df, fold=fold, train=True)\n    ds_v = BirdDataset(df, fold=fold, train=False)\n    dl_t = DataLoader(ds_t,bs,num_workers=NUM_WORKERS,shuffle=True)\n    dl_v = DataLoader(ds_v,bs,num_workers=NUM_WORKERS)\n    data = DataBunch(dl_t,dl_v)\n    model = Model()\n    model = nn.DataParallel(model)\n    learn = Learner(data, model, loss_func=FocalLoss(), opt_func=partial(Over9000,eps=1e-4),\n            metrics=[FBetaMax(),acc_m,AccMax()],).to_fp16(clip=1.0,max_noskip=100)\n    learn.callbacks.append(OneHot())\n    learn.callbacks.append(Mixup(learn))\n    learn.split([model.module.head])\n    learn.freeze_to(-1)\n    learn.fit_one_cycle(1, max_lr=1e-2, div_factor=5, pct_start=0.0)\n    learn.unfreeze()\n    #learn.callbacks.append(ReduceLROnPlateauCallback(learn=learn, monitor='f_beta_max', mode='max',\n    #                patience=2, factor=0.85, min_lr=1e-5))\n    #learn.fit(64, lr=0.75e-3, wd=1e-3, callbacks = [SaveModelCallback(learn,name=f'model',monitor='f_beta_max')])\n    learn.fit_one_cycle(16, max_lr=(1e-3,1e-2), div_factor=50, pct_start=0.0, \n          callbacks = [SaveModelCallback(learn,name=f'model',monitor='f_beta_max')])\n    torch.save(learn.model.module.state_dict(),os.path.join(OUT,f'{fname}_{fold}.pth'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}