{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n\n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import IPython.display as ipd\n\nimport torch\nimport random\nimport numpy as np\nimport pandas as pd\nimport wave\nfrom scipy.io import wavfile\nimport os\nimport librosa\nfrom librosa.feature import melspectrogram\nimport warnings\nfrom sklearn.utils import shuffle\nfrom sklearn.utils import class_weight\nfrom PIL import Image\nfrom uuid import uuid4\nimport sklearn\nfrom tqdm import tqdm\nimport soundfile as sf\nfrom sklearn.metrics import f1_score\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.nn import functional as F\nfrom glob import glob\nimport sklearn\nfrom torch import nn\nimport warnings\nimport cv2\n\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\nimport pandas as pd\n\nIM_SIZE=256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATHS = dict()\nfor x in os.listdir('/kaggle/input/'):\n        for y in os.listdir('/kaggle/input/'+x):\n            if 'csv' not in y:\n                for z in os.listdir('/kaggle/input/'+x+'/'+y):\n                        PATHS[z.split('.')[0]]='/kaggle/input/'+x+'/'+y+'/'+z","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/birdsong-resampled-train-audio-04/train_mod.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntrain[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train, train[\"ebird_code\"])):\n    train.iloc[val_index, -1] = fold_id\n    \ntrain.fold.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sf.read(PATHS['XC124527'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATHS[train.loc[0].filename[:-4]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"species=train.species.value_counts().keys()\nBIRD_NUM=dict(zip(species,range(len(species))))\nNUM_BIRD=dict(zip(range(len(species)),species))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH='/kaggle/input/birdsong-recognition/train_audio/'\nPERIOD=5\nmelspectrogram_parameters= {'n_mels': 128, 'fmin': 20, 'fmax': 16000}\n\n\n\ndef mono_to_color(\n    X: np.ndarray, mean=None, std=None,\n    norm_max=None, norm_min=None, eps=1e-6\n):\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n    def __init__(self,\n                 df,\n                 size=None,\n                 train=True,\n                 melspectrogram_parameters=None\n                ):\n        self.img_size=128\n        self.melspectrogram_parameters = melspectrogram_parameters\n        self.df = df.reset_index(drop=True)\n        self.train = train\n        \n        self.change_size(size)\n    def change_size(self,size):\n        IM_SIZE=size\n        if self.train:\n            self.transform=None\n        else:\n            self.transform =None\n    def get_spectogram(self,item):\n\n        #wav_path = PATH+item.ebird_code+'/'+item.filename\n        wave_data, sr = sf.read(PATHS[item.filename.split('.')[0]])#librosa.load(wav_path)\n        y, _ = librosa.effects.trim(wave_data)\n        sample_length = 5*sr\n        len_y = len(y)\n        effective_length = sr * PERIOD\n        if len_y < effective_length:\n                    new_y = np.zeros(effective_length, dtype=y.dtype)\n                    start = np.random.randint(effective_length - len_y) if self.train else 0\n                    new_y[start:start + len_y] = y\n                    y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n                    start = np.random.randint(len_y - effective_length) if self.train else 0\n                    y = y[start:start + effective_length].astype(np.float32) \n        else:\n                    y = y.astype(np.float32)\n        \n        melspec = librosa.feature.melspectrogram(y, sr=sr, **self.melspectrogram_parameters)\n       \n        melspec = librosa.power_to_db(melspec).astype(np.float32)\n      \n        \n        '''if self.spectrogram_transforms:\n            melspec = self.spectrogram_transforms(melspec)\n        else:\n            pass'''\n        \n        image = mono_to_color(melspec)\n        height, width, _ = image.shape\n        image = cv2.resize(image, (int(width * self.img_size / height), self.img_size))\n        image = np.moveaxis(image, 2, 0)\n        image = (image / 255.0).astype(np.float32)\n\n        #  labels = np.zeros(len(BIRD_CODE), dtype=\"i\")\n        labels = np.zeros(len(BIRD_NUM), dtype=\"f\")\n        labels[BIRD_NUM[item.species]] = 1\n       \n        return image, labels\n        \n    \n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        item = self.df.iloc[index]\n    \n        return self.prepare_img(item)        \n    def get_labels(self):\n        return list(self.df.target.values)\n    def prepare_img(self,item):\n        image,target =self.get_spectogram(item)\n        return torch.tensor(image), torch.tensor(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds=DatasetRetriever(train.reset_index(drop=True),\n                 size=None,\n                 train=True,\n                 melspectrogram_parameters=melspectrogram_parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x,y=ds[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(x.permute(1,2,0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=train\nfold_number=0\ntrain_dataset = DatasetRetriever(\n    df=df[df.fold != fold_number],\n    size=IM_SIZE,\n    train=True,melspectrogram_parameters=melspectrogram_parameters\n)\n\n\nvalidation_dataset = DatasetRetriever(\n    df=df[df.fold==fold_number],\n    size=IM_SIZE,\n    train=False,\n    melspectrogram_parameters=melspectrogram_parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(train_dataset[0][0].permute(1,2,0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n        self.threshold=0.5\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n        self.y_true = np.zeros((1,264))#np.array([1])\n        self.y_pred = np.zeros((1,264))#np.array([1])\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.zeros((1,264))\n        self.y_pred = np.zeros((1,264))\n        self.score = []\n\n    def update(self, y_true, y_pred):\n        \n        #print('in')\n        y_true = y_true.cpu().numpy()\n        y_pred =y_pred.detach().cpu().numpy()\n        self.y_true = np.concatenate((self.y_true, y_true))\n        self.y_pred = np.concatenate((self.y_pred, y_pred))\n        self.score.append((y_true.argmax(axis=1)==y_pred.argmax(axis=1)).mean())#$, average='samples')\n        #print(self.score,'<<<<')\n    @property\n    def avg(self):\n        return np.mean(self.score)\n    \nclass APScoreMeter(object):\n    def __init__(self):\n        self.reset()\n    def reset(self):\n        self.y_true = np.zeros((1,264))\n        self.y_pred = np.zeros((1,264))\n        self.score = []\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy()\n        y_pred = np.where(y_pred.sigmoid().detach().cpu().numpy() > 0.5, 1, 0)\n        self.y_true = np.concatenate((self.y_true, y_true))\n        self.y_pred = np.concatenate((self.y_pred, y_pred))\n        self.score.append(f1_score(y_true, y_pred, average='samples'))\n    @property\n    def avg(self):\n        return np.mean(self.score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    \n    def __init__(self, model, device, config, folder):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_score = 0\n        self.best_loss = 10**5\n        self.best_ap = 0\n        \n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n\n#         self.criterion = FocalLoss(logits=True).to(self.device)\n        self.criterion = nn.BCEWithLogitsLoss().to(self.device)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.train_one_epoch(train_loader)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_loss:\n                self.best_loss = summary_loss.avg\n                self.save_model(f'{self.base_dir}/best-loss-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-loss-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if roc_auc_scores.avg > self.best_score:\n                self.best_score = roc_auc_scores.avg\n                self.save_model(f'{self.base_dir}/best-score-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-score-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if ap_scores.avg > self.best_ap:\n                self.best_ap = ap_scores.avg\n                self.save_model(f'{self.base_dir}/best-ap-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-ap-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                targets = targets.to(self.device).float()\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n                roc_auc_scores.update(targets, outputs)\n                ap_scores.update(targets, outputs)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss, roc_auc_scores, ap_scores\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(train_loader):\n            print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            targets = targets.to(self.device).float()\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n\n            self.optimizer.zero_grad()\n            outputs = self.model(images)\n            loss = self.criterion(outputs, targets)\n            loss.backward()\n            \n            roc_auc_scores.update(targets, outputs)\n            ap_scores.update(targets, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss, roc_auc_scores, ap_scores\n    \n    def save_model(self, path):\n        self.model.eval()\n        torch.save(self.model.state_dict(),path)\n\n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_score': self.best_score,\n            'best_ap': self.best_ap,\n            'best_loss': self.best_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_score = checkpoint['best_score']\n        self.best_ap = checkpoint['best_ap']\n        self.best_loss = checkpoint['best_loss']\n        self.epoch = checkpoint['epoch']\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.species.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.models as models\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.effnet=nn.Sequential(*list(models.resnet34(pretrained=True).children())[:-2])\n        self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self._fc =  nn.Linear(in_features=512, out_features=264, bias=True)\n\n    def forward(self, x):\n        if np.random.rand() > 0.7 and self.training:\n            ids=torch.randperm(x.size()[0])\n            x_ = x[ids,:,:,:].detach()\n            x=x*0.9+x_*0.1\n        xx=self.avg(self.effnet(x)).view(x.size(0),-1)\n        x=self._fc(xx)\n        return x\ndef get_net():\n    net = Model()\n    #net._fc = nn.Linear(in_features=2048, out_features=2, bias=True)\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = get_net()\n#net.load_state_dict(torch.load('/kaggle/input/bird-call/fold0/best-score-checkpoint-021epoch.bin'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 2\n    batch_size = 64\n    n_epochs = 30\n    lr = 0.001#0.5 * 1e-5\n    \n    # -------------------\n    verbose = True\n    verbose_step = 50\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='max',\n        factor=0.95,\n        patience=0,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-7,\n        eps=1e-08\n    )\n    '''\n    SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR\n    scheduler_params = dict(\n       T_max=10, \n    )'''\n    # --------------------\n\n    # -------------------\n    criterion = nn.BCEWithLogitsLoss()\n    # -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.data.sampler import BalanceClassSampler\n\ndef train_fold(fold,size):\n\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        pin_memory=False,\n        shuffle=True,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        \n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        pin_memory=False,\n    )\n\n    fitter = Fitter(model=net, device=torch.device('cuda:0'), config=TrainGlobalConfig, folder=f'fold{fold}')\n    fitter.fit(train_loader, val_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net=net.to(torch.device('cuda'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nfrom datetime import datetime\n\ntrain_fold(fold=0,size=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        pin_memory=False,\n        shuffle=True,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        \n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x,y in train_loader:\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.argmax(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = net(x.to(torch.device('cuda')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(res.argmax(1).cpu()==y.argmax(1)).float().mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}