{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install  resnest > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport librosa as lb\nimport pandas as pd\nfrom pathlib import Path\n\nimport torch\nfrom  torch.utils.data import Dataset, DataLoader\n\nfrom tqdm.notebook import tqdm\n\nfrom resnest.torch import resnest50\n\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import TensorBoardLogger\nimport torch.nn.functional as F\nfrom sklearn.metrics import f1_score\n\nimport pickle, copy, re,time, datetime, random, warnings, gc\nimport albumentations\nfrom albumentations.core.transforms_interface import DualTransform, BasicTransform\nfrom albumentations import Compose","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    batch_size= 12\n    weight_decay=1e-8\n    lr=1e-3\n    num_workers = 2\n    EPOCHS = 4\n    NUM_CLASSES = 24\n    SR = 16_000\n    DURATION =  60\n    ROOT= \".\"\n    DEVICE = \"cuda:0\" if torch.cuda.is_available() else 'cpu'\n    DATA_ROOT = Path(\"../input/rfcx-species-audio-detection\")\n    TRAIN_AUDIO_ROOT = Path(\"../input/rfcx-species-audio-detection/train\")\n    TEST_AUDIO_ROOT = Path(\"../input/rfcx-species-audio-detection/test\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelSpecComputer:\n    def __init__(self, sr, n_mels, fmin, fmax):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax\n\n    def __call__(self, y):\n\n        melspec = lb.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax,\n        )\n\n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef normalize(image, mean=None, std=None):\n    image = image / 255.0\n    if mean is not None and std is not None:\n        image = (image - mean) / std\n    return np.moveaxis(image, 2, 0).astype(np.float32)\n\n\ndef crop_or_pad(y, length, sr, is_train=True):\n    if len(y) < length:\n        y = np.concatenate([y, np.zeros(length - len(y))])\n    elif len(y) > length:\n        if not is_train:\n            start = 0\n        else:\n            start = np.random.randint(len(y) - length)\n\n        y = y[start:start + length]\n\n    y = y.astype(np.float32, copy=False)\n\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RFCXDataset(Dataset):\n\n    def __init__(self, data, sr, n_mels=128, fmin=0, fmax=None,  AUDIO_ROOT=\".\",is_train=False,waveform_transforms=None,\n                 num_classes=24, root=None, duration=10):\n\n        self.data = data\n        \n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        \n        self.root =  AUDIO_ROOT\n\n        self.wav_transfos = get_wav_transforms() if waveform_transforms else None\n\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax)\n\n\n    def __len__(self):\n        return len(self.data)\n    \n    def read_index(self, idx, fill_val=1.0, offset=None, use_offset=True):\n        d = self.data.iloc[idx]\n        record, species = d[\"recording_id\"], d[\"species_id\"]\n        try:\n            if use_offset and (self.duration < d[\"duration\"]+1):\n                offset = offset or np.random.uniform(1, int(d[\"duration\"]-self.duration))\n\n            y, _ = lb.load(self.root.joinpath(record).with_suffix(\".flac\").as_posix(),\n                           sr=self.sr, duration=self.duration, offset=offset)\n            \n            if self.wav_transfos is not None:\n                y = self.wav_transfos(y, self.sr)\n            y = crop_or_pad(y, self.audio_length, sr=self.sr)\n            t = np.zeros(self.num_classes)\n            t[species] = fill_val\n        except Exception as e:\n#             print(e)\n            raise ValueError()  from  e\n            y = np.zeros(self.audio_length)\n            t = np.zeros(self.num_classes)\n        \n        return y,t\n            \n        \n\n    def __getitem__(self, idx):\n\n        y, t = self.read_index(idx)\n        \n        \n        melspec = self.mel_spec_computer(y) \n        image = mono_to_color(melspec)\n        image = normalize(image, mean=None, std=None)\n\n        return image, t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_duration(audio_name, root=config.TRAIN_AUDIO_ROOT):\n    return lb.get_duration(filename=root.joinpath(audio_name).with_suffix(\".flac\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.DataFrame({\n    \"recording_id\": [path.stem for path in Path(config.TRAIN_AUDIO_ROOT).glob(\"*.flac\")],\n})\ndata[\"species_id\"] = [[] for _ in range(len(data))]\n\nprint(data.shape)\ndata[\"duration\"] = data[\"recording_id\"].apply(get_duration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[\"duration\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = RFCXDataset(data=data, sr=config.SR, duration=10, AUDIO_ROOT=config.TRAIN_AUDIO_ROOT, is_train=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = ds[0]\nx.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(np.transpose(x,(1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Audio Transform"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass AudioTransform(BasicTransform):\n    \"\"\"Transform for Audio task\"\"\"\n\n    @property\n    def targets(self):\n        return {\"data\": self.apply}\n    \n    def update_params(self, params, **kwargs):\n        if hasattr(self, \"interpolation\"):\n            params[\"interpolation\"] = self.interpolation\n        if hasattr(self, \"fill_value\"):\n            params[\"fill_value\"] = self.fill_value\n        return params\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NoiseInjection(AudioTransform):\n    \"\"\"It simply add some random value into data by using numpy\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(NoiseInjection, self).__init__(always_apply, p)\n    \n    def apply(self, data, noise_levels=(0, 0.05), **params):\n        sound, sr = data\n        noise_level = np.random.uniform(*noise_levels)\n        noise = np.random.randn(len(sound))\n        \n        noise = (noise - np.amin(np.abs(noise)))/ np.amax(np.abs(noise))\n        if np.isnan(noise).any() or not np.isfinite(noise).all():\n            return sound, sr\n        augmented_sound = (sound - np.amin(np.abs(sound)))/ np.amax(np.abs(sound))\n        if np.isnan(augmented_sound).any() or not np.isfinite(augmented_sound).all():\n            return sound, sr\n        augmented_sound = augmented_sound + noise_level * noise\n#         augmented_sound = sound + noise_level * noise\n        # Cast back to same data type\n        augmented_sound = augmented_sound.astype(type(sound[0]))\n\n        return augmented_sound, sr\n\nclass PitchShift(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(PitchShift, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        n_steps = np.random.randint(-10, 10)\n        augmented_sound = librosa.effects.pitch_shift(sound, sr, n_steps)\n\n        return augmented_sound, sr\n\nclass RandomAudio(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self,  seconds=5, always_apply=False, p=0.5):\n        super(RandomAudio, self).__init__(always_apply, p)\n\n        self.seconds = seconds\n    \n    def apply(self, data, **params):\n        sound, sr = data\n        trim_sound = sound\n\n#         shift = np.random.randint(len(sound))\n#         trim_sound = np.roll(sound, shift)\n\n        min_samples = int(sr * self.seconds)\n\n        if len(trim_sound) < min_samples:\n            padding = min_samples - len(trim_sound)\n            offset = padding // 2\n            trim_sound = np.pad(trim_sound, (offset, padding - offset), \"constant\")\n        else:\n            trim_sound = trim_sound[:min_samples]\n\n        return trim_sound, sr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AddGaussianNoise(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(AddGaussianNoise, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n        noise = 0.005*np.random.uniform()*np.amax(sound)\n        augmented_sound = np.array(sound).astype('float64') + noise * np.random.normal(size=sound.shape[0])\n        return augmented_sound, sr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"audio_augs = Compose([\n    AddGaussianNoise( p=0.5)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training process"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torchvision.models import mobilenet_v2\nfrom torch import nn\n\ndevice = torch.device(\"cuda:0\")\n\nclass MobileNetV2(nn.Module):\n    def __init__(self, pretrained, num_classes):\n        super(MobileNetV2, self).__init__()\n        self.model = mobilenet_v2(pretrained=pretrained)\n        self.classifier = nn.Linear(1280, num_classes)\n        \n    def forward(self, x):\n        bs, _, _, _ = x.shape\n        x = self.model.features(x)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)  # x = x.mean(3).mean(2)        \n        x = self.classifier(x)\n        return x\n\ndef get_model(pretrained=True, n_class=24, weight_path=\"\"):\n    model = MobileNetV2(pretrained=pretrained, num_classes=n_class)\n\n    if weight_path != \"\":\n        model.load_state_dict(torch.load(weight_path['state_dict']))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    \n    for i, xi in enumerate(x):\n        index = np.random.randint(0,batch_size)\n        if index == i:\n            return x, y\n        x[i] = lam * xi + (1 - lam) * x[index]\n        y[i] = y[i] + y[index]\n        print(\"y[i] \", y[i])\n    return x,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseNet(LightningModule):   \n    def __init__(self, batch_size=32, lr=5e-4, weight_decay=1e-8, num_workers=0, \n                  AUDIO_ROOT=\".\",  epochs=1, DEVICE=\"cuda:0\", SR=config.SR):\n        super().__init__()\n        self.batch_size = batch_size\n        self.num_workers = num_workers\n        self.DEVICE = DEVICE\n        \n        \n        self.lr = lr\n        self.epochs=epochs\n        \n        self.weight_decay = weight_decay        \n        self.loss_fn = torch.nn.BCEWithLogitsLoss()\n        self.AUDIO_ROOT = AUDIO_ROOT\n        self.data = pd.DataFrame({\n            \"recording_id\": [path.stem for path in Path(AUDIO_ROOT).glob(\"*.flac\")],\n        })\n        self.data[\"species_id\"] = [[] for _ in range(len(self.data))]\n        self.data[\"duration\"] = self.data[\"recording_id\"].apply(get_duration)\n        self.SR= SR\n\n    def train_dataloader(self):\n        \n        ds = RFCXDataset(data=self.data, sr=self.SR, duration=10 , AUDIO_ROOT=self.AUDIO_ROOT)\n        train_loader= DataLoader(ds, batch_size=self.batch_size,\n                                 shuffle=True, num_workers=self.num_workers,\n                                 pin_memory=True)\n        return train_loader\n\n#     def val_dataloader(self):\n#         val_aug = get_valid_augmentation(config.im_size)\n#         valid_datasets = HuBMAPDataset(self.valid_idx, transforms=val_aug, preprocessing=self.preprocessing)\n#         valid_loader = DataLoader(valid_datasets, batch_size=self.batch_size, \n#                           shuffle=False, num_workers=self.num_workers, pin_memory=True)\n#         return valid_loader\n    \n    def configure_optimizers(self):\n        optim = torch.optim.AdamW(self.parameters(),lr=config.lr, betas= (0.9,0.999), \n                                              weight_decay= self.weight_decay, amsgrad=False)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optim,T_max=self.epochs, eta_min=1e-5)\n        self.optimizer = optim\n        self.scheduler = scheduler        \n\n        return [optim], [scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE=\"cuda:0\"\nclass RFCXNet(BaseNet):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        self.net = get_model(pretrained=True, n_class=24)\n    def forward(self, x):\n        return self.net(x)\n    \n    def training_step(self, batch, batch_idx):\n        x, y = [b.to(self.DEVICE) for b in batch]\n        preds = torch.sigmoid(self(x))\n        loss = self.loss_fn( preds.float(), y.float())\n        with torch.no_grad():\n            preds = preds.cpu()>0.5\n            f1 = f1_score(preds, y.cpu(), average='samples')\n        self.log(\"f1_loss\", f1, prog_bar=True)\n        return loss\n    \n#     @torch.no_grad()\n#     def validation_step(self, batch, batch_idx):\n#         x, y =  [b.to(self.DEVICE) for b in batch]\n#         preds = torch.sigmoid(self(x))\n#         valid_loss = self.loss_fn( preds.float(), y.float()) \n#         self.log(\"val_loss\", valid_loss, on_epoch=True, on_step=True)\n#         return valid_loss\n    \n#     def validation_epoch_end(self, outputs) -> None:\n#         torch.stack([x['val_loss'] for x in outputs]).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RFCXNet(batch_size=config.batch_size, \n                lr= config.lr,AUDIO_ROOT=config.TRAIN_AUDIO_ROOT, weight_decay=config.weight_decay, num_workers=config.num_workers, DEVICE = config.DEVICE)\n\ncheckpoint_callback = ModelCheckpoint(\n    filepath=config.ROOT,\n    save_top_k=5,\n    verbose=0,\n    monitor='val_loss',\n    mode='min',\n    prefix='hubmap_',\n)\n\n\nlogger = TensorBoardLogger(\n    save_dir=config.ROOT,\n    name='lightning_logs'\n)\n\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = Trainer(\n    max_epochs=config.EPOCHS,\n    gradient_clip_val=1,\n    logger=logger,\n    checkpoint_callback=checkpoint_callback,\n    limit_val_batches=1,\n    gpus=int(torch.cuda.is_available())\n)\n\ntrainer.fit(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}