{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Fastai.audio"},{"metadata":{},"cell_type":"markdown","source":"This notebook performs multilabel classification with fastai, with on-the-fly audio to image conversion. The data augmentation can thus be applied on the original signal, rather than on the images."},{"metadata":{},"cell_type":"markdown","source":"Most of the code is an adaptation from this link: https://github.com/cccwam/fastai_audio"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport fastai\nfrom fastai.imports import *\nfrom fastai.basics import *\nfrom fastai.metrics import accuracy\nfrom fastai.torch_core import *\nfrom fastai.vision import *\nfastai.__version__\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport IPython\nimport IPython.display\nimport PIL\nimport pickle\nimport time\nstart_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Audio Clip"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from fastai.torch_core import *\nimport librosa\nfrom scipy.io import wavfile\nfrom IPython.display import display, Audio\n\nimport importlib\nsoundfile_spec = importlib.util.find_spec(\"soundfile\")\nif soundfile_spec is not None:\n    import soundfile as sf\n\nclass AudioClip(ItemBase):\n    def __init__(self, signal, sample_rate, fn):\n        self.data = signal # Contains original signal to start \n        self.original_signal = signal.clone()\n        self.processed_signal = signal.clone()\n        self.sample_rate = sample_rate\n        self.fn = fn\n\n    def __str__(self):\n        return '(duration={}s, sample_rate={:.1f}KHz)'.format(\n            self.duration, self.sample_rate/1000)\n\n    def clone(self):\n        return self.__class__(self.data.clone(), self.sample_rate, self.fn)\n\n    def apply_tfms(self, tfms, **kwargs):\n        for tfm in tfms:\n            self.data = tfm(self.data, **kwargs)\n            if issubclass(type(tfm), MyDataAugmentation):\n                self.processed_signal = self.data.clone().cpu()\n        return self\n    \n    @property\n    def num_samples(self):\n        return len(self.data)\n\n    @property\n    def duration(self):\n        return self.num_samples / self.sample_rate\n\n    def show(self, ax=None, figsize=(5, 1), player=True, title=None, **kwargs):\n        if ax is None:\n            _, ax = plt.subplots(figsize=figsize)\n        if title:\n            ax.set_title(\"Class: \" + str(title) + \" \\nfilename: \" + str(self.fn))\n        \n        timesteps = np.arange(self.original_signal.shape[1]) / self.sample_rate\n        \n        ax.plot(timesteps, self.original_signal[0]) \n        if self.original_signal.size(0) > 1: # Check if mono or stereo\n            ax.plot(timesteps, self.original_signal[1]) \n        ax.set_xlabel('Original Signal Time (s)')\n        plt.show()\n        \n        timesteps = np.arange(self.processed_signal.shape[1]) / self.sample_rate\n\n        _, ax = plt.subplots(figsize=figsize)\n        if title:\n            ax.set_title(\"Class: \" + str(title) + \" \\nfilename: \" + str(self.fn))\n        ax.plot(timesteps, self.processed_signal[0]) \n        if self.processed_signal.size(0) > 1: # Check if mono or stereo\n            ax.plot(timesteps, self.processed_signal[1]) \n        ax.set_xlabel('Processed Signal Time (s)')\n        plt.show()\n        \n        if player:\n            # unable to display an IPython 'Audio' player in plt axes\n            display(\"Original signal\")\n            display(Audio(self.original_signal, rate=self.sample_rate))\n            display(\"Processed signal\")\n            display(Audio(self.processed_signal, rate=self.sample_rate))\n\ndef open_audio(fn, using_librosa:bool=False, downsampling=8000):\n    if using_librosa: \n        x, sr = librosa.core.load(fn, sr=None, mono=False)\n        \n    else:\n        if soundfile_spec is not None:\n            x, sr = sf.read(fn, always_2d=True, dtype=\"float32\")\n        else:\n            raise Exception(\"Cannot load soundfile\")\n            #sr, x = wavfile.read(fn) # 10 times faster than librosa but issues with 24bits wave\n    \n    if len(x.shape) == 1: # Mono signal\n        x = x.reshape(1, -1) # Add 1 channel\n    else:\n        if not using_librosa:\n            x = np.swapaxes(x, 1, 0) # Scipy result is timestep * channels instead of channels * timestep\n    \n    if downsampling is not None:\n        x = librosa.core.resample(x, sr, downsampling)\n        sr = downsampling\n    t = torch.from_numpy(x.astype(np.float32, copy=False))\n    if x.dtype == np.int16:\n        t.div_(32767)\n    elif x.dtype != np.float32:\n        raise OSError('Encountered unexpected dtype: {}'.format(x.dtype))\n    return AudioClip(t, sr, fn)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Audio Data Bunch"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.basic_data import *\nfrom fastai.data_block import *\nfrom fastai.data_block import _maybe_squeeze\nfrom fastai.text import SortSampler, SortishSampler\nfrom fastai.vision import *\nfrom fastai.torch_core import *\nimport gc\nimport copy\n\n\nclass AudioDataBunch(DataBunch):\n    \n    # Subclass because of bug to give dl_tfms to underlying dataloader\n    @classmethod\n    def create(cls, train_ds, valid_ds, \n               tfms:Optional[Collection[Callable]]=None, # There is a bug in LabelLists because dl_tfms is not given to dataloader\n               **kwargs)->'AudioDataBunch':\n        db = super().create(train_ds=train_ds, valid_ds=valid_ds, dl_tfms=tfms, **kwargs)\n\n        return db\n\n\n\n    def show_batch(self, rows:int=5, ds_type:DatasetType=DatasetType.Train, **kwargs):\n        dl = self.dl(ds_type)\n        ds = dl.dl.dataset\n\n        idx = np.random.choice(len(ds), size=rows, replace=False)\n        batch = ds[idx]\n        \n        max_count = min(rows, len(batch))\n        xs, ys, xs_processed, ys_processed = [], [], [], []\n        for i in range(max_count):\n            x, x_processed, y, y_processed = batch[i][0], batch[i][0].data, batch[i][1], torch.tensor(batch[i][1].data)\n            xs.append(x)\n            xs_processed.append(x_processed)\n            ys.append(y)\n            ys_processed.append(y_processed)\n\n        xs_processed = torch.stack(xs_processed, dim=0)\n        ys_processed = torch.stack(ys_processed, dim=0)\n        \n        for tfm in dl.tfms:\n            xs_processed, ys_processed = tfm((xs_processed, ys_processed))\n\n        \n        self.train_ds.show_xys(xs, ys, xs_processed=xs_processed.unbind(dim=0), **kwargs)\n        del xs, ys, xs_processed, ys_processed\n\n    # Inspired by ImageDataBunch\n    def batch_stats(self, funcs:Collection[Callable]=None, ds_type:DatasetType=DatasetType.Train)->Tensor:\n        \"Grab a batch of data and call reduction function `func` per channel\"\n        funcs = ifnone(funcs, [torch.mean,torch.std])\n        x = self.one_batch(ds_type=ds_type, denorm=False)[0].cpu()\n        return [func(channel_view(x), 1) for func in funcs]\n        \n    # Inspired by ImageDataBunch\n    def normalize(self, stats:Collection[Tensor]=None, do_x:bool=True, do_y:bool=False)->None:\n        \"Add normalize transform using `stats` (defaults to `DataBunch.batch_stats`)\"\n        if getattr(self,'norm',False): raise Exception('Can not call normalize twice')\n        if stats is None: self.stats = self.batch_stats()\n        else:             self.stats = stats\n        self.norm,self.denorm = normalize_funcs(*self.stats, do_x=do_x, do_y=do_y)\n        self.add_tfm(self.norm)\n        return self\n\n       \n\n# Inspired by https://docs.fast.ai/tutorial.itemlist.html\nclass AudioItemList(ItemList):\n    _bunch = AudioDataBunch # Needed to include normalize\n    \n    def __init__(self, items:Iterator,\n                 using_librosa=False, downsampling=None, **kwargs):\n        super().__init__(items=items, **kwargs)\n        self.using_librosa = using_librosa\n        self.copy_new.append('using_librosa')\n        self.downsampling = downsampling\n        self.copy_new.append('downsampling')\n\n    def get(self, i):\n        fn = super().get(i)\n        return open_audio(self.path/fn, using_librosa=self.using_librosa, downsampling=self.downsampling)\n\n\n    @classmethod\n    def from_df(cls, df, path, using_librosa=False, folder:PathOrStr=None, downsampling=None, **kwargs):\n        #if folder is not None: path2 = str(path)+folder\n        res = super().from_df(df, path=path, **kwargs)\n        pref = f'{res.path}{os.path.sep}'\n        if folder is not None: pref += f'{folder}{os.path.sep}'\n        res.items = np.char.add(pref, res.items.astype(str))\n        res.using_librosa=using_librosa\n        res.downsampling = downsampling\n        return res\n    \n    \n    def reconstruct(self, t:Tensor, x:Tensor = None): \n        raise Exception(\"Not implemented yet\")\n        # From torch\n        #return ImagePoints(FlowField(x.size, t), scale=False)\n\n    \n    \n    def show_xys(self, xs, ys, xs_processed=None, figsize=None, **kwargs):\n        if xs_processed is None:\n            for x, y in zip(xs, ys):\n                x.show(title=str(y), figsize=figsize, **kwargs)\n        else:\n            for x, y, x_processed in zip(xs, ys, xs_processed):\n                x.show(title=str(y), figsize=figsize, **kwargs)\n                for channel in range(x_processed.size(0)):\n                    Image(x_processed[channel, :, :].unsqueeze(0)).show(figsize=figsize)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Audio Learner"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.torch_core import *\nfrom fastai.train import Learner\n\nfrom fastai.callbacks.hooks import num_features_model, hook_output\nfrom fastai.vision import create_body, create_head, Image\nfrom fastai.vision.learner import cnn_config, _resnet_split, ClassificationInterpretation\n\n\n\n\n# copied from fastai.vision.learner, omitting unused args,\n# and adding channel summing of first convolutional layer\ndef create_cnn(data, arch, pretrained=False, is_mono_input=True, **kwargs):\n    meta = cnn_config(arch)\n    body = create_body(arch, pretrained)\n\n    # sum up the weights of in_channels axis, to reduce to single input channel\n    # Suggestion by David Gutman\n    # https://forums.fast.ai/t/black-and-white-images-on-vgg16/2479/2\n    if is_mono_input:\n        first_conv_layer = body[0][0]\n        first_conv_weights = first_conv_layer.state_dict()['weight']\n        assert first_conv_weights.size(1) == 3 # RGB channels dim\n        summed_weights = torch.sum(first_conv_weights, dim=1, keepdim=True)\n        first_conv_layer.weight.data = summed_weights\n        first_conv_layer.in_channels = 1\n    else:\n        # In this case, the input is a stereo\n        first_conv_layer = body[0]\n        first_conv_weights = first_conv_layer.state_dict()['weight']\n        assert first_conv_weights.size(1) == 3 # RGB channels dim\n        summed_weights = torch.sum(first_conv_weights, dim=1, keepdim=True)\n        first_conv_layer.weight.data = first_conv_weights[:, :2, :, :] # Keep only 2 channels for the weights\n        first_conv_layer.in_channels = 2\n\n    nf = num_features_model(body) * 2\n    head = create_head(nf, data.c, None, 0.5)\n    model = nn.Sequential(body, head)\n    learn = Learner(data, model, **kwargs)\n    learn.split(meta['split'])\n    if pretrained:\n        learn.freeze()\n    apply_init(model[1], nn.init.kaiming_normal_)\n    return learn\n\n\n\ndef my_cl_int_plot_top_losses(self, k, largest=True, figsize=(25,7), heatmap:bool=True, heatmap_thresh:int=16,\n                            return_fig:bool=None)->Optional[plt.Figure]:\n    \"Show images in `top_losses` along with their prediction, actual, loss, and probability of actual class.\"\n    tl_val,tl_idx = self.top_losses(k, largest)\n    classes = self.data.classes\n    cols = math.ceil(math.sqrt(k))\n    rows = math.ceil(k/cols)\n    fig,axes = plt.subplots(rows, cols, figsize=figsize)\n    fig.suptitle('prediction/actual/loss/probability', weight='bold', size=14)\n    for i,idx in enumerate(tl_idx):\n        audio, cl = self.data.dl(self.ds_type).dataset[idx]\n        audio = audio.clone()\n        \n        m = self.learn.model.eval()\n        \n        x, _ = self.data.one_item(audio) # Process one audio into prediction\n        \n        x_consolidated = x.sum(dim=1, keepdim=True) # Sum accross all channels to ease the interpretation\n\n        im = Image(x_consolidated[0, :, :, :].cpu()) # Extract the processed image from the prediction (after dl_tfms) and keep it into CPU\n        cl = int(cl)\n        title = f'{classes[self.pred_class[idx]]}/{classes[cl]} / {self.losses[idx]:.2f} / {self.probs[idx][cl]:.2f}'\n        title = title + f'\\n {audio.fn}'\n        \n        im.show(ax=axes.flat[i], title=title)\n        \n        if heatmap:\n            # Related paper http://openaccess.thecvf.com/content_ICCV_2017/papers/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.pdf\n            with hook_output(m[0]) as hook_a: # hook activations from CNN module\n                with hook_output(m[0], grad= True) as hook_g: # hook gradients from CNN module\n                    preds = m(x) # Forward pass to get activations\n                    preds[0,cl].backward() # Backward pass to get gradients\n            acts = hook_a.stored[0].cpu()\n            if (acts.shape[-1]*acts.shape[-2]) >= heatmap_thresh:\n                grad = hook_g.stored[0][0].cpu() # Hook the gradients from the CNN module and extract the first one (because one item only)\n                grad_chan = grad.mean(1).mean(1) # Mean accross image to keep mean gradients per channel \n                mult = F.relu(((acts*grad_chan[...,None,None])).sum(0)) # Multiply activation with gradients (add 1 dim for height and width)\n                sz = list(im.shape[-2:])\n                axes.flat[i].imshow(mult, alpha=0.35, extent=(0,*sz[::-1],0), interpolation='bilinear', cmap='magma')     \n        \n    if ifnone(return_fig, defaults.return_fig): return fig\n    \n    \nClassificationInterpretation.plot_audio_top_losses = my_cl_int_plot_top_losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mapk_np(preds, targs, k=3):\n    preds = np.argsort(-preds, axis=1)[:, :k]\n    score = 0.\n    for i in range(k):\n        num_hits = (preds[:, i] == targs).sum()\n        score += num_hits * (1. / (i+1.))\n    score /= preds.shape[0]\n    return score\n\n\ndef mapk(preds, targs, k=3):\n    return tensor(mapk_np(to_np(preds), to_np(targs), k))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Audio Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa as lr\nfrom fastai.torch_core import *\nimport gc\n\n\ndef get_data_augmentation_transforms(max_seconds=30, start_at_second=0,\n                                     sample_rate=44100, noise_scl=None, convert_to_mono=True):\n    tfms = []\n    if convert_to_mono:\n        tfms.append(ConvertToMono())\n    max_channels = 1 if convert_to_mono else 2\n    tfms.append(PadToMax(start_at_second=start_at_second, max_seconds=max_seconds, \n                         sample_rate=sample_rate, max_channels=max_channels))\n    \n    if noise_scl is not None:\n        tfms.append(WhiteNoise(noise_scl))\n    return tfms\n\ndef get_frequency_transforms(n_fft=512, n_hop=160, top_db=80,\n                             n_mels=None, f_min=0, f_max=None, sample_rate=44100):\n#    tfms.append(MFCC(n_fft=n_fft, n_mfcc=n_mels, hop_length=n_hop, sample_rate=sample_rate, f_min=f_min, f_max=f_max))\n    tfms = [Spectrogram(n_fft=n_fft, n_hop=n_hop)]\n    tfms.append(FrequencyToMel(n_fft=n_fft, n_mels=n_mels, sr=sample_rate, f_min=f_min, f_max=f_max))\n    tfms.append(ToDecibels(top_db=top_db))\n    \n    return tfms\n\n\ndef get_frequency_batch_transforms(*args, **kwargs):\n    tfms = get_frequency_transforms(*args, **kwargs)\n\n    def _freq_batch_transformer(inputs):\n        xs, ys = inputs\n        for tfm in tfms:\n            xs = tfm(xs)\n        del inputs\n        \n        return xs, ys.detach()\n    return [_freq_batch_transformer]\n\n# Parent classes used to distinguish transforms for data augmentation and transforms to convert audio into image\nclass MyDataAugmentation:\n    pass\n\nclass MySoundToImage:\n    pass\n\n### The below transformers are on the single AudioClip (to help to keep tracks of changes from data augmentation)\n\nclass ConvertToMono(MyDataAugmentation):\n    def __init__(self):\n        pass\n\n    def __call__(self, X):\n        assert(X.dim() == 2) # channels * timestep\n        X = X.sum(0) # Sum over channels\n        X = X.unsqueeze(0)\n        assert(X.dim() == 2) # channels * timestep\n        return X\n\n    \n    \nclass PadToMax(MyDataAugmentation):\n    def __init__(self, start_at_second=0, max_seconds=30, sample_rate=16000, max_channels=1):\n        self.max_seconds = max_seconds\n        self.sample_rate = sample_rate\n        self.max_channels = max_channels\n        self.start_at_second = start_at_second\n        \n\n    def __call__(self, X):\n        # X must be channels * timestep\n        assert(X.dim() == 2)\n        assert(X.size(0) <= 2) # There is only 2 channels at maximum \n        \n        mx = int(self.max_seconds * self.sample_rate)\n        start_at = min(int(self.start_at_second * self.sample_rate), X.size(1))\n        if X.size(1) - start_at <= mx:\n            start_at = max(X.size(1) - mx, 0)\n        \n        if (X.size(1) < mx): \n            X = torch.cat((X, torch.zeros([X.size(0), mx - X.size(1)], device=X.device)), dim=1) # Channels * Timestep\n        if (X.size(1) > mx): \n            X = X[:, start_at:(mx + start_at)]\n        if X.size(0) < self.max_channels:\n            targets = torch.zeros(self.max_channels, X.size(1), device=X.device)\n            targets[:X.size(0), :] = X\n            X = targets\n        \n        return X\n\n    \nclass WhiteNoise(MyDataAugmentation):\n    def __init__(self, noise_scl=0.0005):\n        self.noise_scl= noise_scl\n\n    def __call__(self, X):\n        noise = torch.randn(X.shape, device=X.device) * self.noise_scl \n        assert(X.dim() == 2) # channels * timestep\n        return X + noise\n\n    \n### The below transformers are on the whole batch\n\n    \nclass MFCCLibrosa(MySoundToImage):\n    def __init__(self, sample_rate=16000, n_mfcc=20, n_fft=512, hop_length=512, f_min=0, f_max=None):\n        self.n_mfcc = n_mfcc\n        self.sample_rate = sample_rate\n        self.n_fft = n_fft\n        self.hop_length = hop_length\n        self.f_min=f_min\n        self.f_max=f_max\n    \n    def __call__(self, X):\n        mfcc = torch.zeros([X.size(0), self.n_mfcc, 1+int(X.size(1) / self.hop_length)], device=X.device)\n        for i in range(X.size(0)):\n            single_mfcc = lr.feature.mfcc(y=X[0, :].cpu().numpy(), \n                                   sr=self.sample_rate, n_mfcc=self.n_mfcc, n_fft=self.n_fft, hop_length=self.hop_length,\n                                         fmin=self.f_min, fmax=self.f_max)\n            mfcc[i, :, :] = torch.tensor(single_mfcc, device=X.device)\n        del X\n        return mfcc\n    \n# Returns power spectrogram (magnitude squared)\nclass Spectrogram(MySoundToImage):\n    def __init__(self, n_fft=1024, n_hop=256, window=torch.hann_window,\n                 device=None):\n        self.n_fft = n_fft\n        self.n_hop = n_hop\n        self.window = window(n_fft)\n\n    def __call__(self, x):\n        X_left = torch.stft(x[:, 0, :],\n                       n_fft=self.n_fft,\n                       hop_length=self.n_hop,\n                       win_length=self.n_fft,\n                       window=to_device(self.window, x.device),\n                       onesided=True,\n                       center=True,\n                       pad_mode='constant',\n                       normalized=True)\n        # compute power from real and imag parts (magnitude^2)\n        X_left.pow_(2.0)\n        X_left = X_left[:,:,:,0] + X_left[:,:,:,1]\n        X_left = X_left.unsqueeze(1) # Add channel dimension\n\n        if (x.size(1) > 1):\n            X_right = torch.stft(x[:, 1, :],\n                           n_fft=self.n_fft,\n                           hop_length=self.n_hop,\n                           win_length=self.n_fft,\n                           window=to_device(self.window, x.device),\n                           onesided=True,\n                           center=True,\n                           pad_mode='constant',\n                           normalized=True)        \n            # compute power from real and imag parts (magnitude^2)\n            X_right.pow_(2.0)\n            X_right = X_right[:,:,:,0] + X_right[:,:,:,1]\n            X_right = X_right.unsqueeze(1) # Add channel dimension\n            res = torch.cat([X_left, X_right], dim=1) \n            assert(res.dim() == 4) # Check dim (n sample * channels * h * w)\n            return res\n            \n        else:\n            assert(X_left.dim() == 4) # Check dim (n sample * channels * h * w)\n            return X_left # Return only mono channel\n        \n    \nclass FrequencyToMel(MySoundToImage):\n    def __init__(self, n_mels=40, n_fft=1024, sr=16000,\n                 f_min=0.0, f_max=None, device=None):\n        self.mel_fb = lr.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,\n                                fmin=f_min, fmax=f_max).astype(np.float32)\n\n    def __call__(self, spec_f):\n        spec_m = to_device(torch.from_numpy(self.mel_fb), spec_f.device) @ spec_f\n        assert(spec_m.dim() == 4) # Check dim (n sample * channels * h * w)\n        return spec_m\n\n\nclass ToDecibels(MySoundToImage):\n    def __init__(self,\n                 power=2, # magnitude=1, power=2\n                 ref=1.0,\n                 top_db=None,\n                 normalized=True,\n                 amin=1e-7):\n        self.constant = 10.0 if power == 2 else 20.0\n        self.ref = ref\n        self.top_db = abs(top_db) if top_db else top_db\n        self.normalized = normalized\n        self.amin = amin\n\n    def __call__(self, x):\n        batch_size = x.shape[0]\n        if self.ref == 'max':\n            ref_value = x.contiguous().view(batch_size, -1).max(dim=-1)[0]\n            ref_value.unsqueeze_(1).unsqueeze_(1)\n        else:\n            ref_value = tensor(self.ref)\n        spec_db = x.clamp_min(self.amin).log10_().mul_(self.constant)\n        spec_db.sub_(ref_value.clamp_min_(self.amin).log10_().mul_(10.0))\n        if self.top_db is not None:\n            max_spec = spec_db.view(batch_size, -1).max(dim=-1)[0]\n            max_spec.unsqueeze_(1).unsqueeze_(1).unsqueeze_(1)\n            spec_db = torch.max(spec_db, max_spec - self.top_db)\n            if self.normalized:\n                # normalize to [0, 1]\n                spec_db.add_(self.top_db).div_(self.top_db)\n        assert(spec_db.dim() == 4) # Check dim (n sample * channels * h * w)\n        return spec_db","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TTA"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"Brings TTA (Test Time Functionality) to the `Learner` class. Use `learner.TTA()` instead\"\nfrom fastai.torch_core import *\nfrom fastai.basic_train import *\nfrom fastai.basic_train import _loss_func2activ\nfrom fastai.basic_data import DatasetType\n\n__all__ = []\n\n\ndef _tta_only(learn:Learner, ds_type:DatasetType=DatasetType.Valid) -> Iterator[List[Tensor]]:\n    \"Computes the outputs for several augmented inputs for TTA\"\n    dl = learn.dl(ds_type)\n    ds = dl.dataset\n    old = ds.tfms\n    augm_tfm = [o for o in learn.data.train_ds.tfms]\n    try:\n        pbar = master_bar(range(8))\n        for i in pbar:\n            ds.tfms = augm_tfm\n            yield get_preds(learn.model, dl, pbar=pbar, activ=_loss_func2activ(learn.loss_func))[0]\n    finally: ds.tfms = old\n\n\nLearner.tta_only = _tta_only\n\n\ndef _TTA(learn:Learner, beta:float=0.4, ds_type:DatasetType=DatasetType.Valid, with_loss:bool=False) -> Tensors:\n    \"Applies TTA to predict on `ds_type` dataset.\"\n    preds,y = learn.get_preds(ds_type)\n    all_preds = list(learn.tta_only(ds_type=ds_type))\n    avg_preds = torch.stack(all_preds).mean(0)\n    if beta is None: return preds,avg_preds,y\n    else:\n        final_preds = preds*beta + avg_preds*(1-beta)\n        if with_loss:\n            return final_preds, y, calc_loss(final_preds, y, learn.loss_func)\n        return final_preds, y\n\n\nLearner.TTA = _TTA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA = Path('../input')\nCSV_TRN_CURATED = DATA/'train_curated.csv'\nCSV_TRN_NOISY = DATA/'train_noisy.csv'\nCSV_SUBMISSION = DATA/'sample_submission.csv'\nTRN_CURATED = DATA/'train_curated'\nTRN_NOISY = DATA/'train_noisy'\nTEST = DATA/'test'\n\nWORK = Path('work')\nIMG_TRN_CURATED = WORK/'image/trn_curated'\nIMG_TRN_NOISY = WORK/'image/trn_curated'\nIMG_TEST = WORK/'image/test'\nfor folder in [WORK, IMG_TRN_CURATED, IMG_TRN_NOISY, IMG_TEST]: \n    Path(folder).mkdir(exist_ok=True, parents=True)\n\ndf = pd.read_csv(CSV_TRN_CURATED)\ndf_n = pd.read_csv(CSV_TRN_NOISY)\ntest_df = pd.read_csv(CSV_SUBMISSION)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_fft = 512 # output of fft will have shape [513 x n_frames]\nn_hop = 94  # width of Spectogram = max_seconds * sample rate / n_hop\nn_mels = 128 # Height of spectogram\nsample_rate = 48127\nmax_seconds = 2\nf_min=0\nf_max=8000\nnoise_scl=0.005\n\n\ntrain_tfms = get_data_augmentation_transforms(sample_rate=sample_rate, max_seconds=max_seconds, \n                                              noise_scl=noise_scl)\nvalid_tfms = get_data_augmentation_transforms(sample_rate=sample_rate, max_seconds=max_seconds)\n\ndl_tfms = get_frequency_batch_transforms(n_fft=n_fft, n_hop=n_hop,\n                                            n_mels=n_mels, \n                                            f_min=f_min, f_max=f_max,\n                                            sample_rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\n\naudios = (AudioItemList.from_df(df=df, path=DATA, folder='/train_curated', using_librosa=True)\n          .split_by_rand_pct(0.1)\n          .label_from_df(label_delim=',')\n          .add_test_folder('test')\n          .transform(tfms=(train_tfms, valid_tfms))\n          .databunch(bs=batch_size, tfms=dl_tfms)\n         ).normalize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"audios.c, len(audios.train_ds), len(audios.valid_ds), len(audios.test_ds)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xs, ys = audios.one_batch()\nprint(xs.shape, ys.shape)\ndel xs, ys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"audios.show_batch(3, ds_type=DatasetType.Train, figsize=(10, 5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f_score = partial(fbeta, thresh=0.2)\nacc_02 = partial(accuracy_thresh, thresh=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = create_cnn(audios, models.xresnet50, pretrained=False, metrics=[f_score, acc_02], model_dir='../../')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(3, 3e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, _ = learn.TTA(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, v in enumerate(learn.data.classes):\n    test_df[v] = preds[:, i]\n\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Kernel run time = {(time.time()-start_time)/3600} hours\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}