{"cells":[{"metadata":{},"cell_type":"markdown","source":"Few days ago, I have shared notebook which denoise using sound envelope.(https://www.kaggle.com/takamichitoda/birdcall-noise-reduction)\n\nBut if submit using this method, timeout error is occured.\n\nIn this notebook, I share that reduce to processing time to submit in time.\n\nThis notebook is made from [Hidehisa's notebook](https://www.kaggle.com/hidehisaarai1213/inference-pytorch-birdcall-resnet-baseline), and and noisereduce library is loaded from [here](https://www.kaggle.com/ajax0564/noisereduce).\nThank you Hidehisa Arai and ankit maurya!","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport IPython\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\nfrom torchvision.models import resnet18, resnet50, densenet121, densenet161\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score\nfrom torchvision import models\nfrom matplotlib import pyplot as plt\n\nfrom torchvision.transforms.functional import to_tensor\nfrom torchvision.transforms import Normalize\n\nimport time\nfrom datetime import timedelta as td\nfrom scipy.ndimage import maximum_filter1d\nimport scipy\n\ndevice = torch.device(\"cuda\")\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set my configuration and load dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    TARGET_SR = 32000\n    MELSPECTROGRAM_PARAMETERS = {\"n_mels\": 128, \"fmin\": 20, \"fmax\": 16000}\n    SEED = 416\n    N_LABEL = 264\n    PRETRAINED = False\n    THRESHOLD = 0.5\n    WEIGHTS_PATH = \"../input/birdcall-densenet161/birdcallnet_f0_densenet161.bin\"\n    SED_THRESHOLD = 0.05\n    ENVELOPE = 0.02\n    \n\n# Get Test Set\nTEST = Path(\"../input/birdsong-recognition/test_audio\").exists()\nif TEST:\n    DATA_DIR = Path(\"../input/birdsong-recognition/\")\nelse:\n    # dataset created by @shonenkov, thanks!\n    DATA_DIR = Path(\"../input/birdcall-check/\")\ntest = pd.read_csv(DATA_DIR / \"test.csv\")\ntest_audio = DATA_DIR / \"test_audio\"\nsub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\nsub.to_csv(\"submission.csv\", index=False)\n\n\n# Get BIRD_CODE dict\ntrain_df = pd.read_csv('../input/birdsong-recognition/train.csv')\nkeys = set(train_df.ebird_code)\nvalues = np.arange(0, len(keys))\ncode_dict = dict(zip(sorted(keys), values))\nn_labels = len(code_dict)\nINV_BIRD_CODE = {v: k for k, v in code_dict.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"My model is Densenet161. It local fold-0 f1 score is 0.685494403 and LB score is 0.471.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BirdcallNet(nn.Module):\n    def __init__(self):\n        super(BirdcallNet, self).__init__()\n        self.densenet = densenet161(pretrained=config.PRETRAINED)\n        self.densenet.classifier = nn.Linear(2208, config.N_LABEL)\n\n    def forward(self, x):\n        return self.densenet(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"install and load noisereduce.","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/noisereduce/noisereduce-1.0.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import noisereduce as nr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nLet's try noise reduce and check birdcall.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ebird_code, filename = train_df.sample(1, random_state=123)[[\"ebird_code\", \"filename\"]].values[0]\npath = f\"../input/birdsong-recognition/train_audio/{ebird_code}/{filename}\"\n\nx, sr = librosa.load(path, mono=True, res_type=\"kaiser_fast\")\n\nprint(\"Sampling Rate:\", sr)\nplt.plot(x);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I detect point no birdcall by using Sound Envelope.\n\nI reffered [this notebook](https://www.kaggle.com/jainarindam/imp-remove-background-dead-noise).\nThank you Arindam!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def envelope(y, rate, threshold):\n    y_mean = maximum_filter1d(np.abs(y), mode=\"constant\", size=rate//20)\n    mask = [mean > threshold for mean in y_mean]\n    return mask","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I use initial 2 second to reduce processing time.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_2s = x[:sr*2]\nplt.plot(initial_2s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = envelope(initial_2s, sr, config.ENVELOPE)  # config.ENVELOPE = 0.02\nplt.plot(mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_denoise = nr.reduce_noise(audio_clip=x, noise_clip=initial_2s[np.logical_not(mask)], verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's compare treated audio","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# original\nIPython.display.Audio(data=x, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# denoise\nIPython.display.Audio(data=x_denoise, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Even if initial few second, It seems effective enough.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Predict test data with denoise.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Dataset Class","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def mono_to_color(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\n\nclass TestDataset(data.Dataset):\n    def __init__(self, df, clip):\n        self.df = df\n        self.clip = clip\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = config.TARGET_SR\n        sample = self.df.loc[idx, :]\n        site = sample.site\n        row_id = sample.row_id\n        \n        if site == \"site_3\":\n            y = self.clip.astype(np.float32)\n            len_y = len(y)\n            start = 0\n            end = SR * 5\n            images = []\n            while len_y > start:\n                y_batch = y[start:end].astype(np.float32)\n                if len(y_batch) != (SR * 5):\n                    break\n\n                # Denoise\n                _y_batch = y_batch[:config.TARGET_SR*2]\n                mask = envelope(_y_batch, config.TARGET_SR, threshold=config.ENVELOPE)\n                noise_clip = _y_batch[np.logical_not(mask)]\n                if len(noise_clip):  # noise is exist\n                    x_denoise = nr.reduce_noise(y_batch, noise_clip)\n                    y_batch = x_denoise\n                      \n                start = end\n                end = end + SR * 5\n \n                melspec = librosa.feature.melspectrogram(y_batch,\n                                                         sr=SR,\n                                                         **config.MELSPECTROGRAM_PARAMETERS)\n                melspec = librosa.power_to_db(melspec).astype(np.float32)\n                image = mono_to_color(melspec)\n                image = to_tensor(image)\n                image = Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n                image = image.numpy()\n                images.append(image)\n            images = np.asarray(images)\n            return images, row_id, site\n        else:\n            end_seconds = int(sample.seconds)\n            start_seconds = int(end_seconds - 5)\n            \n            start_index = SR * start_seconds\n            end_index = SR * end_seconds\n            \n            y = self.clip[start_index:end_index].astype(np.float32)\n            \n            # denoise\n            _y = y[:config.TARGET_SR*2]\n            mask = envelope(_y, config.TARGET_SR, threshold=config.ENVELOPE)\n            noise_clip = _y[np.logical_not(mask)]\n            if len(noise_clip):  # noise is exist\n                x_denoise = nr.reduce_noise(y, noise_clip)\n                y = x_denoise\n                \n            melspec = librosa.feature.melspectrogram(y, sr=SR, **config.MELSPECTROGRAM_PARAMETERS)\n            melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n            image = mono_to_color(melspec)\n            image = to_tensor(image)\n            image = Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n            image = image.numpy()\n\n            return image, row_id, site","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Predict Function","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def prediction_for_clip(test_df, clip, model):\n\n    dataset = TestDataset(df=test_df, clip=clip)\n    loader = data.DataLoader(dataset, batch_size=1, shuffle=False)\n    \n    model.eval()\n    prediction_dict = {}\n    for image, row_id, site in progress_bar(loader):\n        site = site[0]\n        row_id = row_id[0]\n        if site in {\"site_1\", \"site_2\"}:\n            image = image.to(device)\n            \n            if image.sum() == 0:\n                labels = []    \n            else:\n                with torch.no_grad():\n                    prediction = model(image)\n                proba = prediction.detach().cpu().sigmoid().numpy().reshape(-1)\n                events = proba >= config.THRESHOLD\n                labels = np.argwhere(events).reshape(-1).tolist()\n                \n        else:\n            image = image.squeeze(0)\n            batch_size = 16\n            whole_size = image.size(0)\n            if whole_size % batch_size == 0:\n                n_iter = whole_size // batch_size\n            else:\n                n_iter = whole_size // batch_size + 1\n                \n            all_events = set()\n            for batch_i in range(n_iter):\n                batch = image[batch_i * batch_size:(batch_i + 1) * batch_size]\n                if batch.ndim == 3:\n                    batch = batch.unsqueeze(0)\n                \n                batch = batch.to(device)\n                with torch.no_grad():\n                    prediction = model(batch)\n                    proba = prediction.detach().cpu().sigmoid().numpy()\n                    \n                events = proba >= config.THRESHOLD\n                for i in range(len(events)):\n                    event = events[i, :]\n                    labels = np.argwhere(event).reshape(-1).tolist()\n                    for label in labels:\n                        all_events.add(label)\n                        \n            labels = list(all_events)\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict\n\ndef prediction(test_df, test_audio):\n    \n    model = BirdcallNet()\n    model.load_state_dict(torch.load(config.WEIGHTS_PATH))\n    model.to(device)\n    model.eval()\n    \n    unique_audio_id = test_df.audio_id.unique()\n\n\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                               sr=config.TARGET_SR,\n                               mono=True,\n                               res_type=\"kaiser_fast\")\n         \n        test_df_for_audio_id = test_df.query(f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        prediction_dict = prediction_for_clip(test_df_for_audio_id, clip=clip, model=model)\n        \n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n            \n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = prediction(test_df=test, test_audio=test_audio)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This result can submit in time. \n\nBut LB score may be not good.\n\nThe one of reason is that the model was trained without denoise processing. We should apply same processing in train time.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}