{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook uses [panns-inference](https://github.com/qiuqiangkong/audioset_tagging_cnn) to predict \"no calls\".\n\npanns-inference has a trained audio segmentation model, and the 111th label is birdcall. \n\nThis notebook is made from [Hidehisa's notebook](https://www.kaggle.com/hidehisaarai1213/inference-pytorch-birdcall-resnet-baseline).\n\nThank you Hidehisa Arai.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\n\nimport IPython\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\nfrom torchvision.models import resnet18, resnet50, densenet121, densenet161\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score\nfrom torchvision import models\nfrom matplotlib import pyplot as plt\n\nfrom torchvision.transforms.functional import to_tensor\nfrom torchvision.transforms import Normalize\n\nimport time\nfrom datetime import timedelta as td\nfrom scipy.ndimage import maximum_filter1d\nimport scipy\n\ndevice = torch.device(\"cuda\")\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set my configuration and load dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    TARGET_SR = 32000\n    MELSPECTROGRAM_PARAMETERS = {\"n_mels\": 128, \"fmin\": 20, \"fmax\": 16000}\n    SEED = 416\n    N_LABEL = 264\n    PRETRAINED = False\n    THRESHOLD = 0.5\n    WEIGHTS_PATH = \"../input/birdcall-densenet161/birdcallnet_f0_densenet161.bin\"\n    SED_THRESHOLD = 0.05\n    \n\n# Get Test Set\nTEST = Path(\"../input/birdsong-recognition/test_audio\").exists()\nif TEST:\n    DATA_DIR = Path(\"../input/birdsong-recognition/\")\nelse:\n    # dataset created by @shonenkov, thanks!\n    DATA_DIR = Path(\"../input/birdcall-check/\")\ntest = pd.read_csv(DATA_DIR / \"test.csv\")\ntest_audio = DATA_DIR / \"test_audio\"\nsub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\nsub.to_csv(\"submission.csv\", index=False)\n\n\n# Get BIRD_CODE dict\ntrain_df = pd.read_csv('../input/birdsong-recognition/train.csv')\nkeys = set(train_df.ebird_code)\nvalues = np.arange(0, len(keys))\ncode_dict = dict(zip(sorted(keys), values))\nn_labels = len(code_dict)\nINV_BIRD_CODE = {v: k for k, v in code_dict.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"My model is Densenet161. It local fold-0 f1 score is 0.685494403 and LB score is 0.471.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BirdcallNet(nn.Module):\n    def __init__(self):\n        super(BirdcallNet, self).__init__()\n        self.densenet = densenet161(pretrained=config.PRETRAINED)\n        self.densenet.classifier = nn.Linear(2208, config.N_LABEL)\n\n    def forward(self, x):\n        return self.densenet(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"install and load panns-inference audio segmentation model.","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!mkdir -p /root/panns_data/\n!cp /kaggle/input/panns-inference/Cnn14_DecisionLevelMax_mAP0.385.pth /root/panns_data/Cnn14_DecisionLevelMax\n!cp /kaggle/input/panns-inference/class_labels_indices.csv /root/panns_data/class_labels_indices.csv\n\n!pip install /kaggle/input/torchlibrosa/torchlibrosa-0.0.4-py3-none-any.whl\n!pip install /kaggle/input/panns-inference/panns_inference-0.0.6-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from panns_inference import SoundEventDetection\n\nsed = SoundEventDetection(device='cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nLet's try audio segmentation and check birdcall.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ebird_code, filename = train_df.sample(1, random_state=123)[[\"ebird_code\", \"filename\"]].values[0]\npath = f\"../input/birdsong-recognition/train_audio/{ebird_code}/{filename}\"\n\nx, sr = librosa.load(path, mono=True, res_type=\"kaiser_fast\")\n\nprint(\"Sampling Rate:\", sr)\nplt.plot(x);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"111th label is birdcall, so the prediction is here.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sed_pred = sed.inference(np.expand_dims(x, 0))\nbirdcall_preds = sed_pred[0,:,111]\nplt.plot(birdcall_preds);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We use thresholds from the audio segmentation results to predict \"no calls\".","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(birdcall_preds>config.SED_THRESHOLD);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check to listen birdcall.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IPython.display.Audio(data=x, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Maybe threshold is more low is better, but now continue.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Predict test data.\n\nWe predict \"no call\" by panns-inference.\n\nIf classification prediction proba is under threshold and panns-inference is over threshold, we use maximum prediction proba.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Dataset Class","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def mono_to_color(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\n\nclass TestDataset(data.Dataset):\n    def __init__(self, df, clip):\n        self.df = df\n        self.clip = clip\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = config.TARGET_SR\n        sample = self.df.loc[idx, :]\n        site = sample.site\n        row_id = sample.row_id\n        \n        if site == \"site_3\":\n            y = self.clip.astype(np.float32)\n            len_y = len(y)\n            start = 0\n            end = SR * 5\n            images = []\n            while len_y > start:\n                y_batch = y[start:end].astype(np.float32)\n                if len(y_batch) != (SR * 5):\n                    break\n\n\n                # predict audio segmentation\n                framewise_output = sed.inference(np.expand_dims(y_batch, 0))\n                _mask = framewise_output[0, :, 111] > config.SED_THRESHOLD\n                                    \n                start = end\n                end = end + SR * 5\n                \n                if sum(_mask) == 0:\n                    continue\n                                \n                melspec = librosa.feature.melspectrogram(y_batch,\n                                                         sr=SR,\n                                                         **config.MELSPECTROGRAM_PARAMETERS)\n                melspec = librosa.power_to_db(melspec).astype(np.float32)\n                image = mono_to_color(melspec)\n                image = to_tensor(image)\n                image = Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n                image = image.numpy()\n                images.append(image)\n            images = np.asarray(images)\n            return images, row_id, site\n        else:\n            end_seconds = int(sample.seconds)\n            start_seconds = int(end_seconds - 5)\n            \n            start_index = SR * start_seconds\n            end_index = SR * end_seconds\n            \n            y = self.clip[start_index:end_index].astype(np.float32)\n            \n            framewise_output = sed.inference(np.expand_dims(y, 0))\n            _mask = framewise_output[0, :, 111] > config.SED_THRESHOLD\n                        \n            if sum(_mask) == 0:\n                image = np.zeros((3, 128, 313))\n                return image, row_id, site\n                \n            melspec = librosa.feature.melspectrogram(y, sr=SR, **config.MELSPECTROGRAM_PARAMETERS)\n            melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n            image = mono_to_color(melspec)\n            image = to_tensor(image)\n            image = Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n            image = image.numpy()\n\n            return image, row_id, site","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Predict Function","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def prediction_for_clip(test_df, clip, model):\n\n    dataset = TestDataset(df=test_df, clip=clip)\n    loader = data.DataLoader(dataset, batch_size=1, shuffle=False)\n    \n    model.eval()\n    prediction_dict = {}\n    for image, row_id, site in progress_bar(loader):\n        site = site[0]\n        row_id = row_id[0]\n        if site in {\"site_1\", \"site_2\"}:\n            image = image.to(device)\n            \n            if image.sum() == 0:\n                labels = []    \n            else:\n                with torch.no_grad():\n                    prediction = model(image)\n                proba = prediction.detach().cpu().sigmoid().numpy().reshape(-1)\n                events = proba >= config.THRESHOLD\n                labels = np.argwhere(events).reshape(-1).tolist()\n                \n                if len(labels) == 0:\n                    labels = [proba.argmax()]\n\n        else:\n            image = image.squeeze(0)\n            batch_size = 16\n            whole_size = image.size(0)\n            if whole_size % batch_size == 0:\n                n_iter = whole_size // batch_size\n            else:\n                n_iter = whole_size // batch_size + 1\n                \n            all_events = set()\n            for batch_i in range(n_iter):\n                batch = image[batch_i * batch_size:(batch_i + 1) * batch_size]\n                if batch.ndim == 3:\n                    batch = batch.unsqueeze(0)\n                \n                batch = batch.to(device)\n                with torch.no_grad():\n                    prediction = model(batch)\n                    proba = prediction.detach().cpu().sigmoid().numpy()\n                    \n                events = proba >= config.THRESHOLD\n                for i in range(len(events)):\n                    event = events[i, :]\n                    labels = np.argwhere(event).reshape(-1).tolist()\n                    for label in labels:\n                        all_events.add(label)\n                        \n            labels = list(all_events)\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict\n\ndef prediction(test_df, test_audio):\n    \n    model = BirdcallNet()\n    model.load_state_dict(torch.load(config.WEIGHTS_PATH))\n    model.to(device)\n    model.eval()\n    \n    unique_audio_id = test_df.audio_id.unique()\n\n\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                               sr=config.TARGET_SR,\n                               mono=True,\n                               res_type=\"kaiser_fast\")\n         \n        test_df_for_audio_id = test_df.query(f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        prediction_dict = prediction_for_clip(test_df_for_audio_id, clip=clip, model=model)\n        \n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n            \n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = prediction(test_df=test, test_audio=test_audio)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If this notebook submit, I got 0.541 LB score.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}