{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8096026,"sourceType":"datasetVersion","datasetId":4780143}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Rating indicates the quality level of the audio (specifically for bird sounds), \nbut I found that the labels are not necessarily accurate, \nfor example, some samples with rating = 0 actually sound quite good.\n\nIn this notebook:\n\n- I developed a model to regress the Rating, and the Mean Squared Error (MSE) for validation is 1.01. \n\n- This way, we can observe the difference between the model's prediction and the actual labels.\n\n- At the same time,we can make predictions on the unlabeled data, select relatively high-quality data, and use them for subsequent tasks.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import glob\nimport os\nimport warnings\nimport time\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport torch\nimport pickle\nfrom torch.utils.data import Dataset\nimport soundfile as sf\nfrom scipy import signal\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nimport librosa\nimport librosa.display\nfrom matplotlib.colorbar import Colorbar\nimport IPython.display as ipd\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:18.912200Z","iopub.execute_input":"2024-04-12T02:51:18.913106Z","iopub.status.idle":"2024-04-12T02:51:21.689511Z","shell.execute_reply.started":"2024-04-12T02:51:18.913059Z","shell.execute_reply":"2024-04-12T02:51:21.688715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_kaggle = True\ninfer_all_on_kaggle = False\nif is_kaggle:\n    data_root = '/kaggle/input/birdclef-2024/'\nelse:\n    data_root = '/home/hw/m2_disk/kaggle/BirdCLEF2024_lhw/data/birdclef-2024/'","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:21.690944Z","iopub.execute_input":"2024-04-12T02:51:21.691503Z","iopub.status.idle":"2024-04-12T02:51:21.696362Z","shell.execute_reply.started":"2024-04-12T02:51:21.691476Z","shell.execute_reply":"2024-04-12T02:51:21.695361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f'{data_root}train_metadata.csv')\ndf['filename'] = f'{data_root}/train_audio/' + df['filename']","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:21.697356Z","iopub.execute_input":"2024-04-12T02:51:21.697595Z","iopub.status.idle":"2024-04-12T02:51:21.831982Z","shell.execute_reply.started":"2024-04-12T02:51:21.697574Z","shell.execute_reply":"2024-04-12T02:51:21.831172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(df['rating'])\nplt.title('rating distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:21.834667Z","iopub.execute_input":"2024-04-12T02:51:21.835288Z","iopub.status.idle":"2024-04-12T02:51:22.218284Z","shell.execute_reply.started":"2024-04-12T02:51:21.835252Z","shell.execute_reply":"2024-04-12T02:51:22.217389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"low_quality_sample = df[df['rating']<=0]\nhigh_quality_sample = df[df['rating']>=5]","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:22.219374Z","iopub.execute_input":"2024-04-12T02:51:22.219632Z","iopub.status.idle":"2024-04-12T02:51:22.229005Z","shell.execute_reply.started":"2024-04-12T02:51:22.219609Z","shell.execute_reply":"2024-04-12T02:51:22.228086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_audio(filename, audio_data=None, sample_rate=None):\n    if audio_data is None:\n        audio_data, sample_rate = librosa.load(filename)\n\n    ipd.display(ipd.Audio(audio_data, rate=sample_rate))\n\n    fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(16, 12))\n\n    axs[0].plot(audio_data)\n    axs[0].set_title('Waveform')\n\n    spectrogram = librosa.amplitude_to_db(librosa.stft(audio_data), ref=np.max)\n    librosa.display.specshow(spectrogram, sr=sample_rate, x_axis='time', y_axis='hz', ax=axs[1])\n    axs[1].set_title('Spectrogram')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-12T02:51:22.230458Z","iopub.execute_input":"2024-04-12T02:51:22.230846Z","iopub.status.idle":"2024-04-12T02:51:22.239302Z","shell.execute_reply.started":"2024-04-12T02:51:22.230806Z","shell.execute_reply":"2024-04-12T02:51:22.238195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_audio(low_quality_sample.iloc[0]['filename'])","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:22.240611Z","iopub.execute_input":"2024-04-12T02:51:22.241083Z","iopub.status.idle":"2024-04-12T02:51:25.938344Z","shell.execute_reply.started":"2024-04-12T02:51:22.241044Z","shell.execute_reply":"2024-04-12T02:51:25.937251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The low-quality samples seem to be mixed with other sounds, such as human voices\n# but is seems ok if added to train","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:25.939698Z","iopub.execute_input":"2024-04-12T02:51:25.940243Z","iopub.status.idle":"2024-04-12T02:51:25.944591Z","shell.execute_reply.started":"2024-04-12T02:51:25.940203Z","shell.execute_reply":"2024-04-12T02:51:25.943723Z"}}},{"cell_type":"code","source":"visualize_audio(high_quality_sample.iloc[2]['filename'])","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:25.945870Z","iopub.execute_input":"2024-04-12T02:51:25.946198Z","iopub.status.idle":"2024-04-12T02:51:29.579217Z","shell.execute_reply.started":"2024-04-12T02:51:25.946169Z","shell.execute_reply":"2024-04-12T02:51:29.577897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For the high-quality ones, there are purer bird calls.","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:29.582653Z","iopub.execute_input":"2024-04-12T02:51:29.582999Z","iopub.status.idle":"2024-04-12T02:51:29.587211Z","shell.execute_reply.started":"2024-04-12T02:51:29.582969Z","shell.execute_reply":"2024-04-12T02:51:29.586181Z"}}},{"cell_type":"code","source":"# now let's check the model for rating regression","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:29.588927Z","iopub.execute_input":"2024-04-12T02:51:29.589321Z","iopub.status.idle":"2024-04-12T02:51:29.600513Z","shell.execute_reply.started":"2024-04-12T02:51:29.589289Z","shell.execute_reply":"2024-04-12T02:51:29.599240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_audio(filepath, sr=32000):\n    waveform_orig, sample_rate = librosa.load(filepath, sr=None, mono=False)\n    if sample_rate != sr:\n        waveform_orig = librosa.resample(waveform_orig, orig_sr=sample_rate, target_sr=sr)\n    return waveform_orig            \n\n\ndef make_frame(audio, duration=20, sr=32000):\n    frame_length = int(duration * sr)\n    frame_step = int(duration * sr)\n    chunks = [\n        audio[i:i+frame_length] \n        for i in range(0, len(audio), frame_step)\n    ]\n    for i in range(len(chunks)):\n        if len(chunks[i]) < frame_length:\n            padding = np.zeros(frame_length - len(chunks[i]))\n            chunks[i] = np.concatenate([chunks[i], padding])\n            \n    return np.array(chunks)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-12T02:51:29.601953Z","iopub.execute_input":"2024-04-12T02:51:29.602302Z","iopub.status.idle":"2024-04-12T02:51:29.613897Z","shell.execute_reply.started":"2024-04-12T02:51:29.602273Z","shell.execute_reply":"2024-04-12T02:51:29.612778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport timm\nimport torch.nn.functional as F\nfrom torchaudio.transforms import MelSpectrogram, AmplitudeToDB\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = torch.nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\nclass NormalizeMelSpec(nn.Module):\n    def __init__(self, eps=1e-6):\n        super().__init__()\n        self.eps = eps\n\n    def forward(self, X):\n        mean = X.mean((1, 2), keepdim=True)\n        std = X.std((1, 2), keepdim=True)\n        Xstd = (X - mean) / (std + self.eps)\n        norm_min, norm_max = \\\n            Xstd.min(-1)[0].min(-1)[0], Xstd.max(-1)[0].max(-1)[0]\n        fix_ind = (norm_max - norm_min) > self.eps * torch.ones_like(\n            (norm_max - norm_min)\n        )\n        V = torch.zeros_like(Xstd)\n        if fix_ind.sum():\n            V_fix = Xstd[fix_ind]\n            norm_max_fix = norm_max[fix_ind, None, None]\n            norm_min_fix = norm_min[fix_ind, None, None]\n            V_fix = torch.max(\n                torch.min(V_fix, norm_max_fix),\n                norm_min_fix,\n            )\n            V_fix = (V_fix - norm_min_fix) / (norm_max_fix - norm_min_fix)\n            V[fix_ind] = V_fix\n        return V\n\n\n\nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.logmelspec_extractor = nn.Sequential(\n            MelSpectrogram(\n                sample_rate=32000,\n                n_mels=128,\n                f_min=20,\n                f_max=16000,\n                n_fft=2048,\n                hop_length=512,\n                normalized=True,\n            ),\n            AmplitudeToDB(top_db=80.0),\n            NormalizeMelSpec(),\n        )\n\n        base_model = timm.create_model(\n            'eca_nfnet_l0',\n            features_only=False,\n            pretrained=False,\n            in_chans=1\n        )\n\n        layers = list(base_model.children())[:-2]\n        self.backbone = nn.Sequential(*layers)\n\n        if hasattr(base_model, \"fc\"):\n            dense_input = base_model.fc.in_features\n        else:\n            dense_input = base_model.feature_info[-1][\"num_chs\"]\n\n        self.global_pool = GeM()\n        self.head = nn.Linear(dense_input, 1)\n\n    def forward(self, x):\n        x = self.logmelspec_extractor(x)[:, None]\n        x = self.backbone(x)\n        x = self.global_pool(x)\n        x = x[:, :, 0, 0]\n        logit = self.head(x)\n        return logit","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-12T02:51:29.615285Z","iopub.execute_input":"2024-04-12T02:51:29.615583Z","iopub.status.idle":"2024-04-12T02:51:31.744216Z","shell.execute_reply.started":"2024-04-12T02:51:29.615558Z","shell.execute_reply":"2024-04-12T02:51:31.743301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_kaggle:\n    model_path = '/kaggle/input/bird-sound-rating-model/regress_rating_model.pth'\nelse:\n    model_path = '/home/hw/m2_disk/kaggle/BirdCLEF2024_lhw/try_regress_rating/try_regress_rating/regress_rating_model.pth'\n\nmodel = Model().cuda()\nmodel.load_state_dict(torch.load(model_path))\nmodel = model.eval()","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:31.745352Z","iopub.execute_input":"2024-04-12T02:51:31.745640Z","iopub.status.idle":"2024-04-12T02:51:33.522446Z","shell.execute_reply.started":"2024-04-12T02:51:31.745613Z","shell.execute_reply":"2024-04-12T02:51:33.521274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_audio_rating_by_model(filepath):\n    audio = load_audio(filepath)\n    only_first_20sec = True # only first 20sec for quik inference, comment if infer all\n    chunks = make_frame(audio)\n    if only_first_20sec:\n        chunks = chunks[0:1]\n    chunks = torch.from_numpy(chunks).float().cuda()\n    with torch.no_grad(), torch.cuda.amp.autocast():\n        chunk_preds = model(chunks).cpu().numpy()\n\n    return chunk_preds","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:33.523643Z","iopub.execute_input":"2024-04-12T02:51:33.523981Z","iopub.status.idle":"2024-04-12T02:51:33.530451Z","shell.execute_reply.started":"2024-04-12T02:51:33.523953Z","shell.execute_reply":"2024-04-12T02:51:33.529363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = calculate_audio_rating_by_model(high_quality_sample.iloc[2]['filename'])\nprint('chunks (20s) rating pred: ', r)\nprint('pred mean rating: ', r.mean())","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:33.531956Z","iopub.execute_input":"2024-04-12T02:51:33.532588Z","iopub.status.idle":"2024-04-12T02:51:34.095860Z","shell.execute_reply.started":"2024-04-12T02:51:33.532562Z","shell.execute_reply":"2024-04-12T02:51:34.094864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_ratings = []\nif infer_all_on_kaggle:\n    for filepath in tqdm.tqdm(df['filename']):\n        r = calculate_audio_rating_by_model(filepath)\n        pred_ratings.append(r.mean())\n    pred_ratings = np.array(pred_ratings)\nelse:\n    pred_ratings = pickle.load(open('/kaggle/input/bird-sound-rating-model/birdclef_2024_pred_rating.pkl', 'rb'))\n    pred_ratings = np.array(pred_ratings, dtype=np.float32)\ndf['pred_rating'] = pred_ratings\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:52:35.024062Z","iopub.execute_input":"2024-04-12T02:52:35.024487Z","iopub.status.idle":"2024-04-12T02:52:35.032714Z","shell.execute_reply.started":"2024-04-12T02:52:35.024456Z","shell.execute_reply":"2024-04-12T02:52:35.031466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"err = np.abs(pred_ratings - df['rating'].values).mean()\nprint('err: ', err)","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:52:37.654674Z","iopub.execute_input":"2024-04-12T02:52:37.655428Z","iopub.status.idle":"2024-04-12T02:52:37.661002Z","shell.execute_reply.started":"2024-04-12T02:52:37.655398Z","shell.execute_reply":"2024-04-12T02:52:37.659932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\n\nplt.subplot(1,2,1)  # (nrows, ncols, index)\nsns.histplot(df['rating'])\nplt.title('Original rating distribution')\n\nplt.subplot(1,2,2)\nsns.histplot(df['pred_rating'])\nplt.title('Predicted rating distribution')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:52:39.504249Z","iopub.execute_input":"2024-04-12T02:52:39.504963Z","iopub.status.idle":"2024-04-12T02:52:40.358333Z","shell.execute_reply.started":"2024-04-12T02:52:39.504928Z","shell.execute_reply":"2024-04-12T02:52:40.357377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's look unlabeled_soundscapes\nif infer_all_on_kaggle:\n    test_files = glob.glob(f'{data_root}/unlabeled_soundscapes/*.ogg')\n    test_files = test_files[:10]\n    unlabeled_pred_ratings = []\n    for filepath in tqdm.tqdm(test_files):\n        r = calculate_audio_rating_by_model(filepath)\n        unlabeled_pred_ratings.append(r.mean())\n    unlabeled_pred_ratings = np.array(unlabeled_pred_ratings)\n\n    # store to DataFrame, column: [filepath(basenam),  pred_rating]\n    df_unlabeled = pd.DataFrame()\n    df_unlabeled['filepath'] = [os.path.basename(f) for f in test_files]\n    df_unlabeled['pred_rating'] = unlabeled_pred_ratings\n    df_unlabeled.to_csv('./unlabeled_pred_ratings.csv', index=False)\nelse:\n    df_unlabeled = pd.read_csv('/kaggle/input/bird-sound-rating-model/unlabeled_pred_ratings.csv')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:52:42.982089Z","iopub.execute_input":"2024-04-12T02:52:42.982982Z","iopub.status.idle":"2024-04-12T02:52:43.034073Z","shell.execute_reply.started":"2024-04-12T02:52:42.982951Z","shell.execute_reply":"2024-04-12T02:52:43.033193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(df_unlabeled['pred_rating'])\nplt.title('unlabeled pred rating distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:52:46.945440Z","iopub.execute_input":"2024-04-12T02:52:46.945830Z","iopub.status.idle":"2024-04-12T02:52:47.213353Z","shell.execute_reply.started":"2024-04-12T02:52:46.945799Z","shell.execute_reply":"2024-04-12T02:52:47.212301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# unlabeled data qulity is worse than train","metadata":{"execution":{"iopub.status.busy":"2024-04-12T02:51:36.402316Z","iopub.status.idle":"2024-04-12T02:51:36.402763Z","shell.execute_reply.started":"2024-04-12T02:51:36.402534Z","shell.execute_reply":"2024-04-12T02:51:36.402552Z"}}}]}