{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport numpy as np\nimport os\n\nimport tensorflow_hub as hub\nimport tensorflow as tf\n\nimport torchaudio\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\n\n\ndf = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\nAUDIO_PATH = Path('/kaggle/input/birdclef-2023/train_audio')\nmodel = hub.load('https://kaggle.com/models/google/bird-vocalization-classifier/frameworks/TensorFlow2/variations/bird-vocalization-classifier/versions/2')\nmodel_labels_df = pd.read_csv(hub.resolve('https://kaggle.com/models/google/bird-vocalization-classifier/frameworks/tensorFlow2/variations/bird-vocalization-classifier/versions/2') + \"/assets/label.csv\")\n\nSAMPLE_RATE = 32000\nWINDOW = 5*SAMPLE_RATE","metadata":{"execution":{"iopub.status.busy":"2023-04-11T04:04:31.980518Z","iopub.execute_input":"2023-04-11T04:04:31.981539Z","iopub.status.idle":"2023-04-11T04:04:37.184407Z","shell.execute_reply.started":"2023-04-11T04:04:31.981489Z","shell.execute_reply":"2023-04-11T04:04:37.182701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bc2023_labels = sorted(df.primary_label.unique())\nlabel_to_index = {v: k for k, v in enumerate(bc2023_labels)}\nmodel_labels = {v: k for k, v in enumerate(model_labels_df.ebird2021)}\nmodel_bc2023_indexes = [model_labels[label] if label in model_labels else -1 for label in bc2023_labels]","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:38:41.863437Z","iopub.execute_input":"2023-04-11T03:38:41.863824Z","iopub.status.idle":"2023-04-11T03:38:41.881775Z","shell.execute_reply.started":"2023-04-11T03:38:41.863784Z","shell.execute_reply":"2023-04-11T03:38:41.880675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save embeddings and predictions for every 5 sec non-overlapping audio","metadata":{}},{"cell_type":"code","source":"# use a torch dataloader to decode audio in parallel on CPU while GPU is running\nclass AudioDataset(Dataset):\n    def __len__(self):\n        return len(df)\n    def __getitem__(self, i):\n        filename = df.filename[i]\n        audio = torchaudio.load(AUDIO_PATH / filename)[0].numpy()[0]\n        return audio, filename\ndataloader = DataLoader(AudioDataset(), batch_size=1, num_workers=os.cpu_count())\n\n\n# embeddings are formated like {\"filename\": np.array(nx1280)} \n# (where n = the number of non overlapping 5 sec chunks in the audio)\nall_embeddings = {}\n\n# predictiones formated like {\"filename\": np.array(nx264)} \nall_predictions = {}\n\nwith tf.device('/gpu:0'):\n    for audio, filename in tqdm(dataloader):\n        audio = audio[0]\n        filename = filename[0]\n        file_embeddings = []\n        file_predictions = []\n        for i in range(0, len(audio), WINDOW):\n            clip = audio[i:i+WINDOW]\n            if len(clip) < WINDOW:\n                clip = np.concatenate([clip, np.zeros(WINDOW - len(clip))])\n            result = model.infer_tf(clip[None, :])\n            file_embeddings.append(result[1][0].numpy())\n            prediction = np.concatenate([result[0].numpy(), -100], axis=None) # add -100 logit for unpredicted birds\n            file_predictions.append(prediction[model_bc2023_indexes])\n        all_embeddings[filename] = np.stack(file_embeddings)\n        all_predictions[filename] = np.stack(file_predictions)\n\ntorch.save(all_embeddings, 'embeddings.pt')\ntorch.save(all_predictions, 'predictions.pt')","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:38:41.883670Z","iopub.execute_input":"2023-04-11T03:38:41.884097Z","iopub.status.idle":"2023-04-11T03:42:51.831028Z","shell.execute_reply.started":"2023-04-11T03:38:41.884058Z","shell.execute_reply":"2023-04-11T03:42:51.829643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scores of predictions on the first 5 seconds of each recording","metadata":{}},{"cell_type":"code","source":"predicted_classes = torch.tensor([row[0].argmax() for row in all_predictions.values()])\nactual_classes = torch.tensor([label_to_index[label] for label in df.primary_label])\ncorrect = predicted_classes == actual_classes\naccuracy = correct.float().mean()\naccuracy","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:44:30.599792Z","iopub.execute_input":"2023-04-11T03:44:30.600195Z","iopub.status.idle":"2023-04-11T03:44:30.638137Z","shell.execute_reply.started":"2023-04-11T03:44:30.600159Z","shell.execute_reply":"2023-04-11T03:44:30.636420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logits = torch.stack([torch.tensor(row[0]) for row in all_predictions.values()])\nce_loss = torch.nn.CrossEntropyLoss()(logits, actual_classes)\nce_loss","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:42:51.908857Z","iopub.execute_input":"2023-04-11T03:42:51.909740Z","iopub.status.idle":"2023-04-11T03:42:52.067965Z","shell.execute_reply.started":"2023-04-11T03:42:51.909694Z","shell.execute_reply":"2023-04-11T03:42:52.066957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual_probs = torch.eye(len(bc2023_labels))[actual_classes]\nbce_loss = torch.nn.BCEWithLogitsLoss()(logits, actual_probs)\nbce_loss","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:42:52.071132Z","iopub.execute_input":"2023-04-11T03:42:52.071440Z","iopub.status.idle":"2023-04-11T03:42:52.203649Z","shell.execute_reply.started":"2023-04-11T03:42:52.071412Z","shell.execute_reply":"2023-04-11T03:42:52.202631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport sklearn.metrics\n\ndef padded_cmap(solution, submission, padding_factor=5):\n    #solution = solution.drop(['row_id'], axis=1, errors='ignore')\n    #submission = submission.drop(['row_id'], axis=1, errors='ignore')\n    new_rows = []\n    for i in range(padding_factor):\n        new_rows.append([1 for i in range(len(solution.columns))])\n    new_rows = pd.DataFrame(new_rows)\n    new_rows.columns = solution.columns\n    padded_solution = pd.concat([solution, new_rows]).reset_index(drop=True).copy()\n    padded_submission = pd.concat([submission, new_rows]).reset_index(drop=True).copy()\n    score = sklearn.metrics.average_precision_score(\n        padded_solution.values,\n        padded_submission.values,\n        average='macro',\n    )\n    return score","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:43:55.993448Z","iopub.execute_input":"2023-04-11T03:43:55.994157Z","iopub.status.idle":"2023-04-11T03:43:56.001957Z","shell.execute_reply.started":"2023-04-11T03:43:55.994115Z","shell.execute_reply":"2023-04-11T03:43:56.000777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution = pd.DataFrame(actual_probs.numpy(), columns=bc2023_labels)\npadded_cmap(\n    solution=solution,\n    submission=pd.DataFrame(torch.softmax(logits, 1).numpy(), columns=bc2023_labels),\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:43:11.773357Z","iopub.execute_input":"2023-04-11T03:43:11.776045Z","iopub.status.idle":"2023-04-11T03:43:24.479786Z","shell.execute_reply.started":"2023-04-11T03:43:11.776003Z","shell.execute_reply":"2023-04-11T03:43:24.478708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"padded_cmap(\n    solution=solution,\n    submission=pd.DataFrame(torch.sigmoid(logits).numpy(), columns=bc2023_labels),\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T03:43:01.189124Z","iopub.execute_input":"2023-04-11T03:43:01.189609Z","iopub.status.idle":"2023-04-11T03:43:11.769033Z","shell.execute_reply.started":"2023-04-11T03:43:01.189569Z","shell.execute_reply":"2023-04-11T03:43:11.767943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analysis","metadata":{}},{"cell_type":"code","source":"df['correct'] = correct.bool().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T04:06:27.225942Z","iopub.execute_input":"2023-04-11T04:06:27.226485Z","iopub.status.idle":"2023-04-11T04:06:27.236925Z","shell.execute_reply.started":"2023-04-11T04:06:27.226447Z","shell.execute_reply":"2023-04-11T04:06:27.235684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\npx.histogram(\n    df,\n    title=\"Distribution of accuracy\",\n    x='primary_label',\n    color='correct',\n).update_xaxes(categoryorder=\"total descending\").show()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T04:01:11.147857Z","iopub.execute_input":"2023-04-11T04:01:11.148772Z","iopub.status.idle":"2023-04-11T04:01:11.265078Z","shell.execute_reply.started":"2023-04-11T04:01:11.148717Z","shell.execute_reply":"2023-04-11T04:01:11.263925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Failure examples","metadata":{}},{"cell_type":"code","source":"from IPython.display import Audio\nimport torchaudio\nimport matplotlib.pyplot as plt\n\ncompute_melspec = torchaudio.transforms.MelSpectrogram(\n    sample_rate=SAMPLE_RATE,\n    n_mels=128,\n    n_fft=2048, \n    hop_length=512,\n    f_min=0,\n    f_max=SAMPLE_RATE // 2,\n)\n\npower_to_db = torchaudio.transforms.AmplitudeToDB(\n    stype=\"power\",\n    top_db=80.0,\n)\n\ndef show_bird(index, start=0):\n    audio = torchaudio.load(AUDIO_PATH / df.filename[index], start, start+WINDOW)[0][0]\n    display(df.iloc[index])\n    display(Audio(audio, rate=SAMPLE_RATE))\n    plt.figure(figsize=(12, 2.5))\n    plt.subplot(121)\n    plt.plot(audio)\n    plt.gca().get_xaxis().set_visible(False)\n    plt.subplot(122)\n    plt.imshow(power_to_db(compute_melspec(audio)))\n    plt.show()\n    return \n\n\n# filter out birds that the model doesn't predict\nmissing_birds = set(np.array(bc2023_labels)[np.array(model_bc2023_indexes) == -1]) \nprobs = torch.sigmoid(logits)\n\nfor count, i in enumerate(df[~df.correct & ~df.primary_label.isin(missing_birds)].sample(10).index):\n    print('### EXAMPLE', count+1,'###')\n    correct_class = df.primary_label[i]\n    predicted_class_index = label_to_index[correct_class]\n    predicted_prob_for_correct_label = probs[i, predicted_class_index]\n    rank = 1 + sorted(probs[i], reverse=True).index(predicted_prob_for_correct_label)\n    \n    predicted_class_index = probs[i].argmax().item()\n    predicted_clas = bc2023_labels[predicted_class_index]\n    max_predicted_prob = probs[i][predicted_class_index]\n    \n    print(f'correct was {correct_class}, given prob {predicted_prob_for_correct_label} and ranked #{rank}')\n    print(f'predicted {predicted_clas}, with prob {max_predicted_prob}')\n    print()\n    show_bird(i)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T04:27:37.753891Z","iopub.execute_input":"2023-04-11T04:27:37.754330Z","iopub.status.idle":"2023-04-11T04:28:00.751462Z","shell.execute_reply.started":"2023-04-11T04:27:37.754293Z","shell.execute_reply":"2023-04-11T04:28:00.750576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}