{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":172219831,"sourceType":"kernelVersion"},{"sourceId":172611569,"sourceType":"kernelVersion"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tqdm import tqdm\nimport plotly.express as px\nfrom IPython.display import Audio\nimport torchaudio\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:13:49.573771Z","iopub.execute_input":"2024-04-18T06:13:49.574341Z","iopub.status.idle":"2024-04-18T06:13:49.582732Z","shell.execute_reply.started":"2024-04-18T06:13:49.574294Z","shell.execute_reply":"2024-04-18T06:13:49.581489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/bc24-data-add-max-volumes/train_metadata_with_volumes.csv')\n\nTRAIN_PATH = Path('/kaggle/input/birdclef-2024/train_audio')\n\nSAMPLE_RATE = 32_000\nWINDOW = 5*SAMPLE_RATE","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:13:52.416860Z","iopub.execute_input":"2024-04-18T06:13:52.417323Z","iopub.status.idle":"2024-04-18T06:13:52.614625Z","shell.execute_reply.started":"2024-04-18T06:13:52.417285Z","shell.execute_reply":"2024-04-18T06:13:52.613345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_volumes = np.max([np.array(df.first5s_max_volume), np.array(df.last5s_max_volume)], 0)\nvolumes = np.minimum(max_volumes[None, :], max_volumes[:, None])","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:22:58.122754Z","iopub.execute_input":"2024-04-18T06:22:58.123298Z","iopub.status.idle":"2024-04-18T06:22:59.450605Z","shell.execute_reply.started":"2024-04-18T06:22:58.123262Z","shell.execute_reply":"2024-04-18T06:22:59.449223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distances = torch.tensor(np.load('/kaggle/input/bc24-data-embedding-distances/distances.npy'))","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:15:35.437820Z","iopub.execute_input":"2024-04-18T06:15:35.438636Z","iopub.status.idle":"2024-04-18T06:16:01.103527Z","shell.execute_reply.started":"2024-04-18T06:15:35.438583Z","shell.execute_reply":"2024-04-18T06:16:01.102111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helpers for displaying bird recordings for manual inspection\ncompute_melspec = torchaudio.transforms.MelSpectrogram(\n    sample_rate=SAMPLE_RATE,\n    n_mels=128,\n    n_fft=2048, \n    hop_length=512,\n)\n\npower_to_db = torchaudio.transforms.AmplitudeToDB(\n    stype=\"power\",\n    top_db=80.0,\n)\n\n\ndef show_bird(index, start=0, secs=5):\n    audio = torchaudio.load(TRAIN_PATH / df.filename[index], start, start+32_000*secs)[0][0]\n    display(Audio(audio, rate=SAMPLE_RATE))\n    plt.figure(figsize=(12, 2.5))\n    plt.subplot(121)\n    plt.plot(audio)\n    plt.gca().get_xaxis().set_visible(False)\n    plt.subplot(122)\n    plt.imshow(power_to_db(compute_melspec(audio)))\n    plt.show()\n    return df.iloc[index]","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:16:01.106688Z","iopub.execute_input":"2024-04-18T06:16:01.108398Z","iopub.status.idle":"2024-04-18T06:16:01.219657Z","shell.execute_reply.started":"2024-04-18T06:16:01.108331Z","shell.execute_reply":"2024-04-18T06:16:01.218330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# these are have low embedding distance to one another but are actually distinct\n# I found them by manually inspecting the spectrograms that were output from the loop below\nknown_non_dupes = {\n ('litgre1/XC459392.ogg', 'commoo3/XC569721.ogg'), #0.43129920959472656 0.0189\n (\"litgre1/XC636770.ogg\", \"comgre/XC636748.ogg\"),\n (\"litgre1/XC459392.ogg\", \"comgre/XC636748.ogg\"),\n (\"commoo3/XC569721.ogg\", \"comgre/XC636748.ogg\"),\n}\n\n# these were found from curration of the below loop\nknown_dupes = {\n  # SOUND_THRESHOLD = 0.1\n ('blhori1/XC743616.ogg', 'blhori1/XC537503.ogg'),\n ('brakit1/XC743675.ogg', 'brakit1/XC537471.ogg'),\n ('brcful1/XC197746.ogg', 'brcful1/XC157971.ogg'),\n ('brnshr/XC510751.ogg', 'brnshr/XC510750.ogg'),\n ('btbeat1/XC743619.ogg', 'btbeat1/XC683300.ogg'),\n ('cohcuc1/XC253418.ogg', 'cohcuc1/XC241127.ogg'),\n ('cohcuc1/XC423422.ogg', 'cohcuc1/XC423419.ogg'),\n ('comgre/XC202776.ogg', 'comgre/XC192404.ogg'),\n ('comgre/XC602468.ogg', 'comgre/XC175341.ogg'),\n ('comkin1/XC207123.ogg', 'comior1/XC207062.ogg'),\n ('comkin1/XC691421.ogg', 'comkin1/XC690633.ogg'),\n ('commyn/XC577887.ogg', 'commyn/XC577886.ogg'),\n ('commyn/XC652903.ogg', 'commyn/XC652901.ogg'),\n ('compea/XC665320.ogg', 'compea/XC644022.ogg'),\n ('comtai1/XC542379.ogg', 'comtai1/XC540352.ogg'),\n ('dafbab1/XC188307.ogg', 'dafbab1/XC187059.ogg'),\n ('dafbab1/XC188308.ogg', 'dafbab1/XC187068.ogg'),\n ('dafbab1/XC188309.ogg', 'dafbab1/XC187069.ogg'),\n ('dafbab1/XC197745.ogg', 'dafbab1/XC157972.ogg'),\n ('eaywag1/XC527600.ogg', 'eaywag1/XC527598.ogg'),\n ('eucdov/XC355153.ogg', 'eucdov/XC355152.ogg'),\n ('eucdov/XC360303.ogg', 'eucdov/XC347428.ogg'),\n ('eucdov/XC365606.ogg', 'eucdov/XC124694.ogg'),\n ('eucdov/XC371039.ogg', 'eucdov/XC368596.ogg'),\n ('goflea1/XC163901.ogg', 'bladro1/XC163901.ogg'),\n ('goflea1/XC208794.ogg', 'bladro1/XC208794.ogg'),\n ('goflea1/XC208795.ogg', 'bladro1/XC208795.ogg'),\n ('goflea1/XC209203.ogg', 'bladro1/XC209203.ogg'),\n ('goflea1/XC209549.ogg', 'bladro1/XC209549.ogg'),\n ('goflea1/XC209564.ogg', 'bladro1/XC209564.ogg'),\n ('graher1/XC357552.ogg', 'graher1/XC357551.ogg'),\n ('grbeat1/XC304004.ogg', 'grbeat1/XC303999.ogg'),\n ('greegr/XC247286.ogg', 'categr/XC197438.ogg'),\n ('grewar3/XC743681.ogg', 'grewar3/XC537475.ogg'),\n ('grnwar1/XC197744.ogg', 'grnwar1/XC157973.ogg'),\n ('grtdro1/XC651708.ogg', 'grtdro1/XC613192.ogg'),\n ('grywag/XC575903.ogg', 'grywag/XC575901.ogg'),\n ('grywag/XC650696.ogg', 'grywag/XC592019.ogg'),\n ('grywag/XC690448.ogg', 'grywag/XC655063.ogg'),\n ('grywag/XC745653.ogg', 'grywag/XC745650.ogg'),\n ('grywag/XC812496.ogg', 'grywag/XC812495.ogg'),\n ('heswoo1/XC357155.ogg', 'heswoo1/XC357149.ogg'),\n ('hoopoe/XC631301.ogg', 'hoopoe/XC365530.ogg'),\n ('hoopoe/XC631304.ogg', 'hoopoe/XC252584.ogg'),\n ('houspa/XC326675.ogg', 'houspa/XC326674.ogg'),\n ('junbab2/XC282587.ogg', 'junbab2/XC282586.ogg'),\n ('labcro1/XC267645.ogg', 'labcro1/XC265731.ogg'),\n ('labcro1/XC447036.ogg', 'houcro1/XC447036.ogg'),\n ('labcro1/XC823514.ogg', 'gybpri1/XC823527.ogg'),\n ('laudov1/XC405375.ogg', 'laudov1/XC405374.ogg'),\n ('laudov1/XC514027.ogg', 'eucdov/XC514027.ogg'),\n ('lblwar1/XC197743.ogg', 'lblwar1/XC157974.ogg'),\n ('lewduc1/XC261506.ogg', 'lewduc1/XC254813.ogg'),\n ('litegr/XC535540.ogg', 'litegr/XC448898.ogg'),\n ('litegr/XC535552.ogg', 'litegr/XC447850.ogg'),\n ('litspi1/XC674522.ogg', 'comtai1/XC674522.ogg'),\n ('litspi1/XC722435.ogg', 'litspi1/XC721636.ogg'),\n ('litspi1/XC722436.ogg', 'litspi1/XC721637.ogg'),\n ('litswi1/XC443070.ogg', 'litswi1/XC440301.ogg'),\n ('lobsun2/XC197742.ogg', 'lobsun2/XC157975.ogg'),\n ('maghor2/XC197740.ogg', 'maghor2/XC157978.ogg'),\n ('maghor2/XC786588.ogg', 'maghor2/XC786587.ogg'),\n ('malpar1/XC197770.ogg', 'malpar1/XC157976.ogg'),\n ('marsan/XC383290.ogg', 'marsan/XC383288.ogg'),\n ('piebus1/XC165050.ogg', 'piebus1/XC122395.ogg'),\n ('piebus1/XC814459.ogg', 'piebus1/XC792272.ogg'),\n ('placuc3/XC490344.ogg', 'placuc3/XC486683.ogg'),\n ('placuc3/XC572952.ogg', 'placuc3/XC572950.ogg'),\n ('purher1/XC467373.ogg', 'graher1/XC467373.ogg'),\n ('pursun3/XC268375.ogg', 'comtai1/XC241382.ogg'),\n ('pursun4/XC514853.ogg', 'pursun4/XC514852.ogg'),\n ('putbab1/XC574864.ogg', 'brcful1/XC574864.ogg'),\n ('rewbul/XC306398.ogg', 'bkcbul1/XC306398.ogg'),\n ('rewlap1/XC733007.ogg', 'rewlap1/XC732874.ogg'),\n ('rorpar/XC199488.ogg', 'rorpar/XC199339.ogg'),\n ('rorpar/XC516404.ogg', 'rorpar/XC516402.ogg'),\n ('sbeowl1/XC522123.ogg', 'brfowl1/XC522123.ogg'),\n ('sohmyn1/XC744700.ogg', 'sohmyn1/XC743682.ogg'),\n ('spodov/XC163930.ogg', 'bladro1/XC163901.ogg'),\n ('spodov/XC163930.ogg', 'goflea1/XC163901.ogg'),\n ('stbkin1/XC360661.ogg', 'stbkin1/XC199815.ogg'),\n ('stbkin1/XC406140.ogg', 'stbkin1/XC406138.ogg'),\n ('vefnut1/XC197738.ogg', 'vefnut1/XC157979.ogg'),\n ('vefnut1/XC293526.ogg', 'vefnut1/XC289785.ogg'),\n ('wemhar1/XC590355.ogg', 'wemhar1/XC590354.ogg'),\n ('whbbul2/XC335671.ogg', 'whbbul2/XC335670.ogg'),\n ('whbsho3/XC856465.ogg', 'whbsho3/XC856463.ogg'),\n ('whbsho3/XC856468.ogg', 'whbsho3/XC856463.ogg'),\n ('whbsho3/XC856468.ogg', 'whbsho3/XC856465.ogg'),\n ('whbwat1/XC840073.ogg', 'whbwat1/XC840071.ogg'),\n ('whbwoo2/XC239509.ogg', 'rufwoo2/XC239509.ogg'),\n ('whcbar1/XC659329.ogg', 'insowl1/XC659329.ogg'),\n ('whtkin2/XC197737.ogg', 'whtkin2/XC157981.ogg'),\n ('whtkin2/XC430267.ogg', 'whtkin2/XC430256.ogg'),\n ('whtkin2/XC503389.ogg', 'comior1/XC503389.ogg'),\n ('woosan/XC184466.ogg', 'marsan/XC184466.ogg'),\n ('zitcis1/XC303866.ogg', 'zitcis1/XC302781.ogg'),\n\n # SOUND_THRESHOLD = 0.02\n ('asbfly/XC724266.ogg', 'asbfly/XC724148.ogg'),\n ('blrwar1/XC662286.ogg', 'blrwar1/XC662285.ogg'),\n ('comsan/XC385909.ogg', 'comsan/XC385908.ogg'),\n ('comsan/XC667807.ogg', 'comsan/XC667806.ogg'),\n ('comtai1/XC305210.ogg', 'comtai1/XC304811.ogg'),\n ('crfbar1/XC615780.ogg', 'crfbar1/XC615778.ogg'),\n ('eucdov/XC747422.ogg', 'eucdov/XC747408.ogg'),\n ('insowl1/XC305214.ogg', 'insowl1/XC301142.ogg'),\n ('laudov1/XC185511.ogg', 'grewar3/XC185505.ogg'),\n ('litegr/XC403621.ogg', 'bcnher/XC403621.ogg'),\n ('litgre1/XC630775.ogg', 'litgre1/XC630560.ogg'),\n ('litgre1/XC776082.ogg', 'litgre1/XC663244.ogg'),\n ('marsan/XC733175.ogg', 'marsan/XC716673.ogg'),\n ('mawthr1/XC455222.ogg', 'mawthr1/XC455211.ogg'),\n ('plaflo1/XC615781.ogg', 'plaflo1/XC614946.ogg'),\n ('rorpar/XC402325.ogg', 'comior1/XC402326.ogg'),\n ('spepic1/XC804432.ogg', 'spepic1/XC804431.ogg'),\n ('spoowl1/XC591485.ogg', 'spoowl1/XC591177.ogg'),\n ('stbkin1/XC266782.ogg', 'stbkin1/XC266682.ogg'),\n ('wemhar1/XC581045.ogg', 'comsan/XC581045.ogg'),\n ('whiter2/XC265271.ogg', 'whiter2/XC265267.ogg'),\n ('woosan/XC825766.ogg', 'grnsan/XC825765.ogg'),\n    \n # SOUND_THRESHOLD = 0.015\n (\"comsan/XC643721.ogg\", \"comsan/XC642698.ogg\"),\n    \n # SOUND_THRESHOLD = 0.014\n (\"purher1/XC827209.ogg\", \"purher1/XC827207.ogg\"),\n \n # EMBEDDING_THRESHOLD = 0.2, SOUND_THRESHOLD = 0.0135\n ('barswa/XC575749.ogg', 'barswa/XC575747.ogg'),\n ('comtai1/XC542375.ogg', 'comtai1/XC540351.ogg'),\n ('grywag/XC459760.ogg', 'grywag/XC457124.ogg'),\n ('rewbul/XC713308.ogg', 'asbfly/XC713467.ogg'),\n    \n # EMBEDDING_THRESHOLD = 0.5\n ('blhori1/XC417215.ogg', 'blhori1/XC417133.ogg'),\n ('btbeat1/XC665307.ogg', 'btbeat1/XC513403.ogg'),\n ('btbeat1/XC743618.ogg', 'btbeat1/XC683300.ogg'),\n ('btbeat1/XC743619.ogg', 'btbeat1/XC743618.ogg'),\n ('comgre/XC64628.ogg', 'comgre/XC58586.ogg'),\n ('eucdov/XC789608.ogg', 'eucdov/XC788267.ogg'),\n ('graher1/XC590235.ogg', 'graher1/XC590144.ogg'),\n ('heswoo1/XC744698.ogg', 'heswoo1/XC665715.ogg'),\n ('houcro1/XC744704.ogg', 'houcro1/XC683047.ogg'),\n \n # EMBEDDING_THRESHOLD = 0.7\n (\"inbrob1/XC744708.ogg\", \"inbrob1/XC744706.ogg\"),\n    \n # others\n (\"whtkin2/XC540094.ogg\", \"whtkin2/XC540087.ogg\"),# 1.1914571523666382 0.958876\n (\"labcro1/XC37773.ogg\", \"labcro1/XC19736.ogg\"),# 1.1931051015853882 0.99630284\n (\"woosan/XC742927.ogg\", \"woosan/XC740798.ogg\"),# 1.524927020072937 0.372408\n (\"woosan/XC587076.ogg\", \"woosan/XC578599.ogg\"),# 1.6371620893478394 0.2585204\n (\"grecou1/XC365426.ogg\", \"grecou1/XC365425.ogg\"),# 1.765889048576355 0.11622\n (\"woosan/XC545316.ogg\", \"woosan/XC476064.ogg\"),# 1.8372114896774292 0.0573818\n (\"labcro1/XC345836.ogg\", \"labcro1/XC312582.ogg\"),# 1.925593614578247 0.49062520265579224\n \n (\"categr/XC787914.ogg\", \"categr/XC438523.ogg\"),# 2.4874114990234375 0.22858968377113342\n (\"bkskit1/XC350251.ogg\", \"bkskit1/XC350249.ogg\"),# 2.575990676879883 0.23503872752189636\n (\"orihob2/XC557991.ogg\", \"orihob2/XC557293.ogg\"),# 2.626605749130249 0.21241611242294312\n (\"comtai1/XC126749.ogg\", \"comtai1/XC122978.ogg\"),# 2.758732318878174 0.8704305291175842\n (\"bcnher/XC669544.ogg\", \"bcnher/XC669542.ogg\"),# 2.767756462097168 0.21584786474704742\n (\"comior1/XC305930.ogg\", \"comior1/XC303819.ogg\"),# 2.850081443786621 0.9506171941757202\n \n (\"asbfly/XC347563.ogg\", \"asbfly/XC341611.ogg\"), # 3.606727361679077 0.3008304834365845\n (\"grewar3/XC507658.ogg\", \"grewar3/XC184040.ogg\") # 4.883322238922119 0.3915347158908844\n}\n\nexclude = known_non_dupes | known_dupes","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:47:09.497918Z","iopub.execute_input":"2024-04-18T06:47:09.498397Z","iopub.status.idle":"2024-04-18T06:47:09.533843Z","shell.execute_reply.started":"2024-04-18T06:47:09.498366Z","shell.execute_reply":"2024-04-18T06:47:09.532423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(known_dupes)","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:47:10.822278Z","iopub.execute_input":"2024-04-18T06:47:10.825051Z","iopub.status.idle":"2024-04-18T06:47:10.832885Z","shell.execute_reply.started":"2024-04-18T06:47:10.824993Z","shell.execute_reply":"2024-04-18T06:47:10.830775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pick a low threshold and then manually currate the list\nEMBEDDING_THRESHOLD = 0.7#5\nVOLUME_THRESHOLD = 0.0135#0.3\n\nnot_eye = (1 - torch.triu(torch.ones(distances.shape))).bool()\npossible_dupes = (\n    (distances < EMBEDDING_THRESHOLD) & \n    (volumes > VOLUME_THRESHOLD) & \n    not_eye\n).nonzero().to('cpu').numpy()\npossible_dupes = sorted(\n  [(a, b, distances[a][b].item()) for a, b in possible_dupes], \n  key=lambda a: a[2]\n)\n\nlen(possible_dupes)","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:48:14.829313Z","iopub.execute_input":"2024-04-18T06:48:14.829851Z","iopub.status.idle":"2024-04-18T06:48:24.002367Z","shell.execute_reply.started":"2024-04-18T06:48:14.829816Z","shell.execute_reply":"2024-04-18T06:48:24.000292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dupes = []\n\nfor a, b, dist in tqdm(possible_dupes):\n    a_filename = df.filename[a]\n    b_filename = df.filename[b]\n    audio_max = volumes[a][b]\n\n    #if (a_filename, b_filename) in exclude:\n    #    continue\n\n    print(a, b, f'(\"{a_filename}\", \"{b_filename}\")', dist, audio_max)\n\n    a_audio = torchaudio.load(TRAIN_PATH / a_filename)[0][0]\n    a_audio = torch.concat([a_audio[:WINDOW], a_audio[-WINDOW:]], -1)\n    b_audio = torchaudio.load(TRAIN_PATH / b_filename)[0][0]\n    b_audio = torch.concat([b_audio[:WINDOW], b_audio[-WINDOW:]], -1)\n    plt.figure(figsize=(12, 5))\n    plt.subplot(121)\n    plt.axis('off')\n    plt.imshow(power_to_db(compute_melspec(a_audio)))\n    plt.subplot(122)\n    plt.imshow(power_to_db(compute_melspec(b_audio)))\n    plt.axis('off')\n    plt.show()\n\n    dupes.append((a_filename, b_filename, dist))","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:48:24.006729Z","iopub.execute_input":"2024-04-18T06:48:24.007378Z","iopub.status.idle":"2024-04-18T06:49:09.472875Z","shell.execute_reply.started":"2024-04-18T06:48:24.007329Z","shell.execute_reply":"2024-04-18T06:49:09.471599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dupe_df = pd.DataFrame([{\"filename_a\": a, \"filename_b\": b, \"distance\": d} for a, b, d in dupes])\ndupe_df.to_csv(\"duplicates.csv\", index=False);","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:49:09.474576Z","iopub.execute_input":"2024-04-18T06:49:09.474995Z","iopub.status.idle":"2024-04-18T06:49:09.489629Z","shell.execute_reply.started":"2024-04-18T06:49:09.474960Z","shell.execute_reply":"2024-04-18T06:49:09.488412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"known_dupes","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:47:28.155281Z","iopub.execute_input":"2024-04-18T06:47:28.155793Z","iopub.status.idle":"2024-04-18T06:47:28.177620Z","shell.execute_reply.started":"2024-04-18T06:47:28.155760Z","shell.execute_reply":"2024-04-18T06:47:28.175970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}