{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"dupes = {\n    (\"asbfly/XC347563.ogg\", \"asbfly/XC341611.ogg\"),\n    (\"grewar3/XC507658.ogg\", \"grewar3/XC184040.ogg\"),\n    \n    ('asbfly/XC724266.ogg', 'asbfly/XC724148.ogg'),\n    ('barswa/XC575749.ogg', 'barswa/XC575747.ogg'),\n    ('bcnher/XC669544.ogg', 'bcnher/XC669542.ogg'),\n    ('bkskit1/XC350251.ogg', 'bkskit1/XC350249.ogg'),\n    ('blhori1/XC417215.ogg', 'blhori1/XC417133.ogg'),\n    ('blhori1/XC743616.ogg', 'blhori1/XC537503.ogg'),\n    ('blrwar1/XC662286.ogg', 'blrwar1/XC662285.ogg'),\n    ('brakit1/XC743675.ogg', 'brakit1/XC537471.ogg'),\n    ('brcful1/XC197746.ogg', 'brcful1/XC157971.ogg'),\n    ('brnshr/XC510751.ogg', 'brnshr/XC510750.ogg'),\n    ('btbeat1/XC665307.ogg', 'btbeat1/XC513403.ogg'),\n    ('btbeat1/XC743618.ogg', 'btbeat1/XC683300.ogg'),\n    ('btbeat1/XC743619.ogg', 'btbeat1/XC683300.ogg'),\n    ('btbeat1/XC743619.ogg', 'btbeat1/XC743618.ogg'),\n    ('categr/XC787914.ogg', 'categr/XC438523.ogg'),\n    ('cohcuc1/XC253418.ogg', 'cohcuc1/XC241127.ogg'),\n    ('cohcuc1/XC423422.ogg', 'cohcuc1/XC423419.ogg'),\n    ('comgre/XC202776.ogg', 'comgre/XC192404.ogg'),\n    ('comgre/XC602468.ogg', 'comgre/XC175341.ogg'),\n    ('comgre/XC64628.ogg', 'comgre/XC58586.ogg'),\n    ('comior1/XC305930.ogg', 'comior1/XC303819.ogg'),\n    ('comkin1/XC207123.ogg', 'comior1/XC207062.ogg'),\n    ('comkin1/XC691421.ogg', 'comkin1/XC690633.ogg'),\n    ('commyn/XC577887.ogg', 'commyn/XC577886.ogg'),\n    ('commyn/XC652903.ogg', 'commyn/XC652901.ogg'),\n    ('compea/XC665320.ogg', 'compea/XC644022.ogg'),\n    ('comsan/XC385909.ogg', 'comsan/XC385908.ogg'),\n    ('comsan/XC643721.ogg', 'comsan/XC642698.ogg'),\n    ('comsan/XC667807.ogg', 'comsan/XC667806.ogg'),\n    ('comtai1/XC126749.ogg', 'comtai1/XC122978.ogg'),\n    ('comtai1/XC305210.ogg', 'comtai1/XC304811.ogg'),\n    ('comtai1/XC542375.ogg', 'comtai1/XC540351.ogg'),\n    ('comtai1/XC542379.ogg', 'comtai1/XC540352.ogg'),\n    ('crfbar1/XC615780.ogg', 'crfbar1/XC615778.ogg'),\n    ('dafbab1/XC188307.ogg', 'dafbab1/XC187059.ogg'),\n    ('dafbab1/XC188308.ogg', 'dafbab1/XC187068.ogg'),\n    ('dafbab1/XC188309.ogg', 'dafbab1/XC187069.ogg'),\n    ('dafbab1/XC197745.ogg', 'dafbab1/XC157972.ogg'),\n    ('eaywag1/XC527600.ogg', 'eaywag1/XC527598.ogg'),\n    ('eucdov/XC355153.ogg', 'eucdov/XC355152.ogg'),\n    ('eucdov/XC360303.ogg', 'eucdov/XC347428.ogg'),\n    ('eucdov/XC365606.ogg', 'eucdov/XC124694.ogg'),\n    ('eucdov/XC371039.ogg', 'eucdov/XC368596.ogg'),\n    ('eucdov/XC747422.ogg', 'eucdov/XC747408.ogg'),\n    ('eucdov/XC789608.ogg', 'eucdov/XC788267.ogg'),\n    ('goflea1/XC163901.ogg', 'bladro1/XC163901.ogg'),\n    ('goflea1/XC208794.ogg', 'bladro1/XC208794.ogg'),\n    ('goflea1/XC208795.ogg', 'bladro1/XC208795.ogg'),\n    ('goflea1/XC209203.ogg', 'bladro1/XC209203.ogg'),\n    ('goflea1/XC209549.ogg', 'bladro1/XC209549.ogg'),\n    ('goflea1/XC209564.ogg', 'bladro1/XC209564.ogg'),\n    ('graher1/XC357552.ogg', 'graher1/XC357551.ogg'),\n    ('graher1/XC590235.ogg', 'graher1/XC590144.ogg'),\n    ('grbeat1/XC304004.ogg', 'grbeat1/XC303999.ogg'),\n    ('grecou1/XC365426.ogg', 'grecou1/XC365425.ogg'),\n    ('greegr/XC247286.ogg', 'categr/XC197438.ogg'),\n    ('grewar3/XC743681.ogg', 'grewar3/XC537475.ogg'),\n    ('grnwar1/XC197744.ogg', 'grnwar1/XC157973.ogg'),\n    ('grtdro1/XC651708.ogg', 'grtdro1/XC613192.ogg'),\n    ('grywag/XC459760.ogg', 'grywag/XC457124.ogg'),\n    ('grywag/XC575903.ogg', 'grywag/XC575901.ogg'),\n    ('grywag/XC650696.ogg', 'grywag/XC592019.ogg'),\n    ('grywag/XC690448.ogg', 'grywag/XC655063.ogg'),\n    ('grywag/XC745653.ogg', 'grywag/XC745650.ogg'),\n    ('grywag/XC812496.ogg', 'grywag/XC812495.ogg'),\n    ('heswoo1/XC357155.ogg', 'heswoo1/XC357149.ogg'),\n    ('heswoo1/XC744698.ogg', 'heswoo1/XC665715.ogg'),\n    ('hoopoe/XC631301.ogg', 'hoopoe/XC365530.ogg'),\n    ('hoopoe/XC631304.ogg', 'hoopoe/XC252584.ogg'),\n    ('houcro1/XC744704.ogg', 'houcro1/XC683047.ogg'),\n    ('houspa/XC326675.ogg', 'houspa/XC326674.ogg'),\n    ('inbrob1/XC744708.ogg', 'inbrob1/XC744706.ogg'),\n    ('insowl1/XC305214.ogg', 'insowl1/XC301142.ogg'),\n    ('junbab2/XC282587.ogg', 'junbab2/XC282586.ogg'),\n    ('labcro1/XC267645.ogg', 'labcro1/XC265731.ogg'),\n    ('labcro1/XC345836.ogg', 'labcro1/XC312582.ogg'),\n    ('labcro1/XC37773.ogg', 'labcro1/XC19736.ogg'),\n    ('labcro1/XC447036.ogg', 'houcro1/XC447036.ogg'),\n    ('labcro1/XC823514.ogg', 'gybpri1/XC823527.ogg'),\n    ('laudov1/XC185511.ogg', 'grewar3/XC185505.ogg'),\n    ('laudov1/XC405375.ogg', 'laudov1/XC405374.ogg'),\n    ('laudov1/XC514027.ogg', 'eucdov/XC514027.ogg'),\n    ('lblwar1/XC197743.ogg', 'lblwar1/XC157974.ogg'),\n    ('lewduc1/XC261506.ogg', 'lewduc1/XC254813.ogg'),\n    ('litegr/XC403621.ogg', 'bcnher/XC403621.ogg'),\n    ('litegr/XC535540.ogg', 'litegr/XC448898.ogg'),\n    ('litegr/XC535552.ogg', 'litegr/XC447850.ogg'),\n    ('litgre1/XC630775.ogg', 'litgre1/XC630560.ogg'),\n    ('litgre1/XC776082.ogg', 'litgre1/XC663244.ogg'),\n    ('litspi1/XC674522.ogg', 'comtai1/XC674522.ogg'),\n    ('litspi1/XC722435.ogg', 'litspi1/XC721636.ogg'),\n    ('litspi1/XC722436.ogg', 'litspi1/XC721637.ogg'),\n    ('litswi1/XC443070.ogg', 'litswi1/XC440301.ogg'),\n    ('lobsun2/XC197742.ogg', 'lobsun2/XC157975.ogg'),\n    ('maghor2/XC197740.ogg', 'maghor2/XC157978.ogg'),\n    ('maghor2/XC786588.ogg', 'maghor2/XC786587.ogg'),\n    ('malpar1/XC197770.ogg', 'malpar1/XC157976.ogg'),\n    ('marsan/XC383290.ogg', 'marsan/XC383288.ogg'),\n    ('marsan/XC733175.ogg', 'marsan/XC716673.ogg'),\n    ('mawthr1/XC455222.ogg', 'mawthr1/XC455211.ogg'),\n    ('orihob2/XC557991.ogg', 'orihob2/XC557293.ogg'),\n    ('piebus1/XC165050.ogg', 'piebus1/XC122395.ogg'),\n    ('piebus1/XC814459.ogg', 'piebus1/XC792272.ogg'),\n    ('placuc3/XC490344.ogg', 'placuc3/XC486683.ogg'),\n    ('placuc3/XC572952.ogg', 'placuc3/XC572950.ogg'),\n    ('plaflo1/XC615781.ogg', 'plaflo1/XC614946.ogg'),\n    ('purher1/XC467373.ogg', 'graher1/XC467373.ogg'),\n    ('purher1/XC827209.ogg', 'purher1/XC827207.ogg'),\n    ('pursun3/XC268375.ogg', 'comtai1/XC241382.ogg'),\n    ('pursun4/XC514853.ogg', 'pursun4/XC514852.ogg'),\n    ('putbab1/XC574864.ogg', 'brcful1/XC574864.ogg'),\n    ('rewbul/XC306398.ogg', 'bkcbul1/XC306398.ogg'),\n    ('rewbul/XC713308.ogg', 'asbfly/XC713467.ogg'),\n    ('rewlap1/XC733007.ogg', 'rewlap1/XC732874.ogg'),\n    ('rorpar/XC199488.ogg', 'rorpar/XC199339.ogg'),\n    ('rorpar/XC402325.ogg', 'comior1/XC402326.ogg'),\n    ('rorpar/XC516404.ogg', 'rorpar/XC516402.ogg'),\n    ('sbeowl1/XC522123.ogg', 'brfowl1/XC522123.ogg'),\n    ('sohmyn1/XC744700.ogg', 'sohmyn1/XC743682.ogg'),\n    ('spepic1/XC804432.ogg', 'spepic1/XC804431.ogg'),\n    ('spodov/XC163930.ogg', 'bladro1/XC163901.ogg'),\n    ('spodov/XC163930.ogg', 'goflea1/XC163901.ogg'),\n    ('spoowl1/XC591485.ogg', 'spoowl1/XC591177.ogg'),\n    ('stbkin1/XC266782.ogg', 'stbkin1/XC266682.ogg'),\n    ('stbkin1/XC360661.ogg', 'stbkin1/XC199815.ogg'),\n    ('stbkin1/XC406140.ogg', 'stbkin1/XC406138.ogg'),\n    ('vefnut1/XC197738.ogg', 'vefnut1/XC157979.ogg'),\n    ('vefnut1/XC293526.ogg', 'vefnut1/XC289785.ogg'),\n    ('wemhar1/XC581045.ogg', 'comsan/XC581045.ogg'),\n    ('wemhar1/XC590355.ogg', 'wemhar1/XC590354.ogg'),\n    ('whbbul2/XC335671.ogg', 'whbbul2/XC335670.ogg'),\n    ('whbsho3/XC856465.ogg', 'whbsho3/XC856463.ogg'),\n    ('whbsho3/XC856468.ogg', 'whbsho3/XC856463.ogg'),\n    ('whbsho3/XC856468.ogg', 'whbsho3/XC856465.ogg'),\n    ('whbwat1/XC840073.ogg', 'whbwat1/XC840071.ogg'),\n    ('whbwoo2/XC239509.ogg', 'rufwoo2/XC239509.ogg'),\n    ('whcbar1/XC659329.ogg', 'insowl1/XC659329.ogg'),\n    ('whiter2/XC265271.ogg', 'whiter2/XC265267.ogg'),\n    ('whtkin2/XC197737.ogg', 'whtkin2/XC157981.ogg'),\n    ('whtkin2/XC430267.ogg', 'whtkin2/XC430256.ogg'),\n    ('whtkin2/XC503389.ogg', 'comior1/XC503389.ogg'),\n    ('whtkin2/XC540094.ogg', 'whtkin2/XC540087.ogg'),\n    ('woosan/XC184466.ogg', 'marsan/XC184466.ogg'),\n    ('woosan/XC545316.ogg', 'woosan/XC476064.ogg'),\n    ('woosan/XC587076.ogg', 'woosan/XC578599.ogg'),\n    ('woosan/XC742927.ogg', 'woosan/XC740798.ogg'),\n    ('woosan/XC825766.ogg', 'grnsan/XC825765.ogg'),\n    ('zitcis1/XC303866.ogg', 'zitcis1/XC302781.ogg'),\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:34.908666Z","iopub.execute_input":"2024-04-18T06:52:34.909033Z","iopub.status.idle":"2024-04-18T06:52:34.937661Z","shell.execute_reply.started":"2024-04-18T06:52:34.909004Z","shell.execute_reply":"2024-04-18T06:52:34.936282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dupes)","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:35.226771Z","iopub.execute_input":"2024-04-18T06:52:35.227167Z","iopub.status.idle":"2024-04-18T06:52:35.239895Z","shell.execute_reply.started":"2024-04-18T06:52:35.227131Z","shell.execute_reply":"2024-04-18T06:52:35.238683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nimport torch\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport torchaudio\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:35.823585Z","iopub.execute_input":"2024-04-18T06:52:35.823956Z","iopub.status.idle":"2024-04-18T06:52:41.458610Z","shell.execute_reply.started":"2024-04-18T06:52:35.823925Z","shell.execute_reply":"2024-04-18T06:52:41.457381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save to csv to make it easier to use downstream\npd.DataFrame([{\"filename_a\": a, \"filename_b\": b} for a, b in dupes]).to_csv(\"dupes.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:41.460746Z","iopub.execute_input":"2024-04-18T06:52:41.461333Z","iopub.status.idle":"2024-04-18T06:52:41.477643Z","shell.execute_reply.started":"2024-04-18T06:52:41.461271Z","shell.execute_reply":"2024-04-18T06:52:41.476222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/birdclef-2024/train_metadata.csv')\n\nTRAIN_PATH = Path('/kaggle/input/birdclef-2024/train_audio')\n\nSAMPLE_RATE = 32_000\nWINDOW = 5*SAMPLE_RATE","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:41.479228Z","iopub.execute_input":"2024-04-18T06:52:41.479663Z","iopub.status.idle":"2024-04-18T06:52:41.685005Z","shell.execute_reply.started":"2024-04-18T06:52:41.479624Z","shell.execute_reply":"2024-04-18T06:52:41.683792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helpers for displaying bird recordings for manual inspection\ncompute_melspec = torchaudio.transforms.MelSpectrogram(\n    sample_rate=SAMPLE_RATE,\n    n_mels=128,\n    n_fft=2048, \n    hop_length=512,\n)\n\npower_to_db = torchaudio.transforms.AmplitudeToDB(\n    stype=\"power\",\n    top_db=80.0,\n)\n\ndef show_audiofile(filename):\n    audio = torchaudio.load(TRAIN_PATH / filename)[0][0]\n    audio = torch.concat([audio[:WINDOW], audio[-WINDOW:]], -1)\n    plt.axis('off')\n    plt.title(filename)\n    plt.imshow(power_to_db(compute_melspec(audio)))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-18T06:52:41.689181Z","iopub.execute_input":"2024-04-18T06:52:41.689575Z","iopub.status.idle":"2024-04-18T06:52:41.795486Z","shell.execute_reply.started":"2024-04-18T06:52:41.689543Z","shell.execute_reply":"2024-04-18T06:52:41.794237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a, b in dupes:\n    print(a, b)\n    plt.figure(figsize=(12, 5))\n    plt.subplot(121)\n    show_audiofile(a)\n    plt.subplot(122)\n    show_audiofile(b)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-18T06:52:41.796966Z","iopub.execute_input":"2024-04-18T06:52:41.797359Z","iopub.status.idle":"2024-04-18T06:53:44.522946Z","shell.execute_reply.started":"2024-04-18T06:52:41.797323Z","shell.execute_reply":"2024-04-18T06:53:44.521967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}