{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8096443,"sourceType":"datasetVersion","datasetId":4780521},{"sourceId":8318251,"sourceType":"datasetVersion","datasetId":4940719},{"sourceId":8416940,"sourceType":"datasetVersion","datasetId":5010206},{"sourceId":8426650,"sourceType":"datasetVersion","datasetId":5017621},{"sourceId":174384054,"sourceType":"kernelVersion"},{"sourceId":174415292,"sourceType":"kernelVersion"},{"sourceId":175005679,"sourceType":"kernelVersion"},{"sourceId":175993701,"sourceType":"kernelVersion"},{"sourceId":179716078,"sourceType":"kernelVersion"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport random\nimport time\nimport warnings\nimport re\nimport copy\n\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\n\n\n\nfrom contextlib import contextmanager\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom glob import glob\n\nimport matplotlib.pyplot as plt\nimport librosa.display","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:49.796468Z","iopub.execute_input":"2024-05-31T11:31:49.796917Z","iopub.status.idle":"2024-05-31T11:31:51.549493Z","shell.execute_reply.started":"2024-05-31T11:31:49.796870Z","shell.execute_reply":"2024-05-31T11:31:51.548020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import Audio\nSR=32000\nless_than = 50","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:51.551807Z","iopub.execute_input":"2024-05-31T11:31:51.552318Z","iopub.status.idle":"2024-05-31T11:31:51.559326Z","shell.execute_reply.started":"2024-05-31T11:31:51.552284Z","shell.execute_reply":"2024-05-31T11:31:51.557496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=43):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    \nset_seed(43)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:51.561151Z","iopub.execute_input":"2024-05-31T11:31:51.561678Z","iopub.status.idle":"2024-05-31T11:31:51.576561Z","shell.execute_reply.started":"2024-05-31T11:31:51.561607Z","shell.execute_reply":"2024-05-31T11:31:51.575361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_paths = glob(\"/kaggle/input/1negative-samples-npy-birdclef2024/noise/*.npy\")\nprint(len(n_paths))\nNP=[]\nfor path in n_paths:\n    y = np.load(path)\n    length = len(y)/32000\n    if length==5.:\n#         print(path,len(y)/32000)\n        NP.append(path)\nprint(len(NP))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:51.579286Z","iopub.execute_input":"2024-05-31T11:31:51.580361Z","iopub.status.idle":"2024-05-31T11:31:52.845464Z","shell.execute_reply.started":"2024-05-31T11:31:51.580315Z","shell.execute_reply":"2024-05-31T11:31:52.844054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dur = pd.read_csv(\"/kaggle/input/2duration-log/bc2024_duration.csv\")\nprint(dur.shape)\ndur.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:52.846818Z","iopub.execute_input":"2024-05-31T11:31:52.847237Z","iopub.status.idle":"2024-05-31T11:31:53.136166Z","shell.execute_reply.started":"2024-05-31T11:31:52.847194Z","shell.execute_reply":"2024-05-31T11:31:53.134772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_audio_paths_birdcode(bird_code):\n    audio_paths = glob(f'/kaggle/input/1-5second-npy-birdclef2024/train_npy0/{bird_code}/*.npy')\\\n                    + glob(f'/kaggle/input/2-5second-npy-birdclef2024/train_npy1/{bird_code}/*.npy')\n    return audio_paths\n\ndef find_preds_paths_birdcode(bird_code):\n    return glob(f'/kaggle/input/2same-save-b0-google-preds/same_residue_preds/{bird_code}/*.npy')\n\ndef find_audio_paths_birdcode_filetags(bird_code,file_tag):\n    audio_paths = glob(f'/kaggle/input/1-5second-npy-birdclef2024/train_npy0/{bird_code}/{file_tag}.npy')\\\n                    + glob(f'/kaggle/input/2-5second-npy-birdclef2024/train_npy1/{bird_code}/{file_tag}.npy')\n    return audio_paths","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.137940Z","iopub.execute_input":"2024-05-31T11:31:53.138804Z","iopub.status.idle":"2024-05-31T11:31:53.146880Z","shell.execute_reply.started":"2024-05-31T11:31:53.138760Z","shell.execute_reply":"2024-05-31T11:31:53.145409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"txt_file = \"/kaggle/input/duplicates-calls-bc2024/duplicates_bird_calls.txt\"\nDL =[]\nwith open(txt_file,'r') as file:\n    lines = file.readlines()\n    for line in lines:\n        DL.append(line)\nfile.close()\nprint(len(DL))\nprint(DL[0])\n\ndiff_birds=[]\nsame_birds=[]\nfor item in DL:\n    a,b = item.split(\",\")[0],item.split(\",\")[1]\n    if a.split(\"/\")[0] != b.split(\"/\")[0]:\n        fla = a.split(\"/\")[1].split(\".\")[0]\n        flb = b.split(\"/\")[1].split(\".\")[0]\n        if fla!=flb:\n            diff_birds.append((a,b.strip()))\n    else:\n#         same_birds.append((a,b.strip()))\n        same_birds.append(a)\nprint(len(diff_birds),len(same_birds))\nprint(diff_birds[0],same_birds[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.148368Z","iopub.execute_input":"2024-05-31T11:31:53.148831Z","iopub.status.idle":"2024-05-31T11:31:53.170264Z","shell.execute_reply.started":"2024-05-31T11:31:53.148800Z","shell.execute_reply":"2024-05-31T11:31:53.168953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #remove the same bird calls from paths\n# val = len(all_paths)\n# print(len(all_paths))\n# temp_FL=[]\n# for n,filename in enumerate(same_birds):\n#     bird_name = filename.split(\"/\")[0]\n#     file_tag = filename.split(\"/\")[1].split(\".\")[0]\n#     path = find_audio_paths_birdcode_filetags(bird_name,file_tag)\n# #     print(n,path[0])\n#     if file_tag not in temp_FL:\n#         all_paths.remove(path[0])\n#     temp_FL.append(file_tag)\n    \n# print(len(all_paths))\n# print(\"diff:\", val - len(all_paths))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.171863Z","iopub.execute_input":"2024-05-31T11:31:53.172287Z","iopub.status.idle":"2024-05-31T11:31:53.178738Z","shell.execute_reply.started":"2024-05-31T11:31:53.172247Z","shell.execute_reply":"2024-05-31T11:31:53.177355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDER = \"/kaggle/input/birdclef-2024\"\ntrain_dir =\"/kaggle/input/birdclef-2024/train_audio\"\nAUDIO_DURATION=5.","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.180239Z","iopub.execute_input":"2024-05-31T11:31:53.180579Z","iopub.status.idle":"2024-05-31T11:31:53.190997Z","shell.execute_reply.started":"2024-05-31T11:31:53.180551Z","shell.execute_reply":"2024-05-31T11:31:53.189669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\n\n\ntrain = pd.read_csv(os.path.join(FOLDER,\"train_metadata.csv\"))\n\n\ntrain['new_target'] = train['primary_label'] + ' ' + train['secondary_labels'].map(lambda x: ' '.join(ast.literal_eval(x)))\n# train['len_new_target'] = train['new_target'].map(lambda x: len(x.split()))\n# train['len_new_target'].value_counts()\ntrain['file_tag'] = train['filename'].map(lambda x: x.split(\".\")[0].split(\"/\")[-1])\nprint(train.shape)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.196518Z","iopub.execute_input":"2024-05-31T11:31:53.197409Z","iopub.status.idle":"2024-05-31T11:31:53.674991Z","shell.execute_reply.started":"2024-05-31T11:31:53.197373Z","shell.execute_reply":"2024-05-31T11:31:53.673642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter=0\nfor n in range(len(train)):\n    pl = train.iloc[n]['primary_label']\n    bird_list = train.iloc[n]['new_target'].split()\n    temp_birdlist=[]\n    temp_birdlist.append(pl)\n    if len(bird_list)>1:\n        for idx in range(len(bird_list)):\n            if idx>0:\n                bird_name = bird_list[idx]\n                if bird_name!=pl:\n                    temp_birdlist.append(bird_name)\n        names = \" \".join(temp_birdlist)\n#         print(pl,names)\n        counter +=1\n        train.loc[n,'new_target']= names\nprint(counter)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:53.676749Z","iopub.execute_input":"2024-05-31T11:31:53.677172Z","iopub.status.idle":"2024-05-31T11:31:58.387810Z","shell.execute_reply.started":"2024-05-31T11:31:53.677137Z","shell.execute_reply":"2024-05-31T11:31:58.386779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a,b in diff_birds:\n    fla = a.split(\"/\")[1].split(\".\")[0]\n    flb = b.split(\"/\")[1].split(\".\")[0]\n    idx = train[train.file_tag==flb].index\n    train.at[idx.values[0],'file_tag']=fla","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.389351Z","iopub.execute_input":"2024-05-31T11:31:58.393634Z","iopub.status.idle":"2024-05-31T11:31:58.474340Z","shell.execute_reply.started":"2024-05-31T11:31:58.393565Z","shell.execute_reply":"2024-05-31T11:31:58.472538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique, counts = np.unique(train.file_tag, return_counts=True)\ncounts_dict = dict(zip(unique, counts))\nprint(len(counts_dict), len(train), len(train)-len(counts_dict))\n\ntemp = {k: v for k, v in sorted(counts_dict.items(), key=lambda item: item[1],reverse=True)}\ntemp = dict(list(temp.items())[:19])\ntags = temp.keys()\nprint(list(tags))\n\nfor tag in tags:\n    temp = list(train[train.file_tag==tag].new_target.values)\n    indxs = train[train.file_tag==tag].index\n#     print(temp)\n    names = \" \".join(temp)\n    for indx in indxs:\n        train.at[indx,'new_target']= names\n        \nlist(train[train.file_tag=='XC574864'].new_target.values)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.476009Z","iopub.execute_input":"2024-05-31T11:31:58.476471Z","iopub.status.idle":"2024-05-31T11:31:58.801356Z","shell.execute_reply.started":"2024-05-31T11:31:58.476427Z","shell.execute_reply":"2024-05-31T11:31:58.800108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = len(train)\nprint(len(train))\ntrain.drop_duplicates(subset='file_tag',inplace=True)\ntrain.reset_index(drop=True, inplace=True)\ntrain['len_new_target'] = train['new_target'].map(lambda x: len(x.split()))\nprint(len(train))\nprint(\"diff:\",val-len(train))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.803107Z","iopub.execute_input":"2024-05-31T11:31:58.803578Z","iopub.status.idle":"2024-05-31T11:31:58.849269Z","shell.execute_reply.started":"2024-05-31T11:31:58.803534Z","shell.execute_reply":"2024-05-31T11:31:58.848091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"secondary_labels = ['asfblu1','indwhe1','bltmun1','magrob','lotshr1','orhthr1']\nss = pd.read_csv(\"/kaggle/input/birdclef-2024/sample_submission.csv\")\nbird_target_names = list(ss.columns)\nbird_target_names.pop(0)\nlen(bird_target_names)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.851068Z","iopub.execute_input":"2024-05-31T11:31:58.851649Z","iopub.status.idle":"2024-05-31T11:31:58.873302Z","shell.execute_reply.started":"2024-05-31T11:31:58.851592Z","shell.execute_reply":"2024-05-31T11:31:58.872079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_names = list(pd.unique(train.common_name))\nlen(common_names)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.874569Z","iopub.execute_input":"2024-05-31T11:31:58.874946Z","iopub.status.idle":"2024-05-31T11:31:58.888352Z","shell.execute_reply.started":"2024-05-31T11:31:58.874916Z","shell.execute_reply":"2024-05-31T11:31:58.886943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax = pd.read_csv(\"/kaggle/input/birdclef-2024/eBird_Taxonomy_v2021.csv\")\nprint(tax.shape)\ntax.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:58.889885Z","iopub.execute_input":"2024-05-31T11:31:58.891292Z","iopub.status.idle":"2024-05-31T11:31:59.000715Z","shell.execute_reply.started":"2024-05-31T11:31:58.891246Z","shell.execute_reply":"2024-05-31T11:31:58.999507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SL= tax[tax.SPECIES_CODE.isin(secondary_labels)][['SPECIES_CODE','PRIMARY_COM_NAME']]\nSL.reset_index(drop=True, inplace=True)\nSL = SL.rename(columns={'SPECIES_CODE':'primary_label','PRIMARY_COM_NAME':'common_name'})\nSL","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:59.002212Z","iopub.execute_input":"2024-05-31T11:31:59.003088Z","iopub.status.idle":"2024-05-31T11:31:59.027477Z","shell.execute_reply.started":"2024-05-31T11:31:59.003043Z","shell.execute_reply":"2024-05-31T11:31:59.026130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = train.merge(dur[['filename','duration']])\nprint(final.shape)\nfinal.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:59.028967Z","iopub.execute_input":"2024-05-31T11:31:59.029367Z","iopub.status.idle":"2024-05-31T11:31:59.094909Z","shell.execute_reply.started":"2024-05-31T11:31:59.029335Z","shell.execute_reply":"2024-05-31T11:31:59.093608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_birdlist = train[['common_name','primary_label']]\ndf_birdlist.drop_duplicates(inplace=True)\ndf_birdlist.reset_index(drop=True, inplace=True)\ndf_birdlist","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:59.096544Z","iopub.execute_input":"2024-05-31T11:31:59.097117Z","iopub.status.idle":"2024-05-31T11:31:59.124078Z","shell.execute_reply.started":"2024-05-31T11:31:59.097074Z","shell.execute_reply":"2024-05-31T11:31:59.122904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_audio = {}\nfor species in bird_target_names:\n    num_audio_files = os.listdir(os.path.join(train_dir,species))\n#     print(species,len(num_audio_files))\n    num_audio[species]=len(num_audio_files)\n    \nnew_dict = {k: v for k, v in sorted(num_audio.items(), key=lambda item: item[1])}\nprint(len(new_dict))\n\nnum_audio = pd.DataFrame(new_dict.items(), columns=['primary_label', 'NUM_AUDIO_FILES',])\nnum_audio = num_audio.merge(df_birdlist,)\nprint(num_audio.shape)\n# num_audio = num_audio.rename(columns={'SPECIES_CODE':'primary_label'})\nnum_audio.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:31:59.125643Z","iopub.execute_input":"2024-05-31T11:31:59.126678Z","iopub.status.idle":"2024-05-31T11:32:01.767961Z","shell.execute_reply.started":"2024-05-31T11:31:59.126635Z","shell.execute_reply":"2024-05-31T11:32:01.766705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_duration = final.groupby('primary_label')['duration'].sum()\ngroup_dur = pd.merge(num_audio,pd.DataFrame(group_duration).reset_index())\nprint(group_dur.shape)\n\ngroup_dur['5_second_duration'] = np.round(group_dur['duration']/AUDIO_DURATION,0)\ngroup_dur['5_second_duration'].describe()\n\ngroup_dur.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:01.769549Z","iopub.execute_input":"2024-05-31T11:32:01.770587Z","iopub.status.idle":"2024-05-31T11:32:01.808723Z","shell.execute_reply.started":"2024-05-31T11:32:01.770545Z","shell.execute_reply":"2024-05-31T11:32:01.807699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### NOISE","metadata":{}},{"cell_type":"code","source":"n_paths = glob(\"/kaggle/input/1negative-samples-npy-birdclef2024/noise/*.npy\")\nprint(len(n_paths))\nNOISE_PATHS=[]\nfor path in n_paths:\n    y = np.load(path)\n    length = len(y)/32000\n    if length==5.:\n#         print(path,len(y)/32000)\n        NOISE_PATHS.append(path)\nprint(len(NOISE_PATHS))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:01.810282Z","iopub.execute_input":"2024-05-31T11:32:01.811037Z","iopub.status.idle":"2024-05-31T11:32:01.940211Z","shell.execute_reply.started":"2024-05-31T11:32:01.811001Z","shell.execute_reply":"2024-05-31T11:32:01.939021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird_matrix = np.zeros((len(bird_target_names),len(bird_target_names)),dtype=np.int16)\nbird_matrix2 = np.zeros((len(bird_target_names),len(bird_target_names)),dtype=np.int16)\nprint(bird_matrix.shape)\n\nfor n in tqdm(range(len(train))):\n    pl = train.iloc[n]['primary_label']\n    bird_list = train.iloc[n]['new_target'].split()\n    pl_indx= bird_target_names.index(pl)\n    bird_matrix[pl_indx,pl_indx] +=1\n    bird_matrix2[pl_indx,pl_indx] +=1\n    if len(bird_list)>1:\n        for idx in range(len(bird_list)):\n            if idx>0:\n                sl = bird_list[idx]\n                if sl!=pl:\n                    if sl not in secondary_labels:\n                        sl_indx = bird_target_names.index(sl)\n                        bird_matrix[pl_indx,sl_indx] +=1\n                        bird_matrix2[sl_indx,pl_indx] +=1\n                        bird_matrix2[pl_indx,sl_indx] +=1                        ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:01.941997Z","iopub.execute_input":"2024-05-31T11:32:01.942775Z","iopub.status.idle":"2024-05-31T11:32:06.964178Z","shell.execute_reply.started":"2024-05-31T11:32:01.942732Z","shell.execute_reply":"2024-05-31T11:32:06.962910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PL = final[final.len_new_target==1]\nPL.reset_index(drop=True, inplace=True)\nprint(PL.shape)\nPL.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:06.965881Z","iopub.execute_input":"2024-05-31T11:32:06.966338Z","iopub.status.idle":"2024-05-31T11:32:06.997552Z","shell.execute_reply.started":"2024-05-31T11:32:06.966307Z","shell.execute_reply":"2024-05-31T11:32:06.996260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### WATER BASED BIRDS","metadata":{}},{"cell_type":"code","source":"water = pd.read_csv(\"/kaggle/input/bc2024-water-tagged-bird-list/watertagged_bird_list2 - final_bird_list.csv\")\nwater.rename(columns={'Unnamed: 2': 'water_tag',},inplace=True)\nwater.fillna('n',inplace=True)\nprint(water.shape)\nwater.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:06.999532Z","iopub.execute_input":"2024-05-31T11:32:07.001008Z","iopub.status.idle":"2024-05-31T11:32:07.026779Z","shell.execute_reply.started":"2024-05-31T11:32:07.000955Z","shell.execute_reply":"2024-05-31T11:32:07.025874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"water_birds = list(water[water.water_tag=='w']['PRIMARY_COM_NAME'].values)\nprint(len(water_birds))\n\nwater_group = group_dur[group_dur.common_name.isin(water_birds)].merge(df_birdlist)\nwater_group.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.028159Z","iopub.execute_input":"2024-05-31T11:32:07.028888Z","iopub.status.idle":"2024-05-31T11:32:07.050402Z","shell.execute_reply.started":"2024-05-31T11:32:07.028851Z","shell.execute_reply":"2024-05-31T11:32:07.049516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird_groups = { 'low_elevation': ['Zitting Cisticola','Plain Prinia','Rufous Treepie','Small Minivet','Gray-headed Swamphen',\n                   'Asian Koel','Laughing Dove','Gray Francolin',],\n  'low-mid_elevation':['Paddyfield Pipit','Common Iora','White-throated Kingfisher','Spotted Owlet',\n                      'Painted Stork','Asian Openbill','Spotted Dove','Red Spurfowl'],\n 'unlikely':['houspa','Brahminy Kite','Eurasian Marsh-Harrier','Eurasian Collared-Dove',\n            'Rock Pigeon','Gray Francolin'],}","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.057783Z","iopub.execute_input":"2024-05-31T11:32:07.058845Z","iopub.status.idle":"2024-05-31T11:32:07.066023Z","shell.execute_reply.started":"2024-05-31T11:32:07.058804Z","shell.execute_reply":"2024-05-31T11:32:07.064047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### LOW ELEVATION","metadata":{}},{"cell_type":"code","source":"low_elevation_birds = list(pd.unique(PL[PL.common_name.isin(bird_groups['low_elevation'])]['common_name']))\nprint(len(low_elevation_birds))\nprint(\"total # audio files:\",group_dur[group_dur.common_name.isin(low_elevation_birds)]['NUM_AUDIO_FILES'].sum())\nprint(\"Possible total # audio files:\",group_dur[group_dur.common_name.\\\n                                                isin(low_elevation_birds)]['5_second_duration'].sum())","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.067739Z","iopub.execute_input":"2024-05-31T11:32:07.068097Z","iopub.status.idle":"2024-05-31T11:32:07.087581Z","shell.execute_reply.started":"2024-05-31T11:32:07.068069Z","shell.execute_reply":"2024-05-31T11:32:07.086297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### LOW-MID ELEVATION","metadata":{}},{"cell_type":"code","source":"low_mid_birds  = list(pd.unique(PL[PL.common_name.isin(bird_groups['low-mid_elevation'])]['common_name']))\nprint(len(low_mid_birds))\nprint(\"total # audio files:\",group_dur[group_dur.common_name.isin(low_mid_birds)]['NUM_AUDIO_FILES'].sum())\nprint(\"Possible total # audio files:\",group_dur[group_dur.common_name.\\\n                                                isin(low_mid_birds)]['5_second_duration'].sum())","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.089372Z","iopub.execute_input":"2024-05-31T11:32:07.089773Z","iopub.status.idle":"2024-05-31T11:32:07.111673Z","shell.execute_reply.started":"2024-05-31T11:32:07.089741Z","shell.execute_reply":"2024-05-31T11:32:07.110507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### UNLIKELY","metadata":{}},{"cell_type":"code","source":"unlikely_birds = list(pd.unique(PL[PL.common_name.isin(bird_groups['unlikely'])]['common_name']))\nprint(len(unlikely_birds))\nprint(\"total # audio files:\",group_dur[group_dur.common_name.isin(unlikely_birds)]['NUM_AUDIO_FILES'].sum())\nprint(\"Possible total # audio files:\",group_dur[group_dur.common_name.\\\n                                                isin(unlikely_birds)]['5_second_duration'].sum())","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.113286Z","iopub.execute_input":"2024-05-31T11:32:07.113757Z","iopub.status.idle":"2024-05-31T11:32:07.129513Z","shell.execute_reply.started":"2024-05-31T11:32:07.113720Z","shell.execute_reply":"2024-05-31T11:32:07.128288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eucdov - low\n# Eurasian Marsh-Harrier - open \n# rock pigeon - jog falls, humans\n# gray francolin - low grasslands,entry of naraikadu, low\n# brahminy kite - open, waterbody,","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.131460Z","iopub.execute_input":"2024-05-31T11:32:07.131910Z","iopub.status.idle":"2024-05-31T11:32:07.140706Z","shell.execute_reply.started":"2024-05-31T11:32:07.131865Z","shell.execute_reply":"2024-05-31T11:32:07.139423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### HIGH PRIORITY BIRDS","metadata":{}},{"cell_type":"code","source":"high_priority_birds = ['Gray Junglefowl', 'Malabar Whistling-Thrush', 'Malabar Barbet', 'White-cheeked Barbet',\n                       'Vernal Hanging-Parrot', 'Rufous Babbler', 'Southern Hill Myna', 'Dark-fronted Babbler', \n                       'Nilgiri Wood-Pigeon', 'Malabar Parakeet', 'Crimson-backed Sunbird', 'Orange Minivet',\n                       'White-bellied Blue Flycatcher', 'Malabar Woodshrike', 'Gray-fronted Green-Pigeon',\n                       'Nilgiri Flycatcher', 'Great Hornbill', 'Square-tailed Bulbul', 'Black-and-orange Flycatcher',\n                       'Nilgiri Flowerpecker', 'Yellow-browed Bulbul', 'Indian Yellow Tit', 'Malabar Trogon', \n                       'Jungle Myna', \"Loten's Sunbird\", 'Palani Laughingthrush', 'Common Flameback', 'White-bellied Woodpecker', \n                       'White-bellied Treepie', 'White-bellied Sholakili', 'Malabar Gray Hornbill', 'Wayanad Laughingthrush', \n                       'Flame-throated Bulbul',\n                        'Brown Wood-Owl','Spot-bellied Eagle-Owl','Brown Fish-Owl','Jungle Owlet','Brown Boobook',\n                       'Great Eared-Nightjar', 'Jungle Nightjar',\n                      ]\nprint(len(high_priority_birds))\nhpb = list(pd.unique(PL[PL.common_name.isin(high_priority_birds)]['primary_label']))\nprint(\"total # audio files:\",group_dur[group_dur.primary_label.isin(hpb)]['NUM_AUDIO_FILES'].sum())\nprint(\"possible total # audio files:\",group_dur[group_dur.primary_label.isin(hpb)]['5_second_duration'].sum())\nprint()\ngroup_dur[group_dur.primary_label.isin(hpb)].shape","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.142007Z","iopub.execute_input":"2024-05-31T11:32:07.142381Z","iopub.status.idle":"2024-05-31T11:32:07.177173Z","shell.execute_reply.started":"2024-05-31T11:32:07.142351Z","shell.execute_reply":"2024-05-31T11:32:07.175446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mid_priority_birds = ['Brown-capped Pygmy Woodpecker', 'Chestnut-headed Bee-eater','Crested Goshawk', 'Velvet-fronted Nuthatch', \n                      \"Jerdon's Bushlark\", 'Indian Scimitar-Babbler', 'Plum-headed Parakeet', \n                      'Rufous Woodpecker', 'Asian Emerald Dove', 'Golden-fronted Leafbird', 'Green Warbler', \n                      'Indian Blackbird', 'Heart-spotted Woodpecker', 'Little Spiderhunter', 'Rusty-tailed Flycatcher', \n                      'Red-whiskered Bulbul', 'White-browed Bulbul', 'Streak-throated Woodpecker', 'Stork-billed Kingfisher', \n                      'White-rumped Munia', 'Large-billed Leaf Warbler', 'Yellow-billed Babbler', 'Bar-winged Flycatcher-shrike', \n                      'Indian Blue Robin', \"Tickell's Leaf Warbler\"]\nprint(len(mid_priority_birds))\nmpb = list(pd.unique(PL[PL.common_name.isin(mid_priority_birds)]['primary_label']))\nprint(\"total # audio files:\",group_dur[group_dur.primary_label.isin(mpb)]['NUM_AUDIO_FILES'].sum())\nprint(\"possible total # audio files:\",group_dur[group_dur.primary_label.isin(mpb)]['5_second_duration'].sum())\nprint()\ngroup_dur[group_dur.primary_label.isin(mpb)].shape\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.179299Z","iopub.execute_input":"2024-05-31T11:32:07.179709Z","iopub.status.idle":"2024-05-31T11:32:07.202036Z","shell.execute_reply.started":"2024-05-31T11:32:07.179677Z","shell.execute_reply":"2024-05-31T11:32:07.200587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"low_priority_birds = [\"Forest Wagtail\",\"Greater Racket-tailed Drongo\",\"Gray-headed Canary-Flycatcher\",\n                     \"Indian Pitta\",\"Jungle Babbler\",\"Lesser Yellownape\",\"Pale-billed Flowerpecker\",\n                     \"Gray-bellied Cuckoo\",\"Purple-rumped Sunbird\",\"Thick-billed Warbler\",\n                     \"Tickell's Blue Flycatcher\",]\n\nprint(len(low_priority_birds))\nlpb = list(pd.unique(PL[PL.common_name.isin(low_priority_birds)]['primary_label']))\nprint(\"total # audio files:\",group_dur[group_dur.primary_label.isin(lpb)]['NUM_AUDIO_FILES'].sum())\nprint(\"possible total # audio files:\",group_dur[group_dur.primary_label.isin(lpb)]['5_second_duration'].sum())\nprint()\ngroup_dur[group_dur.primary_label.isin(lpb)]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.203951Z","iopub.execute_input":"2024-05-31T11:32:07.204425Z","iopub.status.idle":"2024-05-31T11:32:07.240976Z","shell.execute_reply.started":"2024-05-31T11:32:07.204392Z","shell.execute_reply":"2024-05-31T11:32:07.239984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NOCTURNAL BIRDS","metadata":{}},{"cell_type":"code","source":"nb = pd.read_csv(\"/kaggle/input/bc2024-nocturnal-dirunal-birds/Nocturnal_bird_list - final_bird_list.csv\")\nprint(nb.shape)\nnocturnal_birds = list(nb[nb['Dirunal/Nocturnal']=='n']['PRIMARY_COM_NAME'].values)\nnocturnal_birds.remove('Black-crowned Night-Heron')\nprint(nocturnal_birds)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.242753Z","iopub.execute_input":"2024-05-31T11:32:07.243743Z","iopub.status.idle":"2024-05-31T11:32:07.262875Z","shell.execute_reply.started":"2024-05-31T11:32:07.243707Z","shell.execute_reply":"2024-05-31T11:32:07.261413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird_namex = list(pd.unique(PL[PL.common_name.isin(nocturnal_birds)]['primary_label']))\nprint(len(bird_namex))\nprint(\"total # audio files:\",group_dur[group_dur.primary_label.isin(bird_namex)]['NUM_AUDIO_FILES'].sum())\nprint(\"possible total # audio files:\",group_dur[group_dur.primary_label.isin(bird_namex)]['5_second_duration'].sum())\nprint()\ngroup_dur[group_dur.primary_label.isin(bird_namex)]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.264521Z","iopub.execute_input":"2024-05-31T11:32:07.265138Z","iopub.status.idle":"2024-05-31T11:32:07.287902Z","shell.execute_reply.started":"2024-05-31T11:32:07.265105Z","shell.execute_reply":"2024-05-31T11:32:07.286991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"UNLIKELY_P = unlikely_birds+water_birds\nLOW_P      = list(set(low_priority_birds  + low_elevation_birds))\nMID_P      = list(set(mid_priority_birds  + low_mid_birds))\nHIGH_P     = high_priority_birds","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.289172Z","iopub.execute_input":"2024-05-31T11:32:07.289808Z","iopub.status.idle":"2024-05-31T11:32:07.300675Z","shell.execute_reply.started":"2024-05-31T11:32:07.289774Z","shell.execute_reply":"2024-05-31T11:32:07.299384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Low Priority Birds:\",len(LOW_P))\nprint(\"Mid Priority Birds:\",len(MID_P))\nprint(\"High Priority Birds:\",len(HIGH_P))\nprint(\"unlikely Birds:\", len(UNLIKELY_P))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:32:07.302003Z","iopub.execute_input":"2024-05-31T11:32:07.302535Z","iopub.status.idle":"2024-05-31T11:32:07.315845Z","shell.execute_reply.started":"2024-05-31T11:32:07.302504Z","shell.execute_reply":"2024-05-31T11:32:07.314687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"H = ['Brown-cheeked Fulvetta','Bronzed Drongo','Puff-throated Babbler',]\n\nM =['Asian Brown Flycatcher','Ashy Prinia','Black-rumped Flameback','Black Eagle','Black-hooded Oriole',\n   'Coppersmith Barbet','Crested Serpent-Eagle','Common Hawk-Cuckoo','Common Rosefinch',\n    'Green Bee-eater','Little Swift','Mountain Imperial-Pigeon','Oriental Honey-buzzard','Ashy Woodswallow',\n    'Indian Paradise-Flycatcher','Black-crowned Night-Heron','Greater Flameback',\n   ]\n\nL=['Asian Brown Flycatcher','Asian Palm-Swift','Black Drongo','Black Kite','Brown Shrike','Ashy Drongo',\n  'Blue-tailed Bee-eater','Common Myna','Common Tailorbird','Common Kingfisher',\n  'Indian Peafowl','Greater Coucal','Greenish Warbler','Gray Wagtail',\n  'Gray-breasted Prinia','Eurasian Hoopoe','House Crow','House Sparrow','Indian Robin''Indian Roller',\n  'Indian Golden Oriole','Large-billed Crow','Scaly-breasted Munia','Pied Bushchat','Purple Sunbird',\n   'Red-rumped Swallow','Red-vented Bulbul','Rose-ringed Parakeet','Rosy Starling','Shikra','White-browed Wagtail',\n   'Black-winged Kite','Black-naped Monarch',\"Blyth's Reed Warbler\",'Indian Robin','Indian Roller',\n  ]\n\nU = ['Barn Swallow','Brahminy Starling','Eurasian Moorhen','Western Yellow Wagtail',\n            'Eurasian Coot','Painted Stork','Speckled Piculet']\n\n\nprint('H:',len(H))\nprint('M:',len(M))\nprint('L:',len(L))\nprint(\"U:\",len(U))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:33:54.209903Z","iopub.execute_input":"2024-05-31T11:33:54.210367Z","iopub.status.idle":"2024-05-31T11:33:54.222606Z","shell.execute_reply.started":"2024-05-31T11:33:54.210334Z","shell.execute_reply":"2024-05-31T11:33:54.220707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LOW_PP  = list(set(LOW_P  + L))\nMID_PP  = list(set(MID_P  + M))\nHIGH_PP = list(set(HIGH_P + H))\nUNLIKELY_PP = list(set(water_birds+U))\n\nprint(\"Low Priority Birds:\",len(LOW_PP))\nprint(\"Mid Priority Birds:\",len(MID_PP))\nprint(\"High Priority Birds:\",len(HIGH_PP))\nprint(\"unlikely Birds:\", len(UNLIKELY_PP))\n\nprint()\nprint(\"Total #Birds:\",len(LOW_PP)+len(MID_PP)+len(HIGH_PP)+len(UNLIKELY_PP))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:02.353163Z","iopub.execute_input":"2024-05-31T11:34:02.353547Z","iopub.status.idle":"2024-05-31T11:34:02.364020Z","shell.execute_reply.started":"2024-05-31T11:34:02.353516Z","shell.execute_reply":"2024-05-31T11:34:02.362563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SAVE AUDIO","metadata":{}},{"cell_type":"code","source":"def normalize_equal_shapes(signal,SR=32000,USE_SEC=5):\n    signal = signal/ np.linalg.norm(signal)\n    if len(signal)>USE_SEC*SR:\n                signal = signal[:USE_SEC*SR]\n    else:\n        diff = SR*USE_SEC - len(signal)\n        signal = np.pad(signal, (0,diff), 'constant',)\n    return signal\n\ndef change_speed(data,speed_factor=0.8,use_sec=5,sr=32000):\n    #If rate > 1, then the signal is sped up. If rate < 1, then the signal is slowed down.\n    signal = librosa.effects.time_stretch(data, rate=speed_factor)\n    if len(signal)>USE_SEC*SR:\n        signal = signal[:USE_SEC*SR]\n    else:\n        diff = SR*USE_SEC - len(signal)\n        signal = np.pad(signal, (0,diff), 'constant',)\n    return signal\n\ndef pitch_shift(data, sr=32000, pitch_factor=1.8):\n    #how many (fractional) steps to shift y\n    return librosa.effects.pitch_shift(data, sr=sr, n_steps=pitch_factor)\n\ndef audio_shift(data, sampling_rate=32000, shift_max=1, shift_direction='right'):\n    shift = np.random.randint(sampling_rate * shift_max)\n    if shift_direction == 'right':\n        shift = -shift\n    elif shift_direction == 'both':\n        direction = np.random.randint(0, 2)\n        if direction == 1:\n            shift = -shift\n    augmented_data = np.roll(data, shift)\n    # Set to silence for heading/ tailing\n    if shift > 0:\n        augmented_data[:shift] = 0\n    else:\n        augmented_data[shift:] = 0\n    return augmented_data\n\ndef noise_injection(data, noise_factor=0.05):\n    \n    noise_path = np.random.choice(NP)\n    noise = np.load(noise_path)\n    noise = noise[:len(data)]\n    augmented_data = data + noise_factor * noise\n    # Cast back to same data type\n    augmented_data = augmented_data.astype(data.dtype)\n    return augmented_data\n\ndef normalized_custom_noise_injection(data, noise_factor=2):\n    \n    noise_path = np.random.choice(NP)\n    noise = np.load(noise_path)\n    \n    noise = normalize_equal_shapes(noise,)\n    data = normalize_equal_shapes(data,)\n    augmented_data = data + noise_factor * noise\n    # Cast back to same data type\n    augmented_data =augmented_data.astype(data.dtype)\n    return augmented_data\n        \n\n# This code runs only in python 3.10 or above versions\ndef apply_augmentations(rand,signal):\n    match rand:\n        case 0:\n            return change_speed(signal,speed_factor=random.choice([0.4,0.6,0.8,1.0,1.2,1.6,1.8]))\n        case 1:\n            return pitch_shift(signal, sr=32000, pitch_factor=random.choice([0.5,1.5,2.0,2.5,3.0,3.5,4.0]))\n        case 2:\n            return audio_shift(signal, sampling_rate=32000, shift_max=random.choice([0.2,0.4,0.5,0.6,0.8,1.0]), \n                               shift_direction=random.choice(['right','both']))\n        case 3:\n            return normalized_custom_noise_injection(signal, noise_factor=random.choice([2.,3.,4.]))\n        case default:\n            return signal","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:07.134717Z","iopub.execute_input":"2024-05-31T11:34:07.135117Z","iopub.status.idle":"2024-05-31T11:34:07.157133Z","shell.execute_reply.started":"2024-05-31T11:34:07.135088Z","shell.execute_reply":"2024-05-31T11:34:07.155895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SR = 32000\n # 60 # 90 # 60\nCAP_MIX = 10\nOUT_DIR = \"/kaggle/working/\"\nMIN_DUR = 0.47 \nUSE_SEC=5","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:07.336556Z","iopub.execute_input":"2024-05-31T11:34:07.337019Z","iopub.status.idle":"2024-05-31T11:34:07.342346Z","shell.execute_reply.started":"2024-05-31T11:34:07.336987Z","shell.execute_reply":"2024-05-31T11:34:07.341148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Low Priority Birds:\",len(LOW_P))\nprint(\"Mid Priority Birds:\",len(MID_P))\nprint(\"High Priority Birds:\",len(HIGH_P))\nprint(\"unlikely Birds:\", len(UNLIKELY_P))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:08.822189Z","iopub.execute_input":"2024-05-31T11:34:08.822649Z","iopub.status.idle":"2024-05-31T11:34:08.830648Z","shell.execute_reply.started":"2024-05-31T11:34:08.822597Z","shell.execute_reply":"2024-05-31T11:34:08.828734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREDS_FOLDER = \"/kaggle/input/2same-save-b0-google-preds/same_residue_preds/\"\npreds_paths = glob(\"/kaggle/input/2same-save-b0-google-preds/same_residue_preds/*/*.npy\")\nprint(len(preds_paths))\npreds_paths[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:10.093150Z","iopub.execute_input":"2024-05-31T11:34:10.093659Z","iopub.status.idle":"2024-05-31T11:34:10.174220Z","shell.execute_reply.started":"2024-05-31T11:34:10.093606Z","shell.execute_reply":"2024-05-31T11:34:10.172841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:34:11.199941Z","iopub.execute_input":"2024-05-31T11:34:11.201032Z","iopub.status.idle":"2024-05-31T11:34:11.208997Z","shell.execute_reply.started":"2024-05-31T11:34:11.200975Z","shell.execute_reply":"2024-05-31T11:34:11.207598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CURB_AUDIO=25","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:35:22.858510Z","iopub.execute_input":"2024-05-31T11:35:22.859283Z","iopub.status.idle":"2024-05-31T11:35:22.864896Z","shell.execute_reply.started":"2024-05-31T11:35:22.859243Z","shell.execute_reply":"2024-05-31T11:35:22.863674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio_counter=0\ncounter=0\n# ignore_list = ['putbab1','whcbar1']\nignore_list=[]\nhigh_counter,mid_counter,low_counter,rest_counter=0,0,0,0\nPREDS_BIRDS=[]\nfor bird_code in os.listdir(PREDS_FOLDER):\n    bird_name = df_birdlist[df_birdlist.primary_label==bird_code]['common_name'].values[0]\n    audio_paths = os.listdir(os.path.join(PREDS_FOLDER,bird_code))\n    default_audio_paths = os.listdir(os.path.join(train_dir,bird_code))\n    if len(audio_paths)<=100:\n        if bird_code not in ignore_list:\n            counter+=1\n            ptag = \"\"\n            if bird_name in HIGH_PP:\n                ptag=\"HIGH\"\n                high_counter+=1\n                PREDS_BIRDS.append(bird_name) \n            elif bird_name in MID_PP:\n                ptag=\"MID\"\n                mid_counter+=1\n                PREDS_BIRDS.append(bird_name) \n            elif bird_name in LOW_PP:\n                ptag=\"LOW\"\n                low_counter+=1\n                PREDS_BIRDS.append(bird_name) \n            else:\n                ptag=\"None\"\n                rest_counter+=1\n                \n            print(ptag,bird_code,bird_name,len(default_audio_paths),len(audio_paths))\n            audio_counter +=len(audio_paths)\nprint()\nprint(\"total # of birds:\",counter)\nprint()\nprint(\"# of High Birds:\",high_counter)\nprint(\"# of Mid Birds:\",mid_counter)\nprint(\"# of Low Birds:\",low_counter)\nprint(\"# of Rest Birds:\",rest_counter)\nprint()\nprint(\"total # of audio paths:\",audio_counter)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:37:08.524855Z","iopub.execute_input":"2024-05-31T11:37:08.525323Z","iopub.status.idle":"2024-05-31T11:37:08.639962Z","shell.execute_reply.started":"2024-05-31T11:37:08.525289Z","shell.execute_reply":"2024-05-31T11:37:08.638689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(PREDS_BIRDS))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:38:03.354805Z","iopub.execute_input":"2024-05-31T11:38:03.355197Z","iopub.status.idle":"2024-05-31T11:38:03.360851Z","shell.execute_reply.started":"2024-05-31T11:38:03.355168Z","shell.execute_reply":"2024-05-31T11:38:03.359575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Low Priority Birds:\",len(LOW_PP))\nprint(\"Mid Priority Birds:\",len(MID_PP))\nprint(\"High Priority Birds:\",len(HIGH_PP))\nprint(\"unlikely Birds:\", len(UNLIKELY_PP))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:38:11.796643Z","iopub.execute_input":"2024-05-31T11:38:11.797070Z","iopub.status.idle":"2024-05-31T11:38:11.804839Z","shell.execute_reply.started":"2024-05-31T11:38:11.797040Z","shell.execute_reply":"2024-05-31T11:38:11.803429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def SAVE_MIX(FILE1,FILE2,LABEL_LIST):\n    USE_RATING=3.\n    USE_SEC=5\n    SR=32000\n    MIN_SEC=0.47\n\n    NEW_TARGETS=[]\n    NEW_PATHS=[]\n    MIXED_TAGS=[]\n    FILE_LESS,FILE_MORE=[],[]\n\n    for n in tqdm(range(len(FILE1))):\n        filea,fileb = FILE1[n],FILE2[n]\n\n\n        filea_tag = filea.split(\"/\")[-1].split(\".\")[0]\n        fileb_tag = fileb.split(\"/\")[-1].split(\".\")[0]\n\n        filea_signal = np.load(filea)\n        fileb_signal = np.load(fileb)\n\n        if len(filea_signal)>=int(MIN_SEC*SR) or len(fileb_signal)>int(MIN_SEC*SR): #MINSEC=1\n\n            FILE_LESS.append(filea)\n            FILE_MORE.append(fileb)\n\n            #FILEA\n            if len(filea_signal)>USE_SEC*SR:\n                    filea_signal = filea_signal[:USE_SEC*SR]\n            else:\n                diff = SR*USE_SEC - len(filea_signal)\n                filea_signal = np.pad(filea_signal, (0,diff), 'constant',)\n\n            #FILEB USE_STAGE\n    #         start,stop = stage*SR*USE_SEC,(stage+1)*SR*USE_SEC\n    #         fileb_signal = fileb_signal[start:stop]\n            if len(fileb_signal)>USE_SEC*SR:\n                    fileb_signal = fileb_signal[:USE_SEC*SR]\n            else:\n                diff = SR*USE_SEC - len(fileb_signal)\n                fileb_signal = np.pad(fileb_signal, (0,diff), 'constant',)\n\n\n            bird_target = filea.split(\"/\")[-2]\n            mixed_file_tag = LABEL_LIST[n][0]+\"_\"+LABEL_LIST[n][1]+\"_\"+filea_tag + \"_\" + fileb_tag + \"_\" + str(n)\n\n            rand = np.random.randint(4)\n\n            #FILE B\n            fileb_signal = apply_augmentations(rand,fileb_signal)\n\n            with np.errstate(invalid='raise'):\n                try:\n                    mixed_signal = (filea_signal / np.linalg.norm(filea_signal))\\\n                                 + (fileb_signal / np.linalg.norm(fileb_signal))\n                    save_path = os.path.join(save_folder_path,mixed_file_tag)\n\n                    np.save(save_path, mixed_signal)\n                except FloatingPointError:\n                    print('Error: Division by Zero')\n                    print('bird name:',bird_name)\n            ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:38:14.527413Z","iopub.execute_input":"2024-05-31T11:38:14.527803Z","iopub.status.idle":"2024-05-31T11:38:14.544946Z","shell.execute_reply.started":"2024-05-31T11:38:14.527774Z","shell.execute_reply":"2024-05-31T11:38:14.543595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preds - Nocturnal Priority","metadata":{}},{"cell_type":"code","source":"CAP_MIX = 25\noutput_dir = f'preds_nocturnal/'\n\nsave_folder_path = output_dir\n_ = os.makedirs(save_folder_path, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:39:39.317223Z","iopub.execute_input":"2024-05-31T11:39:39.317937Z","iopub.status.idle":"2024-05-31T11:39:39.326574Z","shell.execute_reply.started":"2024-05-31T11:39:39.317893Z","shell.execute_reply":"2024-05-31T11:39:39.324710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_RATING=3.\nUSE_SEC=5\nSR=32000\nMIN_SEC=3\n\n# CURB_NUM_AUDIO = 100\nMAX_STAGE=6\nCOUNTER=0\nFILE1,FILE2 =[],[]\nLABEL_LIST=[]\n\nNUM_PER_COMBINATION =1\n\nTOTAL_FILE_COUNTER = 0\n# bird_dict1 = {bird_name:0 for bird_name in less_birds}\n# audio_count_dic1t = {bird_name:0 for bird_name in less_birds}\n# audio_count_dict2 = {bird_name:[] for bird_name in more_birds}\nFILES_USED = []\n\nfor bird_name1 in tqdm(PREDS_BIRDS):\n    bird_code1 = df_birdlist[df_birdlist.common_name==bird_name1]['primary_label'].values[0]\n    r = re.compile(f\".*{bird_code1}\")\n    audio_paths1 = find_preds_paths_birdcode(bird_code1)\n    PATHS_USED1=[]\n    PATHS_USED2 = []\n    BIRDS_USED2=[]\n    file_counter = 0\n    for _ in range(CAP_MIX):\n        path1 = random.choice(audio_paths1)\n        if path1 not in PATHS_USED1:\n            PATHS_USED1.append(path1)\n            bird_name2 = random.choice(nocturnal_birds)\n            if bird_name2 not in BIRDS_USED2:\n                BIRDS_USED2.append(bird_name2)\n                bird_code2 = df_birdlist[df_birdlist.common_name==bird_name2]['primary_label'].values[0]\n                audio_paths2 = find_audio_paths_birdcode(bird_code2)\n                path2 = random.choice(audio_paths2)\n                if path2 not in PATHS_USED2:\n                    PATHS_USED2.append(path2)\n                    FILE1.append(path1)\n                    FILE2.append(path2)\n                    LABEL_LIST.append([bird_code1,bird_code2])\n                    TOTAL_FILE_COUNTER+=1\n                    file_counter +=1\n                else:\n                    continue\n                    \n            else:\n                continue\n        \n        else:\n            continue\n    print(bird_name1,len(audio_paths1),file_counter)\n                \nprint(\"TOTAL_FILE_COUNTER:\",TOTAL_FILE_COUNTER )","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:39:39.329187Z","iopub.execute_input":"2024-05-31T11:39:39.329742Z","iopub.status.idle":"2024-05-31T11:39:39.676951Z","shell.execute_reply.started":"2024-05-31T11:39:39.329695Z","shell.execute_reply":"2024-05-31T11:39:39.675813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(FILE1),len(FILE2),len(LABEL_LIST)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:39:43.162463Z","iopub.execute_input":"2024-05-31T11:39:43.162882Z","iopub.status.idle":"2024-05-31T11:39:43.170522Z","shell.execute_reply.started":"2024-05-31T11:39:43.162850Z","shell.execute_reply":"2024-05-31T11:39:43.169183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FILE1[0],FILE2[0],LABEL_LIST[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:39:43.413625Z","iopub.execute_input":"2024-05-31T11:39:43.414003Z","iopub.status.idle":"2024-05-31T11:39:43.422761Z","shell.execute_reply.started":"2024-05-31T11:39:43.413973Z","shell.execute_reply":"2024-05-31T11:39:43.421308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVE_MIX(FILE1,FILE2,LABEL_LIST,)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:39:45.778555Z","iopub.execute_input":"2024-05-31T11:39:45.779312Z","iopub.status.idle":"2024-05-31T11:40:06.230113Z","shell.execute_reply.started":"2024-05-31T11:39:45.779276Z","shell.execute_reply":"2024-05-31T11:40:06.226058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mix_paths = glob('/kaggle/working/*/*.npy')\n\nprint(mix_paths[0])\nprint(\"total selfmix audio files:\", len(mix_paths))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:40:06.232062Z","iopub.execute_input":"2024-05-31T11:40:06.232510Z","iopub.status.idle":"2024-05-31T11:40:06.253482Z","shell.execute_reply.started":"2024-05-31T11:40:06.232451Z","shell.execute_reply":"2024-05-31T11:40:06.251910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counter=0\nfor path in tqdm(mix_paths):\n    x = np.load(path)\n    if np.isnan(x).any():\n        counter +=1\ncounter","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:40:19.418081Z","iopub.execute_input":"2024-05-31T11:40:19.418558Z","iopub.status.idle":"2024-05-31T11:40:19.620866Z","shell.execute_reply.started":"2024-05-31T11:40:19.418523Z","shell.execute_reply":"2024-05-31T11:40:19.619971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_df = pd.DataFrame(mix_paths, columns=['file_path'])\npath_df['new_target'] = path_df['file_path'].map(lambda x: x.split(\"/\")[-1].split(\"_\")[0])\nprint(path_df.shape)\npath_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:40:19.622810Z","iopub.execute_input":"2024-05-31T11:40:19.623186Z","iopub.status.idle":"2024-05-31T11:40:19.639158Z","shell.execute_reply.started":"2024-05-31T11:40:19.623149Z","shell.execute_reply":"2024-05-31T11:40:19.637921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for bird_name in list(df_birdlist.common_name.values):\n    bird_code = df_birdlist[df_birdlist.common_name==bird_name]['primary_label'].values[0]\n    num_paths = len(os.listdir(f\"/kaggle/input/birdclef-2024/train_audio/{bird_code}\"))\n    num_files = len(path_df[path_df.new_target==bird_code])\n    if num_files !=0:\n#         if num_files>100:\n        print(bird_name,num_paths,num_files)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:40:21.494190Z","iopub.execute_input":"2024-05-31T11:40:21.494565Z","iopub.status.idle":"2024-05-31T11:40:21.815253Z","shell.execute_reply.started":"2024-05-31T11:40:21.494538Z","shell.execute_reply":"2024-05-31T11:40:21.814057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}