{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport cv2\nimport folium\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nfrom folium.plugins import HeatMap\nfrom folium.features import DivIcon\nfrom IPython.display import Audio, display","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-10T23:17:17.816721Z","iopub.execute_input":"2023-03-10T23:17:17.817987Z","iopub.status.idle":"2023-03-10T23:17:17.825059Z","shell.execute_reply.started":"2023-03-10T23:17:17.817935Z","shell.execute_reply":"2023-03-10T23:17:17.823594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class color:\n   PURPLE = '\\033[95m'\n   CYAN = '\\033[96m'\n   DARKCYAN = '\\033[36m'\n   BLUE = '\\033[94m'\n   GREEN = '\\033[92m'\n   YELLOW = '\\033[93m'\n   RED = '\\033[91m'\n   BOLD = '\\033[1m'\n   UNDERLINE = '\\033[4m'\n   END = '\\033[0m'","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:17:17.827820Z","iopub.execute_input":"2023-03-10T23:17:17.829322Z","iopub.status.idle":"2023-03-10T23:17:17.847079Z","shell.execute_reply.started":"2023-03-10T23:17:17.829274Z","shell.execute_reply":"2023-03-10T23:17:17.845175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_audio(\n    dir_path: str, label: str, example: str\n) -> None:\n    \n    if label == \"\":\n        filename = f\"{dir_path}/{example}.ogg\"\n        label = \"None\"\n    else:\n        filename = f\"{dir_path}/{label}/{example}.ogg\"\n    \n    print(f\"\\nLabel - {color.BOLD}{color.PURPLE}{label}{color.END}, example - {example}:\")\n    display(Audio(filename=filename))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:17:17.849413Z","iopub.execute_input":"2023-03-10T23:17:17.850565Z","iopub.status.idle":"2023-03-10T23:17:17.858816Z","shell.execute_reply.started":"2023-03-10T23:17:17.850329Z","shell.execute_reply":"2023-03-10T23:17:17.857898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir(\"/kaggle/input/birdclef-2023/train_audio\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:17:17.860530Z","iopub.execute_input":"2023-03-10T23:17:17.861442Z","iopub.status.idle":"2023-03-10T23:17:17.903224Z","shell.execute_reply.started":"2023-03-10T23:17:17.861393Z","shell.execute_reply":"2023-03-10T23:17:17.901827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_audio(\"/kaggle/input/birdclef-2023/train_audio\", \"abethr1\", \"XC128013\")\ndisplay_audio(\"/kaggle/input/birdclef-2023/train_audio\", \"abhori1\", \"XC120250\")\ndisplay_audio(\"/kaggle/input/birdclef-2023/train_audio\", \"abythr1\", \"XC115981\")\ndisplay_audio(\"/kaggle/input/birdclef-2023/train_audio\", \"afbfly1\", \"XC200995\")\ndisplay_audio(\"/kaggle/input/birdclef-2023/train_audio\", \"afdfly1\", \"XC115969\")","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:19:08.472600Z","iopub.execute_input":"2023-03-10T23:19:08.473718Z","iopub.status.idle":"2023-03-10T23:19:08.706024Z","shell.execute_reply.started":"2023-03-10T23:19:08.473647Z","shell.execute_reply":"2023-03-10T23:19:08.704155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/birdclef-2023/test_soundscapes","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:19:34.669333Z","iopub.execute_input":"2023-03-10T23:19:34.669813Z","iopub.status.idle":"2023-03-10T23:19:35.826242Z","shell.execute_reply.started":"2023-03-10T23:19:34.669773Z","shell.execute_reply":"2023-03-10T23:19:35.824692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh /kaggle/input/birdclef-2023/test_soundscapes/soundscape_29201.ogg","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:19:44.293406Z","iopub.execute_input":"2023-03-10T23:19:44.293877Z","iopub.status.idle":"2023-03-10T23:19:45.401142Z","shell.execute_reply.started":"2023-03-10T23:19:44.293837Z","shell.execute_reply":"2023-03-10T23:19:45.399493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_audio(\"/kaggle/input/birdclef-2023/test_soundscapes\", \"\", \"soundscape_29201\")","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:19:55.818642Z","iopub.execute_input":"2023-03-10T23:19:55.819341Z","iopub.status.idle":"2023-03-10T23:19:56.106064Z","shell.execute_reply.started":"2023-03-10T23:19:55.819298Z","shell.execute_reply":"2023-03-10T23:19:56.104333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata_df = pd.read_csv(\"/kaggle/input/birdclef-2023/train_metadata.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:20:09.583718Z","iopub.execute_input":"2023-03-10T23:20:09.584165Z","iopub.status.idle":"2023-03-10T23:20:09.724885Z","shell.execute_reply.started":"2023-03-10T23:20:09.584098Z","shell.execute_reply":"2023-03-10T23:20:09.723352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:20:17.219078Z","iopub.execute_input":"2023-03-10T23:20:17.219714Z","iopub.status.idle":"2023-03-10T23:20:17.263946Z","shell.execute_reply.started":"2023-03-10T23:20:17.219668Z","shell.execute_reply":"2023-03-10T23:20:17.262457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Examples count:\", len(train_metadata_df))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:20:27.679791Z","iopub.execute_input":"2023-03-10T23:20:27.680459Z","iopub.status.idle":"2023-03-10T23:20:27.689487Z","shell.execute_reply.started":"2023-03-10T23:20:27.680402Z","shell.execute_reply":"2023-03-10T23:20:27.687144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:25:26.385832Z","iopub.execute_input":"2023-03-10T23:25:26.386345Z","iopub.status.idle":"2023-03-10T23:25:26.426682Z","shell.execute_reply.started":"2023-03-10T23:25:26.386301Z","shell.execute_reply":"2023-03-10T23:25:26.425808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:25:37.081585Z","iopub.execute_input":"2023-03-10T23:25:37.082184Z","iopub.status.idle":"2023-03-10T23:25:37.103790Z","shell.execute_reply.started":"2023-03-10T23:25:37.082135Z","shell.execute_reply":"2023-03-10T23:25:37.102188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primary_label_counts = train_metadata_df.primary_label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:00.479345Z","iopub.execute_input":"2023-03-10T23:26:00.480740Z","iopub.status.idle":"2023-03-10T23:26:00.488768Z","shell.execute_reply.started":"2023-03-10T23:26:00.480688Z","shell.execute_reply":"2023-03-10T23:26:00.487173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Primary labels count:\", len(primary_label_counts.index))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:02.204142Z","iopub.execute_input":"2023-03-10T23:26:02.204578Z","iopub.status.idle":"2023-03-10T23:26:02.213019Z","shell.execute_reply.started":"2023-03-10T23:26:02.204540Z","shell.execute_reply":"2023-03-10T23:26:02.211150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primary_label_counts = train_metadata_df.primary_label.value_counts()\n\nfig = px.bar(x=primary_label_counts.index, y=primary_label_counts.values)\nfig.update_layout(xaxis_title=\"Label\", yaxis_title=\"Count\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:12.400139Z","iopub.execute_input":"2023-03-10T23:26:12.400941Z","iopub.status.idle":"2023-03-10T23:26:14.415379Z","shell.execute_reply.started":"2023-03-10T23:26:12.400899Z","shell.execute_reply":"2023-03-10T23:26:14.414204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"All secondary column occurrences:\", sum(train_metadata_df.secondary_labels != \"[]\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:25.510503Z","iopub.execute_input":"2023-03-10T23:26:25.510932Z","iopub.status.idle":"2023-03-10T23:26:25.522443Z","shell.execute_reply.started":"2023-03-10T23:26:25.510893Z","shell.execute_reply":"2023-03-10T23:26:25.520673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_secondary_labels = sum([eval(x) for x in train_metadata_df.secondary_labels], [])\nall_secondary_labels_counts = pd.value_counts(all_secondary_labels)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:33.897904Z","iopub.execute_input":"2023-03-10T23:26:33.898499Z","iopub.status.idle":"2023-03-10T23:26:34.167038Z","shell.execute_reply.started":"2023-03-10T23:26:33.898454Z","shell.execute_reply":"2023-03-10T23:26:34.165530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(x=all_secondary_labels_counts.index, y=all_secondary_labels_counts.values)\nfig.update_layout(xaxis_title=\"Secondary Label\", yaxis_title=\"Count\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:39.989218Z","iopub.execute_input":"2023-03-10T23:26:39.989661Z","iopub.status.idle":"2023-03-10T23:26:40.055326Z","shell.execute_reply.started":"2023-03-10T23:26:39.989623Z","shell.execute_reply":"2023-03-10T23:26:40.053835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Type column occurrences:\", sum(train_metadata_df.type != \"[]\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:49.719842Z","iopub.execute_input":"2023-03-10T23:26:49.720270Z","iopub.status.idle":"2023-03-10T23:26:49.730743Z","shell.execute_reply.started":"2023-03-10T23:26:49.720232Z","shell.execute_reply":"2023-03-10T23:26:49.728991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_labels = sum([eval(x) for x in train_metadata_df.type], [])\ntype_counts = pd.value_counts(type_labels)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:26:56.032562Z","iopub.execute_input":"2023-03-10T23:26:56.032956Z","iopub.status.idle":"2023-03-10T23:26:57.551978Z","shell.execute_reply.started":"2023-03-10T23:26:56.032922Z","shell.execute_reply":"2023-03-10T23:26:57.550546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(x=type_counts.index, y=type_counts.values)\nfig.update_layout(xaxis_title=\"Audio type\", yaxis_title=\"Count\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:27:09.620720Z","iopub.execute_input":"2023-03-10T23:27:09.621275Z","iopub.status.idle":"2023-03-10T23:27:09.691811Z","shell.execute_reply.started":"2023-03-10T23:27:09.621230Z","shell.execute_reply":"2023-03-10T23:27:09.690416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scientific_name_counts = train_metadata_df.scientific_name.value_counts()\n\nfig = px.bar(x=scientific_name_counts.index, y=scientific_name_counts.values)\nfig.update_layout(xaxis_title=\"Scientific name\", yaxis_title=\"Count\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:27:29.168770Z","iopub.execute_input":"2023-03-10T23:27:29.170882Z","iopub.status.idle":"2023-03-10T23:27:29.253234Z","shell.execute_reply.started":"2023-03-10T23:27:29.170779Z","shell.execute_reply":"2023-03-10T23:27:29.251891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As we considired before we have some NaN values in the data, let's drop it\nfiltered_train_metadata_df = train_metadata_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:27:40.178512Z","iopub.execute_input":"2023-03-10T23:27:40.178913Z","iopub.status.idle":"2023-03-10T23:27:40.211467Z","shell.execute_reply.started":"2023-03-10T23:27:40.178878Z","shell.execute_reply":"2023-03-10T23:27:40.209877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.density_mapbox(\n    filtered_train_metadata_df, \n    lat='latitude', lon='longitude', \n    radius=7, zoom=2,\n    mapbox_style=\"stamen-terrain\", \n    center=dict(\n        lat=filtered_train_metadata_df['latitude'].mean(), \n        lon=filtered_train_metadata_df['longitude'].mean()\n    )\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:27:47.546288Z","iopub.execute_input":"2023-03-10T23:27:47.547168Z","iopub.status.idle":"2023-03-10T23:27:47.666515Z","shell.execute_reply.started":"2023-03-10T23:27:47.547096Z","shell.execute_reply":"2023-03-10T23:27:47.665618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_10 = len(filtered_train_metadata_df) // 10\n\nfig = px.scatter_mapbox(\n    filtered_train_metadata_df.sample(sample_10), \n    lat='latitude', lon='longitude',\n    color='primary_label', zoom=2,\n    mapbox_style=\"stamen-terrain\", \n    center=dict(\n        lat=filtered_train_metadata_df['latitude'].mean(), \n        lon=filtered_train_metadata_df['longitude'].mean()\n    )\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:27:57.853811Z","iopub.execute_input":"2023-03-10T23:27:57.855060Z","iopub.status.idle":"2023-03-10T23:27:58.506762Z","shell.execute_reply.started":"2023-03-10T23:27:57.855008Z","shell.execute_reply":"2023-03-10T23:27:58.505491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ebt_df = pd.read_csv(\"/kaggle/input/birdclef-2023/eBird_Taxonomy_v2021.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:28:09.846458Z","iopub.execute_input":"2023-03-10T23:28:09.846974Z","iopub.status.idle":"2023-03-10T23:28:09.932395Z","shell.execute_reply.started":"2023-03-10T23:28:09.846934Z","shell.execute_reply":"2023-03-10T23:28:09.931278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ebt_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:28:16.666543Z","iopub.execute_input":"2023-03-10T23:28:16.667760Z","iopub.status.idle":"2023-03-10T23:28:16.685886Z","shell.execute_reply.started":"2023-03-10T23:28:16.667699Z","shell.execute_reply":"2023-03-10T23:28:16.684518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ebt_df)","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:28:22.726538Z","iopub.execute_input":"2023-03-10T23:28:22.727414Z","iopub.status.idle":"2023-03-10T23:28:22.734930Z","shell.execute_reply.started":"2023-03-10T23:28:22.727367Z","shell.execute_reply":"2023-03-10T23:28:22.733906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ebt_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-10T23:28:30.886666Z","iopub.execute_input":"2023-03-10T23:28:30.887130Z","iopub.status.idle":"2023-03-10T23:28:30.904573Z","shell.execute_reply.started":"2023-03-10T23:28:30.887070Z","shell.execute_reply":"2023-03-10T23:28:30.903310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}