{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":29955,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📋Table of Contents\n* [Basic EDA](#eda)\n* [Geography](#geo)\n* [Audio Files](#audio)","metadata":{}},{"cell_type":"code","source":"# packages\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport folium\n\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-03-11T19:05:28.775188Z","iopub.execute_input":"2025-03-11T19:05:28.775596Z","iopub.status.idle":"2025-03-11T19:05:28.781082Z","shell.execute_reply.started":"2025-03-11T19:05:28.775559Z","shell.execute_reply":"2025-03-11T19:05:28.779911Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# file overview\n!ls -l '../input/birdclef-2025'","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:04.475670Z","iopub.execute_input":"2025-03-11T19:00:04.475980Z","iopub.status.idle":"2025-03-11T19:00:05.704593Z","shell.execute_reply.started":"2025-03-11T19:00:04.475948Z","shell.execute_reply":"2025-03-11T19:00:05.703289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read train data file\ndf = pd.read_csv('../input/birdclef-2025/train.csv')\n\n# read taxonomy file\ndf_taxo = pd.read_csv('../input/birdclef-2025/taxonomy.csv')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2025-03-11T19:00:05.706435Z","iopub.execute_input":"2025-03-11T19:00:05.706759Z","iopub.status.idle":"2025-03-11T19:00:05.913464Z","shell.execute_reply.started":"2025-03-11T19:00:05.706724Z","shell.execute_reply":"2025-03-11T19:00:05.912438Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# add taxonomy info to data\ndf = pd.merge(left=df, right=df_taxo[['primary_label', 'inat_taxon_id', 'class_name']], how='left', on='primary_label')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:05.914819Z","iopub.execute_input":"2025-03-11T19:00:05.915113Z","iopub.status.idle":"2025-03-11T19:00:05.959032Z","shell.execute_reply.started":"2025-03-11T19:00:05.915084Z","shell.execute_reply":"2025-03-11T19:00:05.958226Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='eda'></a>\n# Basic EDA","metadata":{}},{"cell_type":"code","source":"# preview of data\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:05.962068Z","iopub.execute_input":"2025-03-11T19:00:05.962360Z","iopub.status.idle":"2025-03-11T19:00:05.989036Z","shell.execute_reply.started":"2025-03-11T19:00:05.962314Z","shell.execute_reply":"2025-03-11T19:00:05.988127Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### More than 800 lat/lon values are missing:","metadata":{"execution":{"iopub.status.busy":"2025-03-10T19:27:44.196313Z","iopub.execute_input":"2025-03-10T19:27:44.196696Z","iopub.status.idle":"2025-03-10T19:27:44.201570Z","shell.execute_reply.started":"2025-03-10T19:27:44.196665Z","shell.execute_reply":"2025-03-10T19:27:44.200139Z"}}},{"cell_type":"code","source":"df.latitude.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:05.992171Z","iopub.execute_input":"2025-03-11T19:00:05.992495Z","iopub.status.idle":"2025-03-11T19:00:06.010623Z","shell.execute_reply.started":"2025-03-11T19:00:05.992464Z","shell.execute_reply":"2025-03-11T19:00:06.009485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.longitude.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:06.012179Z","iopub.execute_input":"2025-03-11T19:00:06.012537Z","iopub.status.idle":"2025-03-11T19:00:06.031660Z","shell.execute_reply.started":"2025-03-11T19:00:06.012505Z","shell.execute_reply":"2025-03-11T19:00:06.030449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# convert lat/lon to numeric after removing string entries\ndf.latitude = pd.to_numeric(df.latitude.replace(to_replace='None', value=np.nan), errors='coerce')\ndf.longitude = pd.to_numeric(df.longitude.replace(to_replace='None', value=np.nan), errors='coerce')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:06.033012Z","iopub.execute_input":"2025-03-11T19:00:06.033392Z","iopub.status.idle":"2025-03-11T19:00:06.065614Z","shell.execute_reply.started":"2025-03-11T19:00:06.033323Z","shell.execute_reply":"2025-03-11T19:00:06.064693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# structure details\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:06.067160Z","iopub.execute_input":"2025-03-11T19:00:06.067613Z","iopub.status.idle":"2025-03-11T19:00:06.107863Z","shell.execute_reply.started":"2025-03-11T19:00:06.067570Z","shell.execute_reply":"2025-03-11T19:00:06.106857Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# eval frequencies - primary labels\nprim_freq = df.primary_label.value_counts()\nprim_freq","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:06.109042Z","iopub.execute_input":"2025-03-11T19:00:06.109362Z","iopub.status.idle":"2025-03-11T19:00:06.121378Z","shell.execute_reply.started":"2025-03-11T19:00:06.109310Z","shell.execute_reply":"2025-03-11T19:00:06.120552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# secondary labels\ndf.secondary_labels.value_counts()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:06.122853Z","iopub.execute_input":"2025-03-11T19:00:06.123144Z","iopub.status.idle":"2025-03-11T19:00:06.140946Z","shell.execute_reply.started":"2025-03-11T19:00:06.123115Z","shell.execute_reply":"2025-03-11T19:00:06.139796Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# distribution of class names\ndf.class_name.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:06.142001Z","iopub.execute_input":"2025-03-11T19:00:06.142277Z","iopub.status.idle":"2025-03-11T19:00:06.161005Z","shell.execute_reply.started":"2025-03-11T19:00:06.142244Z","shell.execute_reply":"2025-03-11T19:00:06.159896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# collections\ndf.collection.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:06.162735Z","iopub.execute_input":"2025-03-11T19:00:06.163168Z","iopub.status.idle":"2025-03-11T19:00:06.182743Z","shell.execute_reply.started":"2025-03-11T19:00:06.163125Z","shell.execute_reply":"2025-03-11T19:00:06.181425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ratings\nplt.figure(figsize=(10,4))\ndf.rating.value_counts().sort_index().plot(kind='bar', color='darkblue')\nplt.title('Ratings')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:06.184144Z","iopub.execute_input":"2025-03-11T19:00:06.184544Z","iopub.status.idle":"2025-03-11T19:00:06.426160Z","shell.execute_reply.started":"2025-03-11T19:00:06.184505Z","shell.execute_reply":"2025-03-11T19:00:06.425041Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='geo'></a>\n# Geography","metadata":{}},{"cell_type":"markdown","source":"## Static plots","metadata":{}},{"cell_type":"code","source":"# first simple plot of locations\nplt.figure(figsize=(12,6))\nsns.scatterplot(data=df, x='longitude', y='latitude', \n                color='darkblue')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:06.427609Z","iopub.execute_input":"2025-03-11T19:00:06.427932Z","iopub.status.idle":"2025-03-11T19:00:06.680958Z","shell.execute_reply.started":"2025-03-11T19:00:06.427900Z","shell.execute_reply":"2025-03-11T19:00:06.679965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select first 10 categories and plot in color\ndf_select = df[df.primary_label.isin(prim_freq[0:9+1].index)]\nplt.figure(figsize=(12,6))\nsns.scatterplot(x='longitude', y='latitude', hue='primary_label', data=df_select, palette='colorblind')\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) # move legend out of the plot area\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:06.682228Z","iopub.execute_input":"2025-03-11T19:00:06.682584Z","iopub.status.idle":"2025-03-11T19:00:07.179406Z","shell.execute_reply.started":"2025-03-11T19:00:06.682552Z","shell.execute_reply":"2025-03-11T19:00:07.178216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select next 10 categories and plot in color\ndf_select = df[df.primary_label.isin(prim_freq[10:19+1].index)]\nplt.figure(figsize=(12,6))\nsns.scatterplot(x='longitude', y='latitude', hue='primary_label', data=df_select, palette='colorblind')\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:07.180703Z","iopub.execute_input":"2025-03-11T19:00:07.180991Z","iopub.status.idle":"2025-03-11T19:00:07.525561Z","shell.execute_reply.started":"2025-03-11T19:00:07.180962Z","shell.execute_reply":"2025-03-11T19:00:07.524398Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select next 10 categories and plot in color\ndf_select = df[df.primary_label.isin(prim_freq[20:29+1].index)]\nplt.figure(figsize=(12,6))\nsns.scatterplot(x='longitude', y='latitude', hue='primary_label', data=df_select, palette='colorblind')\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:07.527139Z","iopub.execute_input":"2025-03-11T19:00:07.527587Z","iopub.status.idle":"2025-03-11T19:00:07.846903Z","shell.execute_reply.started":"2025-03-11T19:00:07.527547Z","shell.execute_reply":"2025-03-11T19:00:07.845982Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select next 10 categories and plot in color\ndf_select = df[df.primary_label.isin(prim_freq[30:39+1].index)]\nplt.figure(figsize=(12,6))\nsns.scatterplot(x='longitude', y='latitude', hue='primary_label', data=df_select, palette='colorblind')\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:07.848233Z","iopub.execute_input":"2025-03-11T19:00:07.848657Z","iopub.status.idle":"2025-03-11T19:00:08.157260Z","shell.execute_reply.started":"2025-03-11T19:00:07.848615Z","shell.execute_reply":"2025-03-11T19:00:08.156304Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot by class name","metadata":{}},{"cell_type":"code","source":"classes = df.class_name.value_counts().index.tolist()\nprint(classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:08.158739Z","iopub.execute_input":"2025-03-11T19:00:08.159123Z","iopub.status.idle":"2025-03-11T19:00:08.169727Z","shell.execute_reply.started":"2025-03-11T19:00:08.159074Z","shell.execute_reply":"2025-03-11T19:00:08.168764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for c in classes:\n    df_select = df[df.class_name==c]\n    plt.figure(figsize=(12,6))\n    sns.scatterplot(data=df_select, x='longitude', y='latitude', \n                    color='darkblue')\n    plt.xlim(-180,180)\n    plt.ylim(-70,70)\n    plt.title(c)\n    plt.grid()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:08.171034Z","iopub.execute_input":"2025-03-11T19:00:08.171407Z","iopub.status.idle":"2025-03-11T19:00:08.961282Z","shell.execute_reply.started":"2025-03-11T19:00:08.171335Z","shell.execute_reply":"2025-03-11T19:00:08.960366Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Look at only one category and provide interactive map","metadata":{}},{"cell_type":"code","source":"my_bird = 'grekis'\ndf_example = df[df.primary_label.isin([my_bird])]\ndf_example.shape","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:08.962407Z","iopub.execute_input":"2025-03-11T19:00:08.962666Z","iopub.status.idle":"2025-03-11T19:00:08.971492Z","shell.execute_reply.started":"2025-03-11T19:00:08.962641Z","shell.execute_reply":"2025-03-11T19:00:08.970526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check for missing coordinates\nprint('Missing latitudes:', df_example.latitude.isna().sum())\nprint('Missing longitudes:', df_example.longitude.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:08.972935Z","iopub.execute_input":"2025-03-11T19:00:08.973227Z","iopub.status.idle":"2025-03-11T19:00:08.990376Z","shell.execute_reply.started":"2025-03-11T19:00:08.973199Z","shell.execute_reply":"2025-03-11T19:00:08.989441Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# remove rows with missings\ndf_example = df_example.dropna(axis=0, subset=['latitude','longitude'])\ndf_example.shape","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:08.991616Z","iopub.execute_input":"2025-03-11T19:00:08.991900Z","iopub.status.idle":"2025-03-11T19:00:09.012029Z","shell.execute_reply.started":"2025-03-11T19:00:08.991872Z","shell.execute_reply":"2025-03-11T19:00:09.010939Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# interactive map\nzoom_factor = 1.9\nmy_map_1 = folium.Map(location=[0,0], zoom_start=zoom_factor)\n\nfor i in range(0,df_example.shape[0]):\n    folium.Circle(\n        location=[df_example.iloc[i]['latitude'], df_example.iloc[i]['longitude']],\n        radius=np.sqrt(df_example.iloc[i]['rating'])*25000,\n        color='blue',\n        weight=1,\n        popup='label: ' + df_example.iloc[i]['primary_label'] + '<br>' +\n              'sec_labels: ' + df_example.iloc[i]['secondary_labels'] + '<br>' +\n              'type: ' + df_example.iloc[i]['type'] + '<br>' +\n              'URL: ' + df_example.iloc[i]['url'],\n        fill=True,\n        fill_color='blue').add_to(my_map_1)\n\nmy_map_1 # display","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:09.013548Z","iopub.execute_input":"2025-03-11T19:00:09.013854Z","iopub.status.idle":"2025-03-11T19:00:11.676904Z","shell.execute_reply.started":"2025-03-11T19:00:09.013824Z","shell.execute_reply":"2025-03-11T19:00:11.675028Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Add another one","metadata":{}},{"cell_type":"code","source":"my_bird = 'banana'\ndf_example = df[df.primary_label.isin([my_bird])]\ndf_example = df_example.dropna(axis=0, subset=['latitude','longitude'])\n\n# interactive map\nzoom_factor = 1.9\nmy_map_2 = folium.Map(location=[0,0], zoom_start=zoom_factor)\n\nfor i in range(0,df_example.shape[0]):\n    folium.Circle(\n        location=[df_example.iloc[i]['latitude'], df_example.iloc[i]['longitude']],\n        radius=np.sqrt(df_example.iloc[i]['rating'])*25000,\n        color='red',\n        weight=1,\n        popup='label: ' + df_example.iloc[i]['primary_label'] + '<br>' +\n              'sec_labels: ' + df_example.iloc[i]['secondary_labels'] + '<br>' +\n              'type: ' + df_example.iloc[i]['type'] + '<br>' +\n              'URL: ' + df_example.iloc[i]['url'],\n        fill=True,\n        fill_color='red').add_to(my_map_1)\n\nmy_map_1 # display","metadata":{"execution":{"iopub.status.busy":"2025-03-11T19:00:11.678136Z","iopub.execute_input":"2025-03-11T19:00:11.678467Z","iopub.status.idle":"2025-03-11T19:00:14.381658Z","shell.execute_reply.started":"2025-03-11T19:00:11.678435Z","shell.execute_reply":"2025-03-11T19:00:14.380153Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='audio'></a>\n# Audio Files","metadata":{}},{"cell_type":"code","source":"# look in an example path\n!ls -l '../input/birdclef-2025/train_audio/banana'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:00:14.383084Z","iopub.execute_input":"2025-03-11T19:00:14.383435Z","iopub.status.idle":"2025-03-11T19:00:15.754951Z","shell.execute_reply.started":"2025-03-11T19:00:14.383402Z","shell.execute_reply":"2025-03-11T19:00:15.753661Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load audio file\nfilename='XC112602.ogg'\ny, sr = librosa.load('../input/birdclef-2025/train_audio/banana/' + filename)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:03:55.645198Z","iopub.execute_input":"2025-03-11T19:03:55.645603Z","iopub.status.idle":"2025-03-11T19:03:56.775979Z","shell.execute_reply.started":"2025-03-11T19:03:55.645564Z","shell.execute_reply":"2025-03-11T19:03:56.775123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show wave data\nplt.figure(figsize=(14,5))\nplt.plot(y, color='darkblue')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:03:58.430493Z","iopub.execute_input":"2025-03-11T19:03:58.430839Z","iopub.status.idle":"2025-03-11T19:03:58.793989Z","shell.execute_reply.started":"2025-03-11T19:03:58.430811Z","shell.execute_reply":"2025-03-11T19:03:58.791836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# play sound\nAudio(y, rate=sr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:21:46.213997Z","iopub.execute_input":"2025-03-11T19:21:46.214366Z","iopub.status.idle":"2025-03-11T19:21:46.251312Z","shell.execute_reply.started":"2025-03-11T19:21:46.214318Z","shell.execute_reply":"2025-03-11T19:21:46.249741Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Spectrogram","metadata":{}},{"cell_type":"code","source":"# fourier transform + amplitudes in dB scale\nft = librosa.stft(y)\nS_db = librosa.amplitude_to_db(np.abs(ft), ref=np.max)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:17:23.999282Z","iopub.execute_input":"2025-03-11T19:17:23.999685Z","iopub.status.idle":"2025-03-11T19:17:24.077027Z","shell.execute_reply.started":"2025-03-11T19:17:23.999648Z","shell.execute_reply":"2025-03-11T19:17:24.076094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot spectrogram\nfig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10,7))\nimg = librosa.display.specshow(S_db, y_axis='log', sr=sr, \n                         x_axis='time', ax=ax)\nax.set(title='Spectrogram - log Frequency')\nax.label_outer()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-11T19:22:21.700038Z","iopub.execute_input":"2025-03-11T19:22:21.700414Z","iopub.status.idle":"2025-03-11T19:22:22.826314Z","shell.execute_reply.started":"2025-03-11T19:22:21.700380Z","shell.execute_reply":"2025-03-11T19:22:22.825320Z"}},"outputs":[],"execution_count":null}]}