{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Bird Clef 2023- Data Exploration","metadata":{}},{"cell_type":"markdown","source":"inspired by Rob Mulla EDA","metadata":{}},{"cell_type":"markdown","source":"### Import Libraries","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:06:00.875713Z","iopub.execute_input":"2023-03-24T13:06:00.876178Z","iopub.status.idle":"2023-03-24T13:06:02.007817Z","shell.execute_reply.started":"2023-03-24T13:06:00.876134Z","shell.execute_reply":"2023-03-24T13:06:02.006162Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pylab as plt\nimport plotly.express as px\nimport seaborn as sns\n# import Audio libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\n\nsns.set_theme(style=\"white\", palette=None)\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\n\nfrom itertools import cycle\n\ncolor_cycle = cycle(plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:14:54.709361Z","iopub.execute_input":"2023-03-25T08:14:54.710257Z","iopub.status.idle":"2023-03-25T08:14:58.557942Z","shell.execute_reply.started":"2023-03-25T08:14:54.710208Z","shell.execute_reply":"2023-03-25T08:14:58.556257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Competition Files\nThe Data provided as follows:\n### Audio Files:\n* train_audio The training data consists of short recordings of individual bird calls generously uploaded by users of xenocanto.org. These files have been downsampled to 32 kHz where applicable to match the test set audio and converted to the ogg format.\n\n* test_soundscapes the directory will be populated with approximately 200 recordings to be used for scoring. They are 10 minutes long and in ogg audio format. The file names are randomized.\n\n### CSV Files:\n##### train_metadata.csv :A wide range of metadata is provided for the training data. The most directly relevant fields are:\n\n* primary_label a code for the bird species. You can review detailed information about the bird codes by appending the code to https://ebird.org/species/, such as https://ebird.org/species/amecro for the American Crow.\n* latitude & longitude coordinates for where the recording was taken. Some bird species may have local call 'dialects,' so you may want to seek geographic diversity in your training data.\n* author The user who provided the recording.\n* filename the name of the associated audio file.\n\n##### sample_submission.csv: A valid sample submission.\n\n* row_id A slug of [soundscape_id]_[end_time] for the prediction.\n* [bird_id] There are 264 bird ID columns. You will need to predict the probability of the presence of each bird for each row.\n\n##### eBird_Taxonomy_v2021.csv: Data on the relationships between different species.","metadata":{}},{"cell_type":"code","source":"!ls -GFlash --color ../input/birdclef-2023","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:14:58.560686Z","iopub.execute_input":"2023-03-25T08:14:58.561488Z","iopub.status.idle":"2023-03-25T08:14:59.714779Z","shell.execute_reply.started":"2023-03-25T08:14:58.561428Z","shell.execute_reply":"2023-03-25T08:14:59.712965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read the CSV files\nBASE_DIR = '../input/birdclef-2023/'\ntrain = pd.read_csv(f'{BASE_DIR}/train_metadata.csv')\nss = pd.read_csv(f'{BASE_DIR}/sample_submission.csv')\nebird = pd.read_csv(f'{BASE_DIR}/eBird_Taxonomy_v2021.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:14:59.717217Z","iopub.execute_input":"2023-03-25T08:14:59.718978Z","iopub.status.idle":"2023-03-25T08:14:59.975582Z","shell.execute_reply.started":"2023-03-25T08:14:59.718911Z","shell.execute_reply":"2023-03-25T08:14:59.974302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the shape of the CSV Files\nprint('Train csv shape:{}'.format(train.shape))\nprint('Submission csv shape:{}'.format(ss.shape))\nprint('ebird csv shape:{}'.format(ebird.shape))","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:14:59.978140Z","iopub.execute_input":"2023-03-25T08:14:59.978505Z","iopub.status.idle":"2023-03-25T08:14:59.990462Z","shell.execute_reply.started":"2023-03-25T08:14:59.978472Z","shell.execute_reply":"2023-03-25T08:14:59.987761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:14:59.992446Z","iopub.execute_input":"2023-03-25T08:14:59.993668Z","iopub.status.idle":"2023-03-25T08:15:00.039287Z","shell.execute_reply.started":"2023-03-25T08:14:59.993609Z","shell.execute_reply":"2023-03-25T08:15:00.037949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we will notice that latitue and longitude is missing for 227 values out of 16941\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.041082Z","iopub.execute_input":"2023-03-25T08:15:00.042222Z","iopub.status.idle":"2023-03-25T08:15:00.062381Z","shell.execute_reply.started":"2023-03-25T08:15:00.042175Z","shell.execute_reply":"2023-03-25T08:15:00.060693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.065401Z","iopub.execute_input":"2023-03-25T08:15:00.066323Z","iopub.status.idle":"2023-03-25T08:15:00.077025Z","shell.execute_reply.started":"2023-03-25T08:15:00.066260Z","shell.execute_reply":"2023-03-25T08:15:00.075573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.079137Z","iopub.execute_input":"2023-03-25T08:15:00.079654Z","iopub.status.idle":"2023-03-25T08:15:00.124670Z","shell.execute_reply.started":"2023-03-25T08:15:00.079603Z","shell.execute_reply":"2023-03-25T08:15:00.123618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No Duplicated data in the training dataset","metadata":{}},{"cell_type":"code","source":"ebird.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.126158Z","iopub.execute_input":"2023-03-25T08:15:00.126741Z","iopub.status.idle":"2023-03-25T08:15:00.142585Z","shell.execute_reply.started":"2023-03-25T08:15:00.126704Z","shell.execute_reply":"2023-03-25T08:15:00.141574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ebird.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.146360Z","iopub.execute_input":"2023-03-25T08:15:00.147415Z","iopub.status.idle":"2023-03-25T08:15:00.163993Z","shell.execute_reply.started":"2023-03-25T08:15:00.147366Z","shell.execute_reply":"2023-03-25T08:15:00.162619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualization","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(12,5))\n# check the frequency of labels in the training data\ntrain[\"common_name\"].value_counts().head(20).plot(\n    kind=\"bar\", ax=axs[0], width=1, color=color_pal[0])\n\naxs[0].set_title(\"Top 20 Birds with labels\", fontsize=20)\n\nax = (\n    train[\"common_name\"]\n    .value_counts()\n    .tail(20)\n    .plot(kind=\"bar\", ax=axs[1], width=1, color=color_pal[1]))\n\naxs[1].set_title(\"Bottom 20 Birds with labels\", fontsize=20);","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:00.165342Z","iopub.execute_input":"2023-03-25T08:15:00.166028Z","iopub.status.idle":"2023-03-25T08:15:01.147400Z","shell.execute_reply.started":"2023-03-25T08:15:00.165992Z","shell.execute_reply":"2023-03-25T08:15:01.145994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(12,5))\n# check the frequency of birds with primary name in the training data\ntrain[\"primary_label\"].value_counts().head(20).plot(\n    kind=\"bar\", ax=axs[0], width=1, color=color_pal[0])\n\naxs[0].set_title(\"Top 20 Birds with primary name\", fontsize=15)\n\nax = (\n    train[\"primary_label\"]\n    .value_counts()\n    .tail(20)\n    .plot(kind=\"bar\", ax=axs[1], width=1, color=color_pal[1]))\n\naxs[1].set_title(\"Bottom 20 Birds with primary name\", fontsize=15);","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:01.149253Z","iopub.execute_input":"2023-03-25T08:15:01.150375Z","iopub.status.idle":"2023-03-25T08:15:02.220876Z","shell.execute_reply.started":"2023-03-25T08:15:01.150317Z","shell.execute_reply":"2023-03-25T08:15:02.219491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot lat/long location\nfig = px.scatter_geo(\n    train,\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"common_name\",\n    width=1_000,\n    height=500,\n    title=\"BirdClef 2023 with common name Training Data\",\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:49.822043Z","iopub.execute_input":"2023-03-25T08:15:49.822551Z","iopub.status.idle":"2023-03-25T08:15:51.092619Z","shell.execute_reply.started":"2023-03-25T08:15:49.822509Z","shell.execute_reply":"2023-03-25T08:15:51.091267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot lat/long location\nfig = px.scatter_geo(\n    train,\n    lat=\"latitude\",\n    lon=\"longitude\",\n    color=\"primary_label\",\n    width=1_000,\n    height=500,\n    title=\"BirdClef 2023 with primary labels Training Data\",\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:16:15.209676Z","iopub.execute_input":"2023-03-25T08:16:15.210282Z","iopub.status.idle":"2023-03-25T08:16:16.126448Z","shell.execute_reply.started":"2023-03-25T08:16:15.210229Z","shell.execute_reply":"2023-03-25T08:16:16.125191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# countplot for birds ratings\nfig, ax = plt.subplots(figsize=(15,8))\nsns.countplot(data=train, x=\"rating\", ax=ax);","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:16:48.922431Z","iopub.execute_input":"2023-03-25T08:16:48.923096Z","iopub.status.idle":"2023-03-25T08:16:49.298674Z","shell.execute_reply.started":"2023-03-25T08:16:48.923035Z","shell.execute_reply":"2023-03-25T08:16:49.297444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> * majority of birds with rating greater than or equal 4.0","metadata":{}},{"cell_type":"markdown","source":"### Load examples Training Audio data","metadata":{}},{"cell_type":"code","source":"#Listen to the audio of the first training example\nfn = train[\"filename\"].values[0]\nipd.Audio(f\"{BASE_DIR}train_audio/{fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:05.988505Z","iopub.execute_input":"2023-03-25T08:15:05.988903Z","iopub.status.idle":"2023-03-25T08:15:06.018748Z","shell.execute_reply.started":"2023-03-25T08:15:05.988848Z","shell.execute_reply":"2023-03-25T08:15:06.017574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Barn Swallow Example\nfn = train.loc[train[\"common_name\"] == \"Barn Swallow\"][\"filename\"].values[0]\nipd.Audio(f\"{BASE_DIR}train_audio/{fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:06.020597Z","iopub.execute_input":"2023-03-25T08:15:06.021805Z","iopub.status.idle":"2023-03-25T08:15:06.070004Z","shell.execute_reply.started":"2023-03-25T08:15:06.021753Z","shell.execute_reply":"2023-03-25T08:15:06.068573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Willow warbler Swallow Example\nfn = train.loc[train[\"common_name\"] == \"Willow Warbler\"][\"filename\"].values[0]\nipd.Audio(f\"{BASE_DIR}train_audio/{fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:06.071289Z","iopub.execute_input":"2023-03-25T08:15:06.071693Z","iopub.status.idle":"2023-03-25T08:15:06.115634Z","shell.execute_reply.started":"2023-03-25T08:15:06.071653Z","shell.execute_reply":"2023-03-25T08:15:06.114427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y, sr = librosa.load(f\"{BASE_DIR}train_audio/{fn}\")\nprint(f\"Numpy Array of the sample audio example of {y.shape}, and sample rate {sr}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:15:06.117466Z","iopub.execute_input":"2023-03-25T08:15:06.117848Z","iopub.status.idle":"2023-03-25T08:15:18.921995Z","shell.execute_reply.started":"2023-03-25T08:15:06.117800Z","shell.execute_reply":"2023-03-25T08:15:18.920504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot 10 Random Audio files from training Audio","metadata":{"execution":{"iopub.status.busy":"2023-03-24T21:43:25.742204Z","iopub.execute_input":"2023-03-24T21:43:25.742584Z","iopub.status.idle":"2023-03-24T21:43:25.748029Z","shell.execute_reply.started":"2023-03-24T21:43:25.742555Z","shell.execute_reply":"2023-03-24T21:43:25.746084Z"}}},{"cell_type":"code","source":"# Plot raw Data\ndef plot_raw_audio(filename, birdtype, color):\n    y, sr = librosa.load(f\"{BASE_DIR}train_audio/{filename}\")\n    ax = pd.DataFrame(y).plot(\n        figsize=(10,3), title=f\"{birdtype} Raw Audio\", lw=0.1, \n        color = color\n    )\n    plt.legend().remove()\n    plt.show()\n    \nfor i, d in train.sample(10, random_state=529).iterrows():\n    plot_raw_audio(d[\"filename\"], d[\"common_name\"], next(color_cycle))   ","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:24:11.111453Z","iopub.execute_input":"2023-03-25T08:24:11.112028Z","iopub.status.idle":"2023-03-25T08:24:16.032425Z","shell.execute_reply.started":"2023-03-25T08:24:11.111978Z","shell.execute_reply":"2023-03-25T08:24:16.031074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spectrogram plots for Birds","metadata":{}},{"cell_type":"code","source":"def plot_audio_melspec(filename, birdtype):\n    y, sr = librosa.load(f\"{BASE_DIR}train_audio/{filename}\")\n    S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128, fmax=8000)\n    \n    fig, ax = plt.subplots(figsize=(10,3))\n    S_dB = librosa.power_to_db(S, ref=np.max)\n    img = librosa.display.specshow(\n        S_dB, x_axis=\"time\", y_axis=\"mel\", sr=sr, fmax=8000, ax=ax\n    )\n    fig.colorbar(img, ax=ax, format=\"%+2.0f dB\")\n    ax.set(title=f\"Mel-frequency for {birdtype}\")\n    plt.show()\n    \nfor i, d in train.sample(10, random_state=529).iterrows():\n    plot_audio_melspec(d[\"filename\"], d[\"common_name\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-25T08:54:51.054501Z","iopub.execute_input":"2023-03-25T08:54:51.055417Z","iopub.status.idle":"2023-03-25T08:54:57.808337Z","shell.execute_reply.started":"2023-03-25T08:54:51.055351Z","shell.execute_reply":"2023-03-25T08:54:57.807060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}