{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install fasteda","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-23T21:50:01.713020Z","iopub.execute_input":"2023-03-23T21:50:01.714571Z","iopub.status.idle":"2023-03-23T21:50:18.582033Z","shell.execute_reply.started":"2023-03-23T21:50:01.714506Z","shell.execute_reply":"2023-03-23T21:50:18.579735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom fasteda import fast_eda\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:51:33.652249Z","iopub.execute_input":"2023-03-23T21:51:33.652802Z","iopub.status.idle":"2023-03-23T21:51:33.661543Z","shell.execute_reply.started":"2023-03-23T21:51:33.652755Z","shell.execute_reply":"2023-03-23T21:51:33.659273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/birdclef-2023/eBird_Taxonomy_v2021.csv\")\ntrain_metadata = pd.read_csv(\"/kaggle/input/birdclef-2023/train_metadata.csv\")\ntrain_meta = pd.read_csv(\"/kaggle/input/birdclef-2023-train-meta-w-audio-durations/train_meta_audio_durations.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:51:34.026087Z","iopub.execute_input":"2023-03-23T21:51:34.026601Z","iopub.status.idle":"2023-03-23T21:51:34.275244Z","shell.execute_reply.started":"2023-03-23T21:51:34.026556Z","shell.execute_reply":"2023-03-23T21:51:34.273696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:57:12.328550Z","iopub.execute_input":"2023-03-21T19:57:12.329022Z","iopub.status.idle":"2023-03-21T19:57:12.348940Z","shell.execute_reply.started":"2023-03-21T19:57:12.328982Z","shell.execute_reply":"2023-03-21T19:57:12.347533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T20:04:44.869187Z","iopub.execute_input":"2023-03-21T20:04:44.870223Z","iopub.status.idle":"2023-03-21T20:04:44.900960Z","shell.execute_reply.started":"2023-03-21T20:04:44.870166Z","shell.execute_reply":"2023-03-21T20:04:44.899934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Duplicate values in train metadata","metadata":{}},{"cell_type":"code","source":"ool_idxs = []\n\nfor col in train_metadata.columns:\n col_list = []\n col_list.append(col)\n if col != \"url\": col_list.append(\"url\")\n if col != \"filename\": col_list.append(\"filename\")\n train_metadata_dup = train_metadata.drop(columns = col_list)\n dup_idx = list(train_metadata_dup.loc[train_metadata_dup.duplicated(keep = \"first\")].index)\n print(f\"# of duplicates: {len(dup_idx)} --> columns dropped: {col_list}\")\n\n if col == \"primary_label\":\n ool_idxs.append(dup_idx)\n \ntrain_metadata.loc[sum(ool_idxs, [])][\"primary_label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:57:15.811743Z","iopub.execute_input":"2023-03-21T19:57:15.812178Z","iopub.status.idle":"2023-03-21T19:57:16.040630Z","shell.execute_reply.started":"2023-03-21T19:57:15.812139Z","shell.execute_reply":"2023-03-21T19:57:16.039402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function used to find duplicate audio files - inefficient, use audio durations instead\ndef get_files(df, common_name):\n temp_df = df.query(\"common_name == @common_name\")\n sorted_files = sorted(list(temp_df[\"filename\"]))\n file_list = [os.path.join(\"/kaggle/input/birdclef-2023/train_audio/\", file) for file in sorted_files]\n return file_list","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Duplicate audio files 🔈","metadata":{}},{"cell_type":"code","source":"def visualize_duplicate_audio(file1, file2):\n \n path = \"/kaggle/input/birdclef-2023/train_audio/\"\n filename1 = path + file1\n filename2 = path + file2\n\n fig, axs = plt.subplots(1, 2, figsize=(16, 8))\n\n y1, sr1 = librosa.load(filename1)\n spec1 = librosa.display.specshow(librosa.power_to_db(librosa.feature.melspectrogram(y=y1, sr=sr1, n_mels=128, fmax=8000)), \n y_axis='mel', fmax=8000, x_axis='time', ax=axs[0])\n \n axs[0].set_title(f'Mel-Spectrogram | file: {filename1.split(\"/\")[-1]}')\n plt.colorbar(spec1, format='%+2.0f dB', ax=axs[0])\n\n y2, sr2 = librosa.load(filename2)\n spec2 = librosa.display.specshow(librosa.power_to_db(librosa.feature.melspectrogram(y=y2, sr=sr2, n_mels=128, fmax=8000)), \n y_axis='mel', fmax=8000, x_axis='time', ax=axs[1])\n \n axs[1].set_title(f'Mel-Spectrogram | file: {filename2.split(\"/\")[-1]}')\n plt.colorbar(spec2, format='%+2.0f dB', ax=axs[1])\n\n print(\" \" * 5, filename1.split(\"audio\")[1])\n display(Audio(filename1))\n print(\" \" * 5, filename2.split(\"audio\")[1])\n display(Audio(filename2))\n\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:51:37.504624Z","iopub.execute_input":"2023-03-23T21:51:37.505401Z","iopub.status.idle":"2023-03-23T21:51:37.517128Z","shell.execute_reply.started":"2023-03-23T21:51:37.505352Z","shell.execute_reply":"2023-03-23T21:51:37.515403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Common Bulbul\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"combul2/XC748220.ogg\", \"combul2/XC748221.ogg\") # Common Bulbul","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:10:17.823387Z","iopub.execute_input":"2023-03-19T15:10:17.823856Z","iopub.status.idle":"2023-03-19T15:10:31.176699Z","shell.execute_reply.started":"2023-03-19T15:10:17.823814Z","shell.execute_reply":"2023-03-19T15:10:31.175597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wood Sandpiper\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"woosan/XC740798.ogg\", \"woosan/XC742927.ogg\") # Wood Sandpiper","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:10:47.255139Z","iopub.execute_input":"2023-03-19T15:10:47.255781Z","iopub.status.idle":"2023-03-19T15:10:48.248505Z","shell.execute_reply.started":"2023-03-19T15:10:47.255738Z","shell.execute_reply":"2023-03-19T15:10:48.247481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gray-backed Camaroptera\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"gnbcam2/XC530150.ogg\", \"gnbcam2/XC530151.ogg\") # Gray-backed Camaroptera","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:12:05.828924Z","iopub.execute_input":"2023-03-19T15:12:05.829352Z","iopub.status.idle":"2023-03-19T15:12:07.412356Z","shell.execute_reply.started":"2023-03-19T15:12:05.829314Z","shell.execute_reply":"2023-03-19T15:12:07.411541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Common Buzzard\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"combuz1/XC647786.ogg\", \"combuz1/XC647787.ogg\") # Common Buzzard","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:12:53.836408Z","iopub.execute_input":"2023-03-19T15:12:53.837124Z","iopub.status.idle":"2023-03-19T15:12:55.655183Z","shell.execute_reply.started":"2023-03-19T15:12:53.837084Z","shell.execute_reply":"2023-03-19T15:12:55.652547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cape Robin-Chat\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"carcha1/XC324665.ogg\", \"carcha1/XC324666.ogg\") # Cape Robin-Chat","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:14:14.825229Z","iopub.execute_input":"2023-03-19T15:14:14.825972Z","iopub.status.idle":"2023-03-19T15:14:15.632228Z","shell.execute_reply.started":"2023-03-19T15:14:14.825930Z","shell.execute_reply":"2023-03-19T15:14:15.631185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Little Swift\n\n
\n
\n\n
\n
","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"litswi1/XC443712.ogg\", \"litswi1/XC443713.ogg\") # Little Swift","metadata":{"execution":{"iopub.status.busy":"2023-03-19T15:14:39.181171Z","iopub.execute_input":"2023-03-19T15:14:39.181615Z","iopub.status.idle":"2023-03-19T15:14:40.207600Z","shell.execute_reply.started":"2023-03-19T15:14:39.181572Z","shell.execute_reply":"2023-03-19T15:14:40.206553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Additional duplicate audio files 🔈","metadata":{}},{"cell_type":"code","source":"visualize_duplicate_audio(\"combul2/XC650878.ogg\", \"combul2/XC447669.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:51:42.033608Z","iopub.execute_input":"2023-03-23T21:51:42.034203Z","iopub.status.idle":"2023-03-23T21:51:57.060951Z","shell.execute_reply.started":"2023-03-23T21:51:42.034159Z","shell.execute_reply":"2023-03-23T21:51:57.059848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_duplicate_audio(\"gobbun1/XC394478.ogg\", \"gobbun1/XC395111.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:53:00.219718Z","iopub.execute_input":"2023-03-23T21:53:00.220140Z","iopub.status.idle":"2023-03-23T21:53:01.252279Z","shell.execute_reply.started":"2023-03-23T21:53:00.220103Z","shell.execute_reply":"2023-03-23T21:53:01.250771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_duplicate_audio(\"fislov1/XC503794.ogg\", \"fislov1/XC526237.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:53:15.646169Z","iopub.execute_input":"2023-03-23T21:53:15.646696Z","iopub.status.idle":"2023-03-23T21:53:16.456272Z","shell.execute_reply.started":"2023-03-23T21:53:15.646654Z","shell.execute_reply":"2023-03-23T21:53:16.454885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_duplicate_audio(\"cibwar1/XC395511.ogg\", \"cibwar1/XC432840.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:53:59.313054Z","iopub.execute_input":"2023-03-23T21:53:59.313586Z","iopub.status.idle":"2023-03-23T21:54:00.669181Z","shell.execute_reply.started":"2023-03-23T21:53:59.313541Z","shell.execute_reply":"2023-03-23T21:54:00.667209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_duplicate_audio(\"combul2/XC447668.ogg\", \"combul2/XC650877.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:54:29.956234Z","iopub.execute_input":"2023-03-23T21:54:29.956698Z","iopub.status.idle":"2023-03-23T21:54:31.019050Z","shell.execute_reply.started":"2023-03-23T21:54:29.956658Z","shell.execute_reply":"2023-03-23T21:54:31.017676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Length of audio files 🔈","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (14, 6))\n\nsns.histplot(x = train_meta[\"duration_minutes\"], ax = ax1)\n\nbox_dict = {\"boxprops\": dict(color = \"#000000\", linewidth = 1.5),\n \"capprops\": dict(color = \"#000000\", linewidth = 1),\n \"medianprops\": dict(color = \"#000000\", linewidth = 1),\n \"whiskerprops\": dict(color = \"#000000\", linewidth = 1),\n \"flierprops\": dict(markeredgecolor = \"#ff9900\"),\n \"meanprops\": dict(markeredgecolor = \"#000000\")}\n\ntrain_meta.boxplot(by = \"rating\", column = \"duration_minutes\", widths = 0.8, \n patch_artist = True, vert = False, **box_dict, ax = ax2)\n\nax1.set_xlabel(\"minutes\")\nax2.set_xlabel(\"minutes\")\nax2.set_title(None)\n\nfig.suptitle(\"Histogram of audio duration and Boxplots w/ audio ratings\", size = 20, y = 1.02)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T21:52:39.455983Z","iopub.execute_input":"2023-03-23T21:52:39.456818Z","iopub.status.idle":"2023-03-23T21:52:42.006845Z","shell.execute_reply.started":"2023-03-23T21:52:39.456753Z","shell.execute_reply":"2023-03-23T21:52:42.005747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta[\"primary_label_count\"] = train_meta[\"primary_label\"].map(train_meta[\"primary_label\"].value_counts())\n\nplt.figure(figsize = (10, 6))\n\nax = sns.barplot(x = np.round(train_meta.query(\"primary_label_count > 100\").groupby(\"primary_label\")[\"duration_seconds\"].median().sort_values(ascending = False).values[:10], 1), \n y = train_meta.query(\"primary_label_count > 100\").groupby(\"primary_label\")[\"duration_seconds\"].median().sort_values(ascending = False).keys()[:10],\n palette = \"Greens_r\", edgecolor = \"#000000\")\n\nplt.title(\"10 longest median audio length of birds with > 100 recordings\")\n\nfor container in ax.containers:\n ax.bar_label(container, padding = 5)\n \nax.set_xlabel(\"seconds\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:43:21.495903Z","iopub.execute_input":"2023-03-21T19:43:21.496309Z","iopub.status.idle":"2023-03-21T19:43:21.848113Z","shell.execute_reply.started":"2023-03-21T19:43:21.496274Z","shell.execute_reply":"2023-03-21T19:43:21.846970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\n\nax = sns.barplot(y = list(train_meta.type.value_counts().keys()[:10]), \n x = list(train_meta.type.value_counts().values[:10]),\n palette = \"Reds_r\", edgecolor = \"#000000\")\n\nplt.title(\"Top 10 audio types present in train_metadata.csv\")\n\nfor container in ax.containers:\n ax.bar_label(container, padding = 5)\n \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:43:40.070987Z","iopub.execute_input":"2023-03-21T19:43:40.071461Z","iopub.status.idle":"2023-03-21T19:43:40.430244Z","shell.execute_reply.started":"2023-03-21T19:43:40.071421Z","shell.execute_reply":"2023-03-21T19:43:40.428985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_df = pd.DataFrame()\ncount_df['secondary_label_count'] = train_meta['secondary_labels'].apply(lambda x: len(eval(x)) if x != '[]' else 0)\ncount_df['type_count'] = train_meta['type'].apply(lambda x: len(eval(x)) if x != \"['']\" else 0)\n\nplt.figure(figsize = (10, 6))\n\nax = sns.countplot(data = count_df, x = \"type_count\", \n palette = \"Spectral\", edgecolor = \"#000000\")\n\nplt.title(\"Number of audio types for each recording\")\n\nfor container in ax.containers:\n ax.bar_label(container, padding = 5)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:54:57.244764Z","iopub.execute_input":"2023-03-21T19:54:57.245849Z","iopub.status.idle":"2023-03-21T19:54:57.669121Z","shell.execute_reply.started":"2023-03-21T19:54:57.245773Z","shell.execute_reply":"2023-03-21T19:54:57.667984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\n\nax = sns.countplot(data = count_df, x = \"secondary_label_count\", \n palette = \"Spectral\", edgecolor = \"#000000\")\n\nplt.title(\"Number of secondary labels\")\n\nfor container in ax.containers:\n ax.bar_label(container, padding = 5)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:55:00.435260Z","iopub.execute_input":"2023-03-21T19:55:00.435734Z","iopub.status.idle":"2023-03-21T19:55:00.748187Z","shell.execute_reply.started":"2023-03-21T19:55:00.435650Z","shell.execute_reply":"2023-03-21T19:55:00.747035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 8))\n\nsns.scatterplot(data = train_meta, x = \"longitude\", y = \"latitude\", \n hue = \"rating\", palette = \"Spectral\")\n\nplt.title(\"Scatterplot of audio ratings\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:50:43.393277Z","iopub.execute_input":"2023-03-21T19:50:43.394576Z","iopub.status.idle":"2023-03-21T19:50:44.548876Z","shell.execute_reply.started":"2023-03-21T19:50:43.394519Z","shell.execute_reply":"2023-03-21T19:50:44.547857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fast_eda(df, correlation = False, pairplot = False)","metadata":{"execution":{"iopub.status.busy":"2023-03-21T19:53:49.422315Z","iopub.execute_input":"2023-03-21T19:53:49.423121Z","iopub.status.idle":"2023-03-21T19:53:50.948216Z","shell.execute_reply.started":"2023-03-21T19:53:49.423081Z","shell.execute_reply":"2023-03-21T19:53:50.947248Z"},"trusted":true},"execution_count":null,"outputs":[]}]}