{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Takeaways\n\n- There are 16941 recordings in total, 192.4 hours, 82.6 GB uncompressed (32 khz with 4 bytes per frame)\n- Just in Kenya: 2498 reecordings, 19.9 hours, 8.6 GB uncompressed\n- Note that kaggle's P100 machines have 16 GB of GPU RAM and 12 GB of CPU RAM.\n- There are 264 unique labels in total, 231 unique primary labels in kenya + 1 other secondary label in kenya (note that geo boundary is crude)\n    - So some birds don't appear in Kenya, but will be evaluated? (confirmed from hosts that [not all 264 species occur in the test data](https://www.kaggle.com/competitions/birdclef-2023/discussion/396101#2189267))\n- The top ~11 most frequent birds recorded in the world are mostly absent from Kenya\n- Only 2305 or 13.6% of the recordings have secondary labels\n- 7 birds appear more frequently as secondary labels than primary labels\n- 4 primary labels appear exclusively in kenya","metadata":{}},{"cell_type":"markdown","source":"# All recordings","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:40:51.940718Z","iopub.execute_input":"2023-04-09T22:40:51.941022Z","iopub.status.idle":"2023-04-09T22:40:52.103064Z","shell.execute_reply.started":"2023-04-09T22:40:51.940995Z","shell.execute_reply":"2023-04-09T22:40:52.101949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.filename.str.contains('XC138886')]","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:40:52.167742Z","iopub.execute_input":"2023-04-09T22:40:52.168345Z","iopub.status.idle":"2023-04-09T22:40:52.196612Z","shell.execute_reply.started":"2023-04-09T22:40:52.168284Z","shell.execute_reply":"2023-04-09T22:40:52.195387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\nprimary_label_counts = df.primary_label.value_counts()\n\npx.bar(\n    x=primary_label_counts.keys(), \n    y=primary_label_counts.values,\n    title=\"Distribution of primary labels\",\n    labels={\"x\": \"bird\", \"y\": \"# of recordings\"},\n).show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:40:53.967492Z","iopub.execute_input":"2023-04-09T22:40:53.967841Z","iopub.status.idle":"2023-04-09T22:40:57.407867Z","shell.execute_reply.started":"2023-04-09T22:40:53.967809Z","shell.execute_reply":"2023-04-09T22:40:57.407118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shapely.geometry import Point\nimport geopandas as gpd\n\ngeometry = [Point(xy) for xy in zip(df['longitude'], df['latitude'])]\ngdf = gpd.GeoDataFrame(df, geometry=geometry)\nworld = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\nax = world.plot(figsize=(10, 6))\nax.set_axis_off()\nax.set_title('Distribution of recordings')\ngdf.plot(ax=ax, marker='o', color='pink', markersize=1);","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:40:57.409537Z","iopub.execute_input":"2023-04-09T22:40:57.410473Z","iopub.status.idle":"2023-04-09T22:41:01.061587Z","shell.execute_reply.started":"2023-04-09T22:40:57.410436Z","shell.execute_reply":"2023-04-09T22:41:01.060440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Just in kenya","metadata":{}},{"cell_type":"code","source":"kenya = world[world['name'] == 'Kenya']\nkenya_poly = kenya['geometry'].item()\ndf['is_in_kenya'] = [kenya_poly.contains(p) for p in geometry]\nkenya_df = df[df.is_in_kenya].reset_index()\nkenya_df","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.062642Z","iopub.execute_input":"2023-04-09T22:41:01.063108Z","iopub.status.idle":"2023-04-09T22:41:01.179873Z","shell.execute_reply.started":"2023-04-09T22:41:01.063080Z","shell.execute_reply":"2023-04-09T22:41:01.178768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kenya_primary_label_counts = kenya_df.primary_label.value_counts()\n\npx.bar(\n    x=kenya_primary_label_counts.keys(), \n    y=kenya_primary_label_counts.values,\n    title=\"Distribution of primary labels in kenya\",\n    labels={\"x\": \"bird\", \"y\": \"# of recordings\"},\n).show()\n\nkenya_primary_label_counts","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.182020Z","iopub.execute_input":"2023-04-09T22:41:01.182308Z","iopub.status.idle":"2023-04-09T22:41:01.236831Z","shell.execute_reply.started":"2023-04-09T22:41:01.182276Z","shell.execute_reply":"2023-04-09T22:41:01.235642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kenya_geometry = [Point(xy) for xy in zip(kenya_df['longitude'], kenya_df['latitude'])]\ngdf = gpd.GeoDataFrame(kenya_df, geometry=kenya_geometry)\nax = kenya.plot(figsize=(10, 6))\nax.set_axis_off()\nax.set_title('Distribution of recordings in Kenya')\ngdf.plot(ax=ax, marker='o', color='pink', markersize=1);","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.237850Z","iopub.execute_input":"2023-04-09T22:41:01.238433Z","iopub.status.idle":"2023-04-09T22:41:01.800390Z","shell.execute_reply.started":"2023-04-09T22:41:01.238404Z","shell.execute_reply":"2023-04-09T22:41:01.799253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The world compared to Kenya","metadata":{}},{"cell_type":"code","source":"counts_df = pd.concat([\n    pd.DataFrame({\n        \"label\": primary_label_counts.keys(), \n        \"num_recordings\": 100 * primary_label_counts.values / primary_label_counts.values.sum(),\n        \"place\": \"world\"\n    }),\n    pd.DataFrame({\n        \"label\": kenya_primary_label_counts.keys(), \n        \"num_recordings\": 100 * kenya_primary_label_counts.values / kenya_primary_label_counts.values.sum(),\n        \"place\": \"kenya\"\n    })\n])\n\npx.bar(\n    data_frame=counts_df,\n    x=\"label\", \n    y=\"num_recordings\",\n    color=\"place\",\n    title=\"Distribution of primary labels, as a % of the total recordings in the world or in keyna\",\n    barmode=\"group\",\n    labels={\"num_recordings\": \"% of recordings\"}\n).show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.801662Z","iopub.execute_input":"2023-04-09T22:41:01.801971Z","iopub.status.idle":"2023-04-09T22:41:01.883477Z","shell.execute_reply.started":"2023-04-09T22:41:01.801940Z","shell.execute_reply":"2023-04-09T22:41:01.882476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Just outside Kenya","metadata":{}},{"cell_type":"code","source":"nonkenya_primary_label_counts = df[-df.is_in_kenya].primary_label.value_counts()\n\ncounts_df = pd.concat([\n    pd.DataFrame({\n        \"label\": nonkenya_primary_label_counts.keys(), \n        \"num_recordings\": 100 * nonkenya_primary_label_counts.values / nonkenya_primary_label_counts.values.sum(),\n        \"place\": \"not keyna\"\n    }),\n    pd.DataFrame({\n        \"label\": kenya_primary_label_counts.keys(), \n        \"num_recordings\": 100 * kenya_primary_label_counts.values / kenya_primary_label_counts.values.sum(),\n        \"place\": \"kenya\"\n    })\n])\n\npx.bar(\n    data_frame=counts_df,\n    x=\"label\", \n    y=\"num_recordings\",\n    color=\"place\",\n    title=\"Distribution of primary labels, as a % of the total recordings not in kenya vs in keyna\",\n    barmode=\"group\",\n    labels={\"num_recordings\": \"% of recordings\"}\n).show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.884727Z","iopub.execute_input":"2023-04-09T22:41:01.885848Z","iopub.status.idle":"2023-04-09T22:41:01.955409Z","shell.execute_reply.started":"2023-04-09T22:41:01.885804Z","shell.execute_reply":"2023-04-09T22:41:01.954709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('birds exclusive to kenya')\nset(df[df.is_in_kenya].primary_label.unique()) - set(df[-df.is_in_kenya].primary_label.unique())","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.956330Z","iopub.execute_input":"2023-04-09T22:41:01.957360Z","iopub.status.idle":"2023-04-09T22:41:01.972003Z","shell.execute_reply.started":"2023-04-09T22:41:01.957328Z","shell.execute_reply":"2023-04-09T22:41:01.970609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Secondary labels","metadata":{}},{"cell_type":"code","source":"secondary_labels = df[df.secondary_labels != '[]'].reset_index()\nlabels = [\n    (lbls[2:-2].split(\"', '\"), is_in_kenya)\n    for lbls, is_in_kenya in zip(\n        secondary_labels.secondary_labels, \n        secondary_labels.is_in_kenya,\n    )\n]\nlabels = [(lbl, is_in_kenya) for lbls, is_in_kenya in labels for lbl in lbls]\nsecondary_label_data = pd.DataFrame({\n    'secondary_label': [e[0] for e in labels],\n    'is_in_kenya': [e[1] for e in labels],\n})\n\n\nkenya_primary = set(df[df.is_in_kenya].primary_label.unique())\nkenya_secondary = set(secondary_label_data[secondary_label_data.is_in_kenya].secondary_label)\nnot_kenya_primary = set(df[-df.is_in_kenya].primary_label.unique())\nnot_kenya_secondary = set(secondary_label_data[-secondary_label_data.is_in_kenya].secondary_label)\n\nprint('all kenya birds (including secondary ones):', len(kenya_primary | kenya_secondary))\nprint('birds only in kenya (including secondary ones):', len((kenya_primary | kenya_secondary) - (not_kenya_primary | not_kenya_secondary)))","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:01.973210Z","iopub.execute_input":"2023-04-09T22:41:01.973447Z","iopub.status.idle":"2023-04-09T22:41:01.999560Z","shell.execute_reply.started":"2023-04-09T22:41:01.973422Z","shell.execute_reply":"2023-04-09T22:41:01.998379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"secondary_labels = df[df.secondary_labels != '[]'].reset_index()\nprint (len(secondary_labels) / len(df))\nsecondary_labels","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:02.003471Z","iopub.execute_input":"2023-04-09T22:41:02.003737Z","iopub.status.idle":"2023-04-09T22:41:02.035843Z","shell.execute_reply.started":"2023-04-09T22:41:02.003712Z","shell.execute_reply":"2023-04-09T22:41:02.034597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [e[2:-2].split(\"', '\") for e in secondary_labels.secondary_labels]\nlabels = [e for li in labels for e in li]\nsecondary_label_counts = pd.DataFrame({'secondary_label': labels}).secondary_label.value_counts()\n\npx.bar(\n    x=secondary_label_counts.keys(), \n    y=secondary_label_counts.values,\n    title=\"Distribution of secondary labels\",\n    labels={\"x\": \"bird\", \"y\": \"# of recordings\"},\n).show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:02.038836Z","iopub.execute_input":"2023-04-09T22:41:02.039095Z","iopub.status.idle":"2023-04-09T22:41:02.092606Z","shell.execute_reply.started":"2023-04-09T22:41:02.039070Z","shell.execute_reply":"2023-04-09T22:41:02.091547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primarey_secondary_counts_df = pd.concat([\n    pd.DataFrame({\n        \"label\": primary_label_counts.keys(), \n        \"num_recordings\": primary_label_counts.values,\n        \"type\": \"primary\"\n    }),\n    pd.DataFrame({\n        \"label\": secondary_label_counts.keys(), \n        \"num_recordings\": secondary_label_counts.values,\n        \"type\": \"secondary\"\n    })\n])\n\npx.bar(\n    data_frame=primarey_secondary_counts_df,\n    x=\"label\", \n    y=\"num_recordings\",\n    color=\"type\",\n    title=\"Distribution of primary vs secondary labels\",\n    barmode=\"group\",\n    labels={\"num_recordings\": \"# of recordings\"}\n).show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:02.093849Z","iopub.execute_input":"2023-04-09T22:41:02.094117Z","iopub.status.idle":"2023-04-09T22:41:02.159742Z","shell.execute_reply.started":"2023-04-09T22:41:02.094090Z","shell.execute_reply":"2023-04-09T22:41:02.158752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_birds = pd.DataFrame({\n    \"label\": primary_label_counts.keys(), \n    \"num_primary_recordings\": primary_label_counts.values,\n}).set_index('label').join(\n    pd.DataFrame({\n        \"label\": secondary_label_counts.keys(), \n        \"num_secondary_recordings\": secondary_label_counts.values,\n    }).set_index('label'),\n)\n\nprint('birds that are more frequent as secondary labels:')\nall_birds[all_birds.num_primary_recordings < all_birds.num_secondary_recordings]","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:02.160901Z","iopub.execute_input":"2023-04-09T22:41:02.162199Z","iopub.status.idle":"2023-04-09T22:41:02.184544Z","shell.execute_reply.started":"2023-04-09T22:41:02.162130Z","shell.execute_reply":"2023-04-09T22:41:02.183589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio length","metadata":{}},{"cell_type":"code","source":"import torchaudio\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nimport os\n\ntrain_path = '/kaggle/input/birdclef-2023/train_audio/'\n\nmetadatas = Parallel(n_jobs=os.cpu_count())(\n    delayed(lambda filename: torchaudio.info(train_path + filename))(filename) \n    for filename in tqdm(df.filename)\n)\n    \ndf['num_frames'] = [m.num_frames for m in metadatas]\n\n(\n    set([m.sample_rate for m in metadatas]),\n    set([m.encoding for m in metadatas]),\n    set([m.num_channels for m in metadatas]),\n    set([m.bits_per_sample for m in metadatas]),\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:41:02.185706Z","iopub.execute_input":"2023-04-09T22:41:02.185969Z","iopub.status.idle":"2023-04-09T22:44:12.812820Z","shell.execute_reply.started":"2023-04-09T22:41:02.185945Z","shell.execute_reply":"2023-04-09T22:44:12.811789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_rate = metadatas[0].sample_rate\nnum_samples = df['num_frames'].sum()\nnum_hours = num_samples / sample_rate / 60 / 60\nmax_min = df['num_frames'].max() / sample_rate / 60\nprint('totale # of samples:', num_samples)\nprint('total hours:', num_hours)\nminutes = df['num_frames'] / sample_rate / 60\nminutes.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:26.490760Z","iopub.execute_input":"2023-04-09T22:45:26.491102Z","iopub.status.idle":"2023-04-09T22:45:26.506596Z","shell.execute_reply.started":"2023-04-09T22:45:26.491072Z","shell.execute_reply":"2023-04-09T22:45:26.505415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(pd.DataFrame({\"minutes\": minutes}), x=\"minutes\", title=\"Distribution of recording lengths (minutes)\")","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:26.887010Z","iopub.execute_input":"2023-04-09T22:45:26.887401Z","iopub.status.idle":"2023-04-09T22:45:27.298388Z","shell.execute_reply.started":"2023-04-09T22:45:26.887344Z","shell.execute_reply":"2023-04-09T22:45:27.297320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primary_label_frames = df.groupby('primary_label').num_frames.sum()\n\nprimarey_secondary_counts_df = pd.concat([\n    pd.DataFrame({\n        \"label\": primary_label_counts.keys(), \n        \"percent\": 100 * primary_label_counts.values / primary_label_counts.values.sum(),\n        \"aggregation\": \"num_recordings\"\n    }),\n    pd.DataFrame({\n        \"label\": primary_label_frames.keys(), \n        \"percent\": 100 * primary_label_frames.values / primary_label_frames.values.sum(),\n        \"aggregation\": \"num_samples\"\n    })\n])\n\npx.bar(\n    data_frame=primarey_secondary_counts_df,\n    x=\"label\", \n    y=\"percent\",\n    color=\"aggregation\",\n    title=\"Distribution of primary labels, # of recordings and sum of samples\",\n    barmode=\"group\",\n).show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:27.372798Z","iopub.execute_input":"2023-04-09T22:45:27.373259Z","iopub.status.idle":"2023-04-09T22:45:27.445715Z","shell.execute_reply.started":"2023-04-09T22:45:27.373225Z","shell.execute_reply":"2023-04-09T22:45:27.444079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_samples_kenya = df[df.is_in_kenya].num_frames.sum()\nnum_hours_kenya = num_samples_kenya / sample_rate / 60 / 60\nmax_min_kenya = df[df.is_in_kenya].num_frames.max() / sample_rate / 60\nprint('totale # of kenya samples:', num_samples_kenya)\nprint('total kenya hours:', num_hours_kenya)\nminutes_kenya = df[df.is_in_kenya].num_frames / sample_rate / 60\nminutes_kenya.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:28.120879Z","iopub.execute_input":"2023-04-09T22:45:28.121278Z","iopub.status.idle":"2023-04-09T22:45:28.142920Z","shell.execute_reply.started":"2023-04-09T22:45:28.121241Z","shell.execute_reply":"2023-04-09T22:45:28.141484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, rate = torchaudio.load(train_path + df.filename.iloc[0])\nbytes_per_sample = data.element_size()\ntotal_gigs = df.num_frames.sum() * bytes_per_sample / 2**30\ntotal_kenya_gigs = df[df.is_in_kenya].num_frames.sum() * bytes_per_sample / 2**30\ntotal_gigs, total_kenya_gigs","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:28.733454Z","iopub.execute_input":"2023-04-09T22:45:28.734628Z","iopub.status.idle":"2023-04-09T22:45:28.806805Z","shell.execute_reply.started":"2023-04-09T22:45:28.734581Z","shell.execute_reply":"2023-04-09T22:45:28.805856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How filtering effects dataset size","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\npx.line(\n    title='Accumulated size of dataset (sorting by filesize)',\n    y=np.cumsum(sorted(df.num_frames * 4)) / 2**30,\n    labels={\"x\": \"# of files\", \"y\": \"Dataset size (GB)\"},\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:44.580249Z","iopub.execute_input":"2023-04-09T22:45:44.580614Z","iopub.status.idle":"2023-04-09T22:45:44.692552Z","shell.execute_reply.started":"2023-04-09T22:45:44.580580Z","shell.execute_reply":"2023-04-09T22:45:44.690809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"line = px.line(\n    np.cumsum(4 * df.groupby('rating').num_frames.sum()[::-1]) / 2 ** 30,\n    title='Accumulated size of dataset (sorting by rating, reversed)',\n    labels={\"value\": \"Dataset size (GB)\"},\n)\nline.layout.update(showlegend=False)\nline.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:45:59.023602Z","iopub.execute_input":"2023-04-09T22:45:59.024784Z","iopub.status.idle":"2023-04-09T22:45:59.096354Z","shell.execute_reply.started":"2023-04-09T22:45:59.024732Z","shell.execute_reply":"2023-04-09T22:45:59.095402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving metadata","metadata":{}},{"cell_type":"code","source":"df.to_csv('./train_metadata_with_kenya_and_num_frames.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T22:46:12.225353Z","iopub.execute_input":"2023-04-09T22:46:12.225702Z","iopub.status.idle":"2023-04-09T22:46:13.633461Z","shell.execute_reply.started":"2023-04-09T22:46:12.225673Z","shell.execute_reply":"2023-04-09T22:46:13.632135Z"},"trusted":true},"execution_count":null,"outputs":[]}]}