{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Comprehensive Data Analysis","metadata":{}},{"cell_type":"markdown","source":"In this notebook I planned to write comprehensive code for the BirdCLEF 2025 competition. The notebook will be updated daily.","metadata":{}},{"cell_type":"markdown","source":"This notebook contains a thorough exploration of the BirdCLEF+ 2025 competition dataset, analyzing both metadata and audio characteristics to inform our modeling approach.\n\n## Table of Contents\n1. [Setup](#1-setup)\n2. [Load & Explore Metadata](#2-load--explore-metadata)\n   - [Basic Statistics & Data Types](#21-basic-statistics--data-types)\n   - [Primary Label Analysis](#22-primary-label-analysis)\n   - [Secondary Labels Analysis](#23-secondary-labels-analysis)\n   - [Taxonomy Analysis](#24-taxonomy-analysis)\n   - [Geographic Distribution](#25-geographic-distribution)\n   - [Rating Distribution](#26-rating-distribution)\n   - [Collection Sources](#27-collection-sources)\n   - [Missing Values](#28-missing-values)\n3. [Audio Exploration](#3-audio-exploration)\n   - [Audio Loading](#31-audio-loading)\n   - [Duration Analysis](#32-duration-analysis)\n   - [Waveform Analysis](#33-waveform-analysis)\n   - [Spectrogram Analysis](#34-spectrogram-analysis)\n   - [Audio Quality Assessment](#35-audio-quality-assessment)\n4. [Unlabeled Soundscapes](#4-unlabeled-soundscapes)\n   - [Structure Analysis](#41-structure-analysis)\n   - [Audio Characteristics](#42-audio-characteristics)\n5. [Feature Engineering Insights](#5-feature-engineering-insights)\n6. [Validation Strategy](#6-validation-strategy)\n7. [Summary & Insights](#7-summary--insights)","metadata":{}},{"cell_type":"markdown","source":"## 1. Setup\n\nLet's start by importing the necessary libraries and setting up our environment.","metadata":{}},{"cell_type":"code","source":"\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport librosa\nimport librosa.display\nimport soundfile as sf\nimport json\nimport ast\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom collections import Counter\nimport warnings\nimport IPython.display as ipd\n\n# Set seeds for reproducibility\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\n# Configure visualization settings\nplt.style.use('fivethirtyeight')\nsns.set(font_scale=1.2)\nwarnings.filterwarnings('ignore')\n\n# Directory paths - adjust as needed\nBASE_DIR = '/kaggle/input/birdclef-2025'  # Kaggle path\nif not os.path.exists(BASE_DIR):\n    BASE_DIR = './birdclef-plus-2025'  # Local path\n\nTRAIN_AUDIO_DIR = os.path.join(BASE_DIR, 'train_audio')\nTRAIN_SOUNDSCAPE_DIR = os.path.join(BASE_DIR, 'train_soundscapes')\nTEST_SOUNDSCAPE_DIR = os.path.join(BASE_DIR, 'test_soundscapes')\n\n# Constants\nSAMPLE_RATE = 32000  # Competition specifies 32kHz\nN_CLASSES = 206  # Total number of species classes\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:35:46.573294Z","iopub.execute_input":"2025-05-03T15:35:46.573598Z","iopub.status.idle":"2025-05-03T15:35:46.780698Z","shell.execute_reply.started":"2025-05-03T15:35:46.573578Z","shell.execute_reply":"2025-05-03T15:35:46.779706Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Load & Explore Metadata\nLet's load the metadata files and explore their basic characteristics.","metadata":{}},{"cell_type":"code","source":"# Load metadata\ntrain_df = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\ntaxonomy_df = pd.read_csv(os.path.join(BASE_DIR, 'taxonomy.csv'))\nsample_submission = pd.read_csv(os.path.join(BASE_DIR, 'sample_submission.csv'))\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Taxonomy data shape: {taxonomy_df.shape}\")\nprint(f\"Sample submission shape: {sample_submission.shape}\")\n\n### 2.1 Basic Statistics & Data Types\n\n#Let's examine the structure of our metadata files and get basic statistics.\n\n\n# Display column info\nprint(\"Train columns:\", train_df.columns.tolist())\nprint(\"Taxonomy columns:\", taxonomy_df.columns.tolist())\nprint(\"Sample submission columns:\", sample_submission.columns.tolist())\n\n# Check data types\nprint(\"\\nTrain data types:\")\nprint(train_df.dtypes)\n\nprint(\"\\nTaxonomy data types:\")\nprint(taxonomy_df.dtypes)\n\n# Basic statistics\nprint(\"\\nTrain data description:\")\ntrain_df.describe(include='all').T\n\n#Now let's check for missing values in our metadata.\n# Check for missing values\nprint(\"Missing values in train data:\")\nprint(train_df.isnull().sum())\n\nprint(\"\\nMissing values in taxonomy data:\")\nprint(taxonomy_df.isnull().sum())\n\n# Get a sample of rows\nprint(\"\\nSample rows from train data:\")\ntrain_df.sample(5)\n\nprint(\"\\nSample rows from taxonomy data:\")\ntaxonomy_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:37:54.042736Z","iopub.execute_input":"2025-05-03T15:37:54.043222Z","iopub.status.idle":"2025-05-03T15:37:54.275909Z","shell.execute_reply.started":"2025-05-03T15:37:54.043191Z","shell.execute_reply":"2025-05-03T15:37:54.274967Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2 Primary Label Analysis\nLet's analyze the distribution of primary labels (species).","metadata":{}},{"cell_type":"code","source":"# Count primary labels\nprimary_label_counts = train_df['primary_label'].value_counts()\nprint(f\"Number of unique primary labels: {len(primary_label_counts)}\")\n\n# Plot distribution\nplt.figure(figsize=(20, 8))\nax = sns.barplot(x=primary_label_counts.index[:30], y=primary_label_counts.values[:30])\nplt.title('Distribution of Top 30 Primary Labels')\nplt.xticks(rotation=90)\nplt.xlabel('Primary Label')\nplt.ylabel('Count')\nplt.tight_layout()\nplt.show()\n\n# Show statistics about class imbalance\nprint(f\"Most common species: {primary_label_counts.index[0]} with {primary_label_counts.values[0]} samples\")\nprint(f\"Least common species: {primary_label_counts.index[-1]} with {primary_label_counts.values[-1]} samples\")\nprint(f\"Imbalance ratio (most common / least common): {primary_label_counts.values[0] / primary_label_counts.values[-1]:.2f}\")\n\n# Analyze the long tail\nplt.figure(figsize=(12, 6))\nplt.plot(range(len(primary_label_counts)), sorted(primary_label_counts.values, reverse=True))\nplt.title('Species Sample Count (Sorted)')\nplt.xlabel('Species Rank')\nplt.ylabel('Number of Samples')\nplt.grid(True)\nplt.show()\n\n# Determine rare classes (< 10 samples)\nrare_classes = primary_label_counts[primary_label_counts < 10]\nprint(f\"Number of rare classes (<10 samples): {len(rare_classes)}\")\nprint(f\"Rare classes: {rare_classes.to_dict()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:38:43.816442Z","iopub.execute_input":"2025-05-03T15:38:43.816776Z","iopub.status.idle":"2025-05-03T15:38:44.651046Z","shell.execute_reply.started":"2025-05-03T15:38:43.816751Z","shell.execute_reply":"2025-05-03T15:38:44.650060Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.3 Secondary Labels Analysis\n\nLet's examine the secondary labels and their format.","metadata":{}},{"cell_type":"code","source":"# Check the format of secondary_labels\nprint(\"First few secondary_labels:\")\nfor i in range(5):\n    print(f\"{i+1}: {train_df['secondary_labels'].iloc[i]}\")\n\n# Parse secondary_labels (assuming they're in string representation of list format)\ndef parse_secondary_labels(label_str):\n    if pd.isna(label_str):\n        return []\n    try:\n        return ast.literal_eval(label_str)\n    except:\n        return []\n\ntrain_df['parsed_secondary_labels'] = train_df['secondary_labels'].apply(parse_secondary_labels)\n\n# Count how many recordings have secondary labels\nhas_secondary = train_df['parsed_secondary_labels'].apply(lambda x: len(x) > 0)\nprint(f\"\\nRecordings with secondary labels: {has_secondary.sum()} ({has_secondary.sum()/len(train_df)*100:.2f}%)\")\n\n# Get distribution of number of secondary labels per recording\nsecondary_counts = train_df['parsed_secondary_labels'].apply(len)\nplt.figure(figsize=(10, 6))\nsns.countplot(x=secondary_counts)\nplt.title('Number of Secondary Labels per Recording')\nplt.xlabel('Count of Secondary Labels')\nplt.ylabel('Number of Recordings')\nplt.grid(True)\nplt.show()\n\n# Most common secondary labels\nall_secondary = []\nfor labels in train_df['parsed_secondary_labels']:\n    all_secondary.extend(labels)\n    \nsecondary_counter = Counter(all_secondary)\nprint(\"\\nTop 10 most common secondary labels:\")\nfor label, count in secondary_counter.most_common(10):\n    print(f\"{label}: {count}\")\n\n# Create multi-label representation\ndef create_multilabel_vector(row):\n    # Initialize with primary label\n    labels = [row['primary_label']]\n    # Add secondary labels if they exist\n    if not pd.isna(row['secondary_labels']):\n        try:\n            secondary = ast.literal_eval(row['secondary_labels'])\n            if isinstance(secondary, list):\n                labels.extend(secondary)\n        except:\n            pass\n    return labels\n\ntrain_df['all_labels'] = train_df.apply(create_multilabel_vector, axis=1)\n\n# Count total labels per recording\ntrain_df['total_label_count'] = train_df['all_labels'].apply(len)\nprint(f\"\\nAverage number of labels per recording: {train_df['total_label_count'].mean():.2f}\")\nprint(f\"Maximum number of labels in a recording: {train_df['total_label_count'].max()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:39:51.047452Z","iopub.execute_input":"2025-05-03T15:39:51.048203Z","iopub.status.idle":"2025-05-03T15:39:52.212747Z","shell.execute_reply.started":"2025-05-03T15:39:51.048177Z","shell.execute_reply":"2025-05-03T15:39:52.211874Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.4 Taxonomy Analysis\n\nLet's explore the taxonomy information and how it relates to the labels.","metadata":{}},{"cell_type":"code","source":"# Analyze class_name distribution in taxonomy\nclass_counts = taxonomy_df['class_name'].value_counts()\nprint(\"Distribution of classes:\")\nprint(class_counts)\n\n# Plot class distribution\nplt.figure(figsize=(10, 6))\nsns.barplot(x=class_counts.index, y=class_counts.values)\nplt.title('Distribution of Taxonomic Classes')\nplt.xticks(rotation=45)\nplt.xlabel('Class')\nplt.ylabel('Count')\nplt.grid(True)\nplt.show()\n\n# Merge train data with taxonomy for further analysis\ntrain_with_taxonomy = pd.merge(train_df, taxonomy_df, on='primary_label', how='left')\n\n# Check for any missing matches\nmissing_taxonomy = train_with_taxonomy[train_with_taxonomy['class_name'].isna()]\nif len(missing_taxonomy) > 0:\n    print(f\"Warning: {len(missing_taxonomy)} training samples don't have matching taxonomy entries\")\n    print(missing_taxonomy['primary_label'].unique())\n\n# Analyze recordings by taxonomic class\nclass_recording_counts = train_with_taxonomy['class_name'].value_counts()\nplt.figure(figsize=(12, 6))\nsns.barplot(x=class_recording_counts.index, y=class_recording_counts.values)\nplt.title('Number of Recordings by Taxonomic Class')\nplt.xticks(rotation=45)\nplt.xlabel('Class')\nplt.ylabel('Number of Recordings')\nplt.grid(True)\nplt.show()\n\n# Check for discrepancies between train and taxonomy scientific names\nscientific_name_match = train_with_taxonomy['scientific_name_x'] == train_with_taxonomy['scientific_name_y']\nif not scientific_name_match.all():\n    print(f\"Warning: {(~scientific_name_match).sum()} records have mismatched scientific names\")\n    mismatched = train_with_taxonomy[~scientific_name_match][['primary_label', 'scientific_name_x', 'scientific_name_y']]\n    print(mismatched.sample(min(5, len(mismatched))))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:40:28.471714Z","iopub.execute_input":"2025-05-03T15:40:28.472056Z","iopub.status.idle":"2025-05-03T15:40:28.910395Z","shell.execute_reply.started":"2025-05-03T15:40:28.472035Z","shell.execute_reply":"2025-05-03T15:40:28.909318Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.5 Geographic Distribution\n\nLet's analyze the geographic distribution of recordings.","metadata":{}},{"cell_type":"code","source":"# Check how many recordings have location data\nhas_location = ~(train_df['latitude'].isna() | train_df['longitude'].isna())\nprint(f\"Recordings with location data: {has_location.sum()} ({has_location.sum()/len(train_df)*100:.2f}%)\")\n\n# Plot geographic distribution of recordings\nplt.figure(figsize=(15, 10))\nplt.scatter(\n    train_df[has_location]['longitude'], \n    train_df[has_location]['latitude'],\n    alpha=0.5,\n    s=5\n)\nplt.title('Geographic Distribution of Recordings')\nplt.xlabel('Longitude')\nplt.ylabel('Latitude')\nplt.grid(True)\nplt.axhline(y=0, color='r', linestyle='-', alpha=0.3)  # Equator\nplt.axvline(x=0, color='r', linestyle='-', alpha=0.3)  # Prime Meridian\nplt.tight_layout()\nplt.show()\n\n# Let's try to get a rough idea of continental distribution\ndef assign_continent(lat, lon):\n    if pd.isna(lat) or pd.isna(lon):\n        return \"Unknown\"\n    \n    # Very rough approximation\n    if -30 <= lat <= 75 and -20 <= lon <= 40:\n        return \"Europe/Africa\"\n    elif -30 <= lat <= 75 and 40 <= lon <= 180:\n        return \"Asia/Australia\"\n    elif -60 <= lat <= 75 and -170 <= lon <= -20:\n        return \"Americas\"\n    else:\n        return \"Other\"\n\ntrain_df['continent'] = train_df.apply(lambda row: assign_continent(row['latitude'], row['longitude']), axis=1)\n\n# Plot continental distribution\nplt.figure(figsize=(10, 6))\nsns.countplot(y=train_df['continent'])\nplt.title('Approximate Continental Distribution of Recordings')\nplt.xlabel('Count')\nplt.ylabel('Region')\nplt.grid(True)\nplt.show()\n\n# Analyze species by continent\nif 'class_name' in train_with_taxonomy.columns:\n    plt.figure(figsize=(12, 8))\n    continent_class = pd.crosstab(train_with_taxonomy['continent'], train_with_taxonomy['class_name'])\n    continent_class_norm = continent_class.div(continent_class.sum(axis=1), axis=0)\n    sns.heatmap(continent_class_norm, cmap='viridis', annot=True, fmt='.2f')\n    plt.title('Proportion of Taxonomic Classes by Region')\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:41:29.456068Z","iopub.execute_input":"2025-05-03T15:41:29.456915Z","iopub.status.idle":"2025-05-03T15:41:30.373084Z","shell.execute_reply.started":"2025-05-03T15:41:29.456882Z","shell.execute_reply":"2025-05-03T15:41:30.371742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.6 Rating Distribution\n\nLet's analyze the rating distribution and its relationship to other variables.","metadata":{}},{"cell_type":"code","source":"# Rating distribution\nplt.figure(figsize=(10, 6))\nsns.countplot(x=train_df['rating'])\nplt.title('Distribution of Ratings')\nplt.xlabel('Rating')\nplt.ylabel('Count')\nplt.grid(True)\nplt.show()\n\nprint(f\"Average rating: {train_df['rating'].mean():.2f}\")\nprint(f\"Median rating: {train_df['rating'].median()}\")\n\n# Rating by collection source\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='collection', y='rating', data=train_df)\nplt.title('Rating Distribution by Collection Source')\nplt.xlabel('Collection Source')\nplt.ylabel('Rating')\nplt.grid(True)\nplt.show()\n\n# Rating by taxonomic class (if merged)\nif 'class_name' in train_with_taxonomy.columns:\n    plt.figure(figsize=(14, 8))\n    sns.boxplot(x='class_name', y='rating', data=train_with_taxonomy)\n    plt.title('Rating Distribution by Taxonomic Class')\n    plt.xlabel('Class')\n    plt.ylabel('Rating')\n    plt.xticks(rotation=45)\n    plt.grid(True)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:42:49.672631Z","iopub.execute_input":"2025-05-03T15:42:49.673922Z","iopub.status.idle":"2025-05-03T15:42:50.605785Z","shell.execute_reply.started":"2025-05-03T15:42:49.673875Z","shell.execute_reply":"2025-05-03T15:42:50.604961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.7 Collection Sources\n\nLet's analyze the distribution of collection sources and their characteristics.","metadata":{}},{"cell_type":"code","source":"# Collection source distribution\nplt.figure(figsize=(10, 6))\ncollection_counts = train_df['collection'].value_counts()\nsns.barplot(x=collection_counts.index, y=collection_counts.values)\nplt.title('Distribution of Collection Sources')\nplt.xlabel('Collection Source')\nplt.ylabel('Count')\nplt.grid(True)\nplt.show()\n\nprint(f\"Collection source distribution:\")\nfor source, count in collection_counts.items():\n    print(f\"{source}: {count} ({count/len(train_df)*100:.2f}%)\")\n\n# Species distribution by collection source\ncollection_species = pd.crosstab(train_df['collection'], train_df['primary_label'])\ncollection_species_norm = collection_species.div(collection_species.sum(axis=1), axis=0)\n\n# Top 10 species per collection source\nfor source in collection_counts.index:\n    print(f\"\\nTop 10 species in {source}:\")\n    top_species = collection_species.loc[source].sort_values(ascending=False).head(10)\n    for species, count in top_species.items():\n        print(f\"{species}: {count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:43:22.477335Z","iopub.execute_input":"2025-05-03T15:43:22.478153Z","iopub.status.idle":"2025-05-03T15:43:22.705890Z","shell.execute_reply.started":"2025-05-03T15:43:22.478125Z","shell.execute_reply":"2025-05-03T15:43:22.705011Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.8 Missing Values\n\nLet's do a more detailed analysis of missing values and their patterns.","metadata":{}},{"cell_type":"code","source":"# Calculate percentage of missing values per column\nmissing_percentage = (train_df.isnull().sum() / len(train_df) * 100).sort_values(ascending=False)\nprint(\"Percentage of missing values per column:\")\nfor col, percentage in missing_percentage.items():\n    if percentage > 0:\n        print(f\"{col}: {percentage:.2f}%\")\n\n# Visualize missing values\nplt.figure(figsize=(12, 8))\nsns.heatmap(train_df.isnull(), cbar=False, yticklabels=False, cmap='viridis')\nplt.title('Missing Values Heatmap')\nplt.tight_layout()\nplt.show()\n\n# Check for patterns in missing values\n# For example, are missing ratings correlated with specific collections?\nif train_df['rating'].isnull().sum() > 0:\n    missing_ratings_by_collection = train_df[train_df['rating'].isnull()]['collection'].value_counts(normalize=True)\n    total_by_collection = train_df['collection'].value_counts(normalize=True)\n    \n    print(\"\\nProportion of missing ratings by collection source:\")\n    comparison = pd.DataFrame({\n        'Missing Ratings %': missing_ratings_by_collection * 100,\n        'Overall %': total_by_collection * 100\n    })\n    print(comparison)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:43:58.613401Z","iopub.execute_input":"2025-05-03T15:43:58.613709Z","iopub.status.idle":"2025-05-03T15:43:59.509120Z","shell.execute_reply.started":"2025-05-03T15:43:58.613687Z","shell.execute_reply":"2025-05-03T15:43:59.508206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Audio Exploration\n\nNow let's explore the audio files themselves.\nFirst, let's create a function to load and analyze audio files","metadata":{}},{"cell_type":"code","source":"def load_audio_file(file_path, sr=SAMPLE_RATE):\n    \"\"\"Load an audio file and return the signal.\"\"\"\n    try:\n        audio, _ = librosa.load(file_path, sr=sr)\n        return audio\n    except Exception as e:\n        print(f\"Error loading {file_path}: {e}\")\n        return None\n\n# Get list of audio files\ntrain_audio_files = []\nfor root, _, files in os.walk(TRAIN_AUDIO_DIR):\n    for file in files:\n        if file.endswith('.ogg'):\n            train_audio_files.append(os.path.join(root, file))\n\nprint(f\"Total number of training audio files: {len(train_audio_files)}\")\n\n# Sample a few random files for analysis\nrandom.seed(SEED)\nsample_audio_files = random.sample(train_audio_files, min(5, len(train_audio_files)))\n\n# Load sample files\nsample_audios = []\nfor file_path in sample_audio_files:\n    audio = load_audio_file(file_path)\n    if audio is not None:\n        filename = os.path.basename(file_path)\n        label = filename.split('_')[0]  # Assuming filename format starts with label\n        sample_audios.append({\n            'path': file_path,\n            'audio': audio,\n            'duration': len(audio) / SAMPLE_RATE,\n            'filename': filename,\n            'label': label\n        })\n\nprint(f\"Loaded {len(sample_audios)} sample audio files\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:44:55.614458Z","iopub.execute_input":"2025-05-03T15:44:55.615316Z","iopub.status.idle":"2025-05-03T15:45:44.764514Z","shell.execute_reply.started":"2025-05-03T15:44:55.615280Z","shell.execute_reply":"2025-05-03T15:45:44.763589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.2 Duration Analysis\n\nLet's analyze the duration of audio recordings.","metadata":{}},{"cell_type":"code","source":"# Function to get audio duration\ndef get_audio_duration(file_path, sr=SAMPLE_RATE):\n    try:\n        audio, _ = librosa.load(file_path, sr=sr)\n        return len(audio) / sr\n    except Exception as e:\n        return None\n\n# Sample audio files for duration analysis (this can be slow, so we'll sample)\nduration_sample_size = min(500, len(train_audio_files))\nduration_sample_files = random.sample(train_audio_files, duration_sample_size)\n\n# Get durations\nprint(f\"Calculating durations for {duration_sample_size} files...\")\ndurations = []\nfor file_path in tqdm(duration_sample_files):\n    duration = get_audio_duration(file_path)\n    if duration is not None:\n        durations.append(duration)\n\n# Plot duration distribution\nplt.figure(figsize=(12, 6))\nplt.hist(durations, bins=50)\nplt.title('Distribution of Audio Durations')\nplt.xlabel('Duration (seconds)')\nplt.ylabel('Count')\nplt.grid(True)\nplt.show()\n\nprint(f\"Duration statistics:\")\nprint(f\"Mean duration: {np.mean(durations):.2f} seconds\")\nprint(f\"Median duration: {np.median(durations):.2f} seconds\")\nprint(f\"Min duration: {np.min(durations):.2f} seconds\")\nprint(f\"Max duration: {np.max(durations):.2f} seconds\")\n\n# Check for very short recordings (< 1s)\nshort_recordings = [d for d in durations if d < 1]\nprint(f\"Very short recordings (<1s): {len(short_recordings)} ({len(short_recordings)/len(durations)*100:.2f}%)\")\n\n# Check for long recordings (> 60s)\nlong_recordings = [d for d in durations if d > 60]\nprint(f\"Long recordings (>60s): {len(long_recordings)} ({len(long_recordings)/len(durations)*100:.2f}%)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:46:25.492010Z","iopub.execute_input":"2025-05-03T15:46:25.492596Z","iopub.status.idle":"2025-05-03T15:46:50.718805Z","shell.execute_reply.started":"2025-05-03T15:46:25.492570Z","shell.execute_reply":"2025-05-03T15:46:50.717706Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.3 Waveform Analysis\n\nLet's visualize waveforms for our sample audio files.","metadata":{}},{"cell_type":"code","source":"# Plot waveforms for sample files\nfor i, sample in enumerate(sample_audios):\n    plt.figure(figsize=(14, 3))\n    librosa.display.waveshow(sample['audio'], sr=SAMPLE_RATE)\n    plt.title(f\"Waveform for {sample['filename']} ({sample['label']}, {sample['duration']:.2f}s)\")\n    plt.tight_layout()\n    plt.show()\n    \n    # Play the audio (uncomment in notebook environment)\n    # display(ipd.Audio(sample['audio'], rate=SAMPLE_RATE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:47:35.698222Z","iopub.execute_input":"2025-05-03T15:47:35.698498Z","iopub.status.idle":"2025-05-03T15:47:38.284224Z","shell.execute_reply.started":"2025-05-03T15:47:35.698481Z","shell.execute_reply":"2025-05-03T15:47:38.283374Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.4 Spectrogram Analysis\n\nLet's create and visualize spectrograms for our sample audio files.","metadata":{}},{"cell_type":"code","source":"# Function to create mel spectrogram\ndef create_mel_spectrogram(audio, sr=SAMPLE_RATE, n_mels=128, fmin=20, fmax=16000):\n    mel_spec = librosa.feature.melspectrogram(\n        y=audio, \n        sr=sr, \n        n_mels=n_mels,\n        fmin=fmin,\n        fmax=fmax\n    )\n    # Convert to dB scale\n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n    return mel_spec_db\n\n# Create and display spectrograms\nfor i, sample in enumerate(sample_audios):\n    mel_spec = create_mel_spectrogram(sample['audio'])\n    \n    plt.figure(figsize=(14, 5))\n    librosa.display.specshow(\n        mel_spec, \n        sr=SAMPLE_RATE, \n        x_axis='time', \n        y_axis='mel', \n        fmin=20,\n        fmax=16000\n    )\n    plt.colorbar(format='%+2.0f dB')\n    plt.title(f\"Mel Spectrogram for {sample['filename']} ({sample['label']})\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:48:07.026143Z","iopub.execute_input":"2025-05-03T15:48:07.026734Z","iopub.status.idle":"2025-05-03T15:48:13.823364Z","shell.execute_reply.started":"2025-05-03T15:48:07.026709Z","shell.execute_reply":"2025-05-03T15:48:13.822495Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.5 Audio Quality Assessment\n\nLet's assess the audio quality of our samples.","metadata":{}},{"cell_type":"code","source":"# Calculate signal-to-noise ratio (SNR) - simple approximation\ndef estimate_snr(audio):\n    # Simple SNR estimation\n    signal_power = np.mean(audio**2)\n    noise_power = np.var(audio)\n    if noise_power == 0:\n        return float('inf')\n    snr = 10 * np.log10(signal_power / noise_power)\n    return snr\n\n# Calculate SNR for sample files\nfor sample in sample_audios:\n    snr = estimate_snr(sample['audio'])\n    print(f\"Estimated SNR for {sample['filename']}: {snr:.2f} dB\")\n\n# Find recordings from train_df with different ratings\nif len(train_df) > 0:\n    # Get files with different ratings\n    rating_examples = {}\n    for rating in sorted(train_df['rating'].unique()):\n        if pd.isna(rating):\n            continue\n        examples = train_df[train_df['rating'] == rating].head(2)\n        for _, row in examples.iterrows():\n            file_path = os.path.join(TRAIN_AUDIO_DIR, row['primary_label'], row['filename'])\n            if os.path.exists(file_path):\n                audio = load_audio_file(file_path)\n                if audio is not None:\n                    if rating not in rating_examples:\n                        rating_examples[rating] = []\n                    rating_examples[rating].append({\n                        'path': file_path,\n                        'audio': audio,\n                        'duration': len(audio) / SAMPLE_RATE,\n                        'filename': row['filename'],\n                        'label': row['primary_label']\n                    })\n    \n    # Plot waveforms and spectrograms for different ratings\n    for rating, examples in rating_examples.items():\n        for example in examples:\n            plt.figure(figsize=(14, 3))\n            librosa.display.waveshow(example['audio'], sr=SAMPLE_RATE)\n            plt.title(f\"Waveform for Rating {rating}: {example['filename']} ({example['label']})\")\n            plt.tight_layout()\n            plt.show()\n            \n            mel_spec = create_mel_spectrogram(example['audio'])\n            plt.figure(figsize=(14, 5))\n            librosa.display.specshow(\n                mel_spec, \n                sr=SAMPLE_RATE, \n                x_axis='time', \n                y_axis='mel', \n                fmin=20,\n                fmax=16000\n            )\n            plt.colorbar(format='%+2.0f dB')\n            plt.title(f\"Mel Spectrogram for Rating {rating}: {example['filename']} ({example['label']})\")\n            plt.tight_layout()\n            plt.show()\n            \n            # Play the audio (uncomment in notebook environment)\n            # display(ipd.Audio(example['audio'], rate=SAMPLE_RATE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:48:53.011207Z","iopub.execute_input":"2025-05-03T15:48:53.011848Z","iopub.status.idle":"2025-05-03T15:48:53.103250Z","shell.execute_reply.started":"2025-05-03T15:48:53.011806Z","shell.execute_reply":"2025-05-03T15:48:53.102173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Unlabeled Soundscapes\n\nLet's explore the unlabeled soundscapes provided for the competition.","metadata":{}},{"cell_type":"code","source":"# Check soundscape files\nsoundscape_files = []\nfor root, _, files in os.walk(TRAIN_SOUNDSCAPE_DIR):\n    for file in files:\n        if file.endswith('.ogg'):\n            soundscape_files.append(os.path.join(root, file))\n\nprint(f\"Total number of training soundscape files: {len(soundscape_files)}\")\n\n# Sample soundscape files\nsample_soundscape_files = random.sample(soundscape_files, min(3, len(soundscape_files)))\nsample_soundscapes = []\n\nfor file_path in sample_soundscape_files:\n    audio = load_audio_file(file_path)\n    if audio is not None:\n        filename = os.path.basename(file_path)\n        sample_soundscapes.append({\n            'path': file_path,\n            'audio': audio,\n            'duration': len(audio) / SAMPLE_RATE,\n            'filename': filename\n        })\n\nprint(f\"Loaded {len(sample_soundscapes)} sample soundscape files\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:49:25.481961Z","iopub.execute_input":"2025-05-03T15:49:25.482266Z","iopub.status.idle":"2025-05-03T15:49:35.898948Z","shell.execute_reply.started":"2025-05-03T15:49:25.482245Z","shell.execute_reply":"2025-05-03T15:49:35.897901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4.2 Audio Characteristics","metadata":{}},{"cell_type":"code","source":"# Analyze soundscape durations\nfor sample in sample_soundscapes:\n    print(f\"Soundscape {sample['filename']} duration: {sample['duration']:.2f} seconds\")\n\n# Plot waveforms and spectrograms for sample soundscapes\nfor sample in sample_soundscapes:\n    plt.figure(figsize=(14, 3))\n    librosa.display.waveshow(sample['audio'], sr=SAMPLE_RATE)\n    plt.title(f\"Waveform for Soundscape: {sample['filename']}\")\n    plt.tight_layout()\n    plt.show()\n    \n    # Create and plot 5-second chunks to simulate test environment\n    chunk_duration = 5  # seconds\n    chunk_samples = chunk_duration * SAMPLE_RATE\n    num_chunks = int(np.floor(len(sample['audio']) / chunk_samples))\n    \n    print(f\"Number of 5-second chunks in {sample['filename']}: {num_chunks}\")\n    \n    # Plot first two chunks as examples\n    for i in range(min(2, num_chunks)):\n        chunk_start = i * chunk_samples\n        chunk_end = (i + 1) * chunk_samples\n        chunk = sample['audio'][chunk_start:chunk_end]\n        \n        plt.figure(figsize=(10, 3))\n        librosa.display.waveshow(chunk, sr=SAMPLE_RATE)\n        plt.title(f\"Waveform for 5s Chunk {i+1} of {sample['filename']}\")\n        plt.tight_layout()\n        plt.show()\n        \n        mel_spec = create_mel_spectrogram(chunk)\n        plt.figure(figsize=(10, 5))\n        librosa.display.specshow(\n            mel_spec, \n            sr=SAMPLE_RATE, \n            x_axis='time', \n            y_axis='mel', \n            fmin=20,\n            fmax=16000\n        )\n        plt.colorbar(format='%+2.0f dB')\n        plt.title(f\"Mel Spectrogram for 5s Chunk {i+1} of {sample['filename']}\")\n        plt.tight_layout()\n        plt.show()\n        \n        # Play the chunk (uncomment in notebook environment)\n        # display(ipd.Audio(chunk, rate=SAMPLE_RATE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:50:01.482115Z","iopub.execute_input":"2025-05-03T15:50:01.482404Z","iopub.status.idle":"2025-05-03T15:50:09.072177Z","shell.execute_reply.started":"2025-05-03T15:50:01.482386Z","shell.execute_reply":"2025-05-03T15:50:09.071164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Feature Engineering Insights\n\nBased on our EDA, let's summarize some insights for feature engineering.","metadata":{}},{"cell_type":"code","source":"# Spectrogram parameter exploration\ndef explore_spectrogram_params(audio, sr=SAMPLE_RATE):\n    \"\"\"Explore different spectrogram parameters on a sample audio\"\"\"\n    # Try different n_mels values\n    n_mels_values = [64, 128, 256]\n    fmax_values = [8000, 16000]\n    \n    for n_mels in n_mels_values:\n        for fmax in fmax_values:\n            plt.figure(figsize=(10, 4))\n            mel_spec = librosa.feature.melspectrogram(\n                y=audio, \n                sr=sr, \n                n_mels=n_mels,\n                fmin=20,\n                fmax=fmax\n            )\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n            \n            librosa.display.specshow(\n                mel_spec_db, \n                sr=sr, \n                x_axis='time', \n                y_axis='mel', \n                fmin=20,\n                fmax=fmax\n            )\n            plt.colorbar(format='%+2.0f dB')\n            plt.title(f\"Mel Spectrogram: n_mels={n_mels}, fmax={fmax}\")\n            plt.tight_layout()\n            plt.show()\n\n# Run parameter exploration on a sample audio (choose one with clear vocalizations)\nif len(sample_audios) > 0:\n    # Find a sample with good vocalization (using the first one for simplicity)\n    explore_spectrogram_params(sample_audios[0]['audio'])\n\n# Try different time/frequency masks (SpecAugment-like)\ndef show_augmentation_examples(audio, sr=SAMPLE_RATE):\n    \"\"\"Show examples of different augmentation techniques\"\"\"\n    # Original spectrogram\n    mel_spec_orig = librosa.feature.melspectrogram(y=audio, sr=sr, n_mels=128)\n    mel_spec_db_orig = librosa.power_to_db(mel_spec_orig, ref=np.max)\n    \n    # Time masking\n    mel_spec_time = mel_spec_orig.copy()\n    mask_size = 25  # Adjust based on your spectrogram shape\n    start = np.random.randint(0, mel_spec_time.shape[1] - mask_size)\n    mel_spec_time[:, start:start+mask_size] = 0\n    mel_spec_db_time = librosa.power_to_db(mel_spec_time, ref=np.max)\n    \n    # Frequency masking\n    mel_spec_freq = mel_spec_orig.copy()\n    mask_size = 20  # Adjust based on your spectrogram shape\n    start = np.random.randint(0, mel_spec_freq.shape[0] - mask_size)\n    mel_spec_freq[start:start+mask_size, :] = 0\n    mel_spec_db_freq = librosa.power_to_db(mel_spec_freq, ref=np.max)\n    \n    # Plot all three\n    fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n    \n    librosa.display.specshow(\n        mel_spec_db_orig, \n        sr=sr, \n        x_axis='time', \n        y_axis='mel', \n        ax=axes[0]\n    )\n    axes[0].set_title(\"Original\")\n    \n    librosa.display.specshow(\n        mel_spec_db_time, \n        sr=sr, \n        x_axis='time', \n        y_axis='mel', \n        ax=axes[1]\n    )\n    axes[1].set_title(\"Time Masking\")\n    \n    librosa.display.specshow(\n        mel_spec_db_freq, \n        sr=sr, \n        x_axis='time', \n        y_axis='mel', \n        ax=axes[2]\n    )\n    axes[2].set_title(\"Frequency Masking\")\n    \n    plt.tight_layout()\n    plt.show()\n\n# Show augmentation examples on a sample\nif len(sample_audios) > 0:\n    show_augmentation_examples(sample_audios[0]['audio'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:50:43.553541Z","iopub.execute_input":"2025-05-03T15:50:43.554390Z","iopub.status.idle":"2025-05-03T15:50:49.567177Z","shell.execute_reply.started":"2025-05-03T15:50:43.554364Z","shell.execute_reply":"2025-05-03T15:50:49.566127Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Validation Strategy\n\nLet's implement and visualize our validation strategy.","metadata":{}},{"cell_type":"code","source":"# Implement stratified k-fold validation\nn_folds = 5\nskf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED)\n\n# Get counts for each class\nclass_counts = train_df['primary_label'].value_counts()\n\n# Add a 'fold' column to the train_df\ntrain_df['fold'] = -1\n\n# Apply stratified k-fold\nfor fold_idx, (train_idx, val_idx) in enumerate(skf.split(train_df, train_df['primary_label'])):\n    train_df.loc[val_idx, 'fold'] = fold_idx\n\n# Check fold distribution\nfold_class_counts = []\nfor fold in range(n_folds):\n    fold_counts = train_df[train_df['fold'] == fold]['primary_label'].value_counts()\n    fold_class_counts.append(fold_counts)\n\n# Plot fold distribution for top 20 classes\ntop_20_classes = class_counts.index[:20]\nfold_dist_df = pd.DataFrame(index=top_20_classes)\n\nfor fold in range(n_folds):\n    fold_dist_df[f'Fold {fold}'] = fold_class_counts[fold].reindex(top_20_classes).fillna(0)\n\n# Convert to percentage of original class count\nfor cls in top_20_classes:\n    fold_dist_df.loc[cls] = fold_dist_df.loc[cls] / class_counts[cls] * 100\n\n# Plot the distribution\nplt.figure(figsize=(15, 10))\nfold_dist_df.plot(kind='bar', figsize=(15, 10))\nplt.title('Distribution of Classes Across Folds (% of Total Class Count)')\nplt.xlabel('Class')\nplt.ylabel('Percentage in Fold')\nplt.legend(title='Fold')\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()\n\n# Check for classes with zero samples in any fold\nzero_in_fold = []\nfor cls in train_df['primary_label'].unique():\n    for fold in range(n_folds):\n        if cls not in fold_class_counts[fold]:\n            zero_in_fold.append((cls, fold))\n\nif zero_in_fold:\n    print(f\"Warning: {len(zero_in_fold)} class-fold combinations have zero samples\")\n    for cls, fold in zero_in_fold[:10]:  # Show just the first 10\n        print(f\"Class {cls} has zero samples in fold {fold}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:51:29.568002Z","iopub.execute_input":"2025-05-03T15:51:29.568365Z","iopub.status.idle":"2025-05-03T15:51:30.328789Z","shell.execute_reply.started":"2025-05-03T15:51:29.568339Z","shell.execute_reply":"2025-05-03T15:51:30.327925Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Summary & Insights\n\nLet's summarize our key findings from the EDA.","metadata":{}},{"cell_type":"code","source":"# Summary of key EDA findings\n\nprint(\"=\" * 50)\nprint(\"BirdCLEF+ 2025 EDA Summary\")\nprint(\"=\" * 50)\n\n# Dataset size\nprint(f\"\\n1. Dataset Size:\")\nprint(f\"   - Training samples: {len(train_df)}\")\nprint(f\"   - Total classes: {N_CLASSES}\")\nprint(f\"   - Training audio files: {len(train_audio_files)}\")\nprint(f\"   - Unlabeled soundscapes: {len(soundscape_files)}\")\n\n# Class distribution\nprint(f\"\\n2. Class Distribution:\")\nprint(f\"   - Most common class: {primary_label_counts.index[0]} ({primary_label_counts.values[0]} samples)\")\nprint(f\"   - Least common class: {primary_label_counts.index[-1]} ({primary_label_counts.values[-1]} samples)\")\nprint(f\"   - Imbalance ratio: {primary_label_counts.values[0] / primary_label_counts.values[-1]:.2f}\")\nprint(f\"   - Classes with <10 samples: {len(rare_classes)}\")\n\n# Secondary labels\nhas_secondary = train_df['parsed_secondary_labels'].apply(lambda x: len(x) > 0)\nprint(f\"\\n3. Secondary Labels:\")\nprint(f\"   - Recordings with secondary labels: {has_secondary.sum()} ({has_secondary.sum()/len(train_df)*100:.2f}%)\")\nprint(f\"   - Average labels per recording: {train_df['total_label_count'].mean():.2f}\")\n\n# Taxonomy\nif 'class_name' in train_with_taxonomy.columns:\n    print(f\"\\n4. Taxonomy:\")\n    for cls, count in class_counts.items():\n        print(f\"   - {cls}: {count} species\")\n\n# Audio characteristics\nprint(f\"\\n5. Audio Characteristics:\")\nprint(f\"   - Average duration: {np.mean(durations):.2f} seconds\")\nprint(f\"   - Duration range: {np.min(durations):.2f} - {np.max(durations):.2f} seconds\")\nprint(f\"   - Sample rate: {SAMPLE_RATE} Hz\")\n\n# Metadata completeness\nprint(f\"\\n6. Metadata Completeness:\")\nfor col, percentage in missing_percentage.items():\n    if percentage > 0:\n        print(f\"   - {col}: {percentage:.2f}% missing\")\n\n# Collection sources\nprint(f\"\\n7. Collection Sources:\")\nfor source, count in collection_counts.items():\n    print(f\"   - {source}: {count} ({count/len(train_df)*100:.2f}%)\")\n\nprint(f\"\\n8. Geographic Coverage:\")\nhas_location = ~(train_df['latitude'].isna() | train_df['longitude'].isna())\nprint(f\"   - Recordings with location data: {has_location.sum()} ({has_location.sum()/len(train_df)*100:.2f}%)\")\n\nprint(f\"\\n9. Main Challenges:\")\nprint(f\"   - Class imbalance (potential solutions: class weighting, focal loss, data augmentation)\")\nprint(f\"   - Variable-length recordings (potential solutions: padding/chunking, attention mechanisms)\")\nprint(f\"   - Multi-label nature (requires appropriate loss function and evaluation metrics)\")\nprint(f\"   - Background noise (potential solutions: noise augmentation, robust feature extraction)\")\n\nprint(f\"\\n10. Validation Strategy:\")\nprint(f\"   - {n_folds}-fold stratified cross-validation based on primary_label\")\nif zero_in_fold:\n    print(f\"   - Warning: {len(zero_in_fold)} class-fold combinations have zero samples\")\n    print(f\"   - Consider using StratifiedGroupKFold or custom splitting for rare classes\")\n\nprint(\"\\n11. Next Steps:\")\nprint(\"   - Implement audio loading and preprocessing pipeline\")\nprint(\"   - Develop spectrogram generation with optimal parameters\")\nprint(\"   - Create multi-label target encoding\")\nprint(\"   - Implement data augmentation\")\nprint(\"   - Design and train baseline model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-03T15:52:12.184639Z","iopub.execute_input":"2025-05-03T15:52:12.185014Z","iopub.status.idle":"2025-05-03T15:52:12.207688Z","shell.execute_reply.started":"2025-05-03T15:52:12.184989Z","shell.execute_reply":"2025-05-03T15:52:12.206782Z"}},"outputs":[],"execution_count":null}]}