{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11590046,"sourceType":"datasetVersion","datasetId":7267557},{"sourceId":234880372,"sourceType":"kernelVersion"},{"sourceId":236105823,"sourceType":"kernelVersion"},{"sourceId":236675586,"sourceType":"kernelVersion"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install umap plotly umap-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-30T17:10:21.333389Z","iopub.execute_input":"2025-04-30T17:10:21.333635Z","iopub.status.idle":"2025-04-30T17:10:27.674817Z","shell.execute_reply.started":"2025-04-30T17:10:21.333609Z","shell.execute_reply":"2025-04-30T17:10:27.674126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport umap.umap_ as umap\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport librosa\nimport cv2\nfrom torch.serialization import add_safe_globals\nfrom numpy.core.multiarray import scalar as np_scalar\nfrom numpy import dtype as np_dtype\nfrom numpy.dtypes import Float64DType\n\n# Add numpy scalar and dtype to safe globals\nadd_safe_globals([np_scalar, np_dtype, Float64DType])\n\n# Define CFG class for safe loading\nclass CFG:\n    def __init__(self):\n        pass\n\n# Add CFG to safe globals\nadd_safe_globals([CFG])\n\nclass BaselineModel(nn.Module):\n    def __init__(self, model_name, in_channels=1):\n        super().__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=False,\n            in_chans=in_channels,\n            drop_rate=0.0,\n            drop_path_rate=0.0\n        )\n        \n        backbone_out = self.backbone.classifier.in_features\n        self.backbone.classifier = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.feat_dim = backbone_out\n        \n    def forward(self, x):\n        features = self.backbone(x)\n        if isinstance(features, dict):\n            features = features['features']\n        if len(features.shape) == 4:\n            features = self.pooling(features)\n            features = features.view(features.size(0), -1)\n        return features\n\n    def load_state_dict(self, state_dict, strict=True):\n        \"\"\"Custom load_state_dict that handles different state dict formats\"\"\"\n        if isinstance(state_dict, dict):\n            if 'state_dict' in state_dict:\n                state_dict = state_dict['state_dict']\n            elif 'model_state_dict' in state_dict:\n                state_dict = state_dict['model_state_dict']\n            elif 'model' in state_dict:\n                state_dict = state_dict['model']\n                \n        # Remove classifier weights from state_dict\n        state_dict_no_clf = {k: v for k, v in state_dict.items() \n                            if not k.startswith('classifier.')}\n        return super().load_state_dict(state_dict_no_clf, strict=False)\n\ndef audio2melspec(audio_data, cfg):\n    \"\"\"Convert audio data to mel spectrogram\"\"\"\n    if np.isnan(audio_data).any():\n        mean_signal = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_signal)\n\n    mel_spec = librosa.feature.melspectrogram(\n        y=audio_data,\n        sr=cfg.FS,\n        n_fft=cfg.N_FFT,\n        hop_length=cfg.HOP_LENGTH,\n        n_mels=cfg.N_MELS,\n        fmin=cfg.FMIN,\n        fmax=cfg.FMAX,\n        power=2.0\n    )\n\n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n    mel_spec_norm = (mel_spec_db - mel_spec_db.min()) / (mel_spec_db.max() - mel_spec_db.min() + 1e-8)\n    \n    return mel_spec_norm\n\ndef process_audio_file(audio_path, cfg):\n    \"\"\"Process a single audio file to extract features\"\"\"\n    try:\n        audio_data, _ = librosa.load(audio_path, sr=cfg.FS)\n        \n        # Take center 5 seconds if longer\n        target_samples = cfg.FS * cfg.WINDOW_SIZE\n        if len(audio_data) > target_samples:\n            start = (len(audio_data) - target_samples) // 2\n            audio_data = audio_data[start:start + target_samples]\n        else:\n            # Pad if shorter\n            audio_data = np.pad(audio_data, (0, target_samples - len(audio_data)), mode='constant')\n        \n        mel_spec = audio2melspec(audio_data, cfg)\n        \n        if mel_spec.shape != cfg.TARGET_SHAPE:\n            mel_spec = cv2.resize(mel_spec, cfg.TARGET_SHAPE, interpolation=cv2.INTER_LINEAR)\n        \n        return mel_spec.astype(np.float32)\n        \n    except Exception as e:\n        print(f\"Error processing {audio_path}: {e}\")\n        return None\n\ndef generate_embeddings(model, audio_files, cfg, device):\n    \"\"\"Generate embeddings for a list of audio files\"\"\"\n    model.eval()\n    embeddings = []\n    processed_files = []\n    labels = []\n    \n    for audio_file in tqdm(audio_files, desc=\"Generating embeddings\"):\n        try:\n            # Extract label from filepath (assuming format: species/file.ogg)\n            label = Path(audio_file).parent.name\n            \n            # Process audio to mel spectrogram\n            mel_spec = process_audio_file(audio_file, cfg)\n            if mel_spec is None:\n                continue\n                \n            # Convert to tensor\n            mel_spec = torch.tensor(mel_spec, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n            mel_spec = mel_spec.to(device)\n            \n            # Generate embedding\n            with torch.no_grad():\n                embedding = model(mel_spec)\n                embedding = embedding.cpu().numpy()\n            \n            embeddings.append(embedding[0])\n            processed_files.append(audio_file)\n            labels.append(label)\n            \n        except Exception as e:\n            print(f\"Error processing {audio_file}: {e}\")\n            continue\n    \n    return np.array(embeddings), processed_files, labels\n\ndef plot_embeddings(embeddings, labels, output_path):\n    \"\"\"Create visualization of embeddings\"\"\"\n    print(f\"Processing embeddings shape: {embeddings.shape}\")\n    \n    # Save original embeddings\n    embeddings_save_path = output_path.replace('.html', '_embeddings.npz')\n    print(f\"Saving original embeddings to {embeddings_save_path}...\")\n    np.savez_compressed(\n        embeddings_save_path,\n        embeddings=embeddings,\n        labels=np.array(labels)\n    )\n    \n    # Reduce dimensionality\n    print(\"Fitting UMAP...\")\n    reducer = umap.UMAP(n_components=3, random_state=42, metric='cosine', min_dist=0.1)\n    embeddings_umap = reducer.fit_transform(embeddings)\n    \n    # Save UMAP reduced embeddings\n    umap_save_path = output_path.replace('.html', '_umap.npz')\n    print(f\"Saving UMAP reduced embeddings to {umap_save_path}...\")\n    np.savez_compressed(\n        umap_save_path,\n        embeddings_umap=embeddings_umap,\n        labels=np.array(labels)\n    )\n    \n    print(\"Creating plotly figure...\")\n    # Create figure\n    fig = go.Figure()\n    \n    # Add traces for embeddings\n    unique_labels = sorted(list(set(labels)))\n    print(f\"Processing {len(unique_labels)} unique labels...\")\n    \n    # Create a color map\n    colors = px.colors.qualitative.Set3 * (len(unique_labels) // len(px.colors.qualitative.Set3) + 1)\n    color_map = {label: colors[i] for i, label in enumerate(unique_labels)}\n    \n    for label in tqdm(unique_labels, desc=\"Processing labels\"):\n        mask = [l == label for l in labels]\n        if sum(mask) > 0:  # Only add trace if there are points for this label\n            fig.add_trace(\n                go.Scatter3d(\n                    x=embeddings_umap[mask, 0],\n                    y=embeddings_umap[mask, 1],\n                    z=embeddings_umap[mask, 2],\n                    mode='markers',\n                    name=label,\n                    showlegend=True,\n                    marker=dict(\n                        size=4,\n                        color=color_map[label],\n                        opacity=0.7\n                    )\n                )\n            )\n    \n    # Update layout\n    fig.update_layout(\n        title=\"Embedding Space Visualization\",\n        height=800,\n        width=1000,\n        scene=dict(\n            xaxis_title=\"UMAP 1\",\n            yaxis_title=\"UMAP 2\",\n            zaxis_title=\"UMAP 3\",\n            camera=dict(\n                up=dict(x=0, y=0, z=1),\n                center=dict(x=0, y=0, z=0),\n                eye=dict(x=1.5, y=1.5, z=1.5)\n            )\n        )\n    )\n    \n    print(f\"Saving plot to {output_path}...\")\n    fig.write_html(output_path)\n    print(f\"Plot saved to {output_path}\")\n\ndef main():\n    # Configuration\n    class CFG:  #               /kaggle/input/pub-bird25-b-422-ppv15-v2-s-focallossbce\n        baseline_model_path = \"/kaggle/input/pub-bird25-b-422-ppv15-v2-s-focallossbce\"  # Update this path\n        train_audio_dir = \"/kaggle/input/birdclef-2025/train_audio\"                    # Update this path\n        output_dir = \"/kaggle/working/embeddings_visualization\"\n        output_path = os.path.join(output_dir, \"embedding_visualization.html\")\n        \n        # Model settings\n        model_name = \"tf_efficientnetv2_s.in21k_ft_in1k\"\n        in_channels = 1\n        device = 'cuda' if torch.cuda.is_available() else 'cpu'\n        \n        # Audio processing settings\n        FS = 32000\n        WINDOW_SIZE = 5\n        N_FFT = 2048\n        HOP_LENGTH =128\n        N_MELS = 512\n        FMIN = 20\n        FMAX = 16000\n        TARGET_SHAPE = (256, 256)\n        max_samples_per_class = 100\n        min_samples_per_class = 10\n    \n    cfg = CFG()\n    \n    # Create output directory if it doesn't exist\n    os.makedirs(cfg.output_dir, exist_ok=True)\n    \n    # Load models\n    print(\"Loading models...\")\n    models = []\n    model_files = list(Path(cfg.baseline_model_path).glob('*.pth'))\n    \n    if not model_files:\n        print(f\"No model files found in {cfg.baseline_model_path}\")\n        return\n        \n    print(f\"Found {len(model_files)} model files\")\n    \n    for model_path in model_files:\n        try:\n            print(f\"Loading model: {model_path}\")\n            model = BaselineModel(cfg.model_name, in_channels=cfg.in_channels).to(cfg.device)\n            checkpoint = torch.load(str(model_path), map_location=cfg.device)\n            model.load_state_dict(checkpoint)\n            model.eval()\n            models.append(model)\n        except Exception as e:\n            print(f\"Error loading model {model_path}: {e}\")\n            continue\n    \n    if not models:\n        print(\"No models could be loaded successfully!\")\n        return\n        \n    print(f\"Successfully loaded {len(models)} models\")\n    \n    # Get audio files\n    print(\"Collecting audio files...\")\n    audio_files = []\n    for species_dir in Path(cfg.train_audio_dir).iterdir():\n        if species_dir.is_dir():\n            files = list(species_dir.glob(\"*.ogg\"))\n            if len(files) >= cfg.min_samples_per_class:\n                # Randomly sample if more than max_samples_per_class\n                if len(files) > cfg.max_samples_per_class:\n                    files = np.random.choice(files, cfg.max_samples_per_class, replace=False)\n                audio_files.extend(files)\n    \n    # Generate embeddings using ensemble of models\n    print(\"Generating embeddings...\")\n    all_embeddings = []\n    processed_files = []\n    labels = []\n    \n    for audio_file in tqdm(audio_files, desc=\"Processing audio files\"):\n        try:\n            # Extract label from filepath\n            label = audio_file.parent.name\n            \n            # Process audio to mel spectrogram\n            audio_data, _ = librosa.load(str(audio_file), sr=cfg.FS)\n            \n            # Take center 5 seconds if longer\n            target_samples = cfg.FS * cfg.WINDOW_SIZE\n            if len(audio_data) > target_samples:\n                start = (len(audio_data) - target_samples) // 2\n                audio_data = audio_data[start:start + target_samples]\n            else:\n                # Pad if shorter\n                audio_data = np.pad(audio_data, (0, target_samples - len(audio_data)), mode='constant')\n            \n            mel_spec = process_audio_file(str(audio_file), cfg)\n            mel_spec = torch.tensor(mel_spec, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n            mel_spec = mel_spec.to(cfg.device)\n            \n            # Get embeddings from all models\n            file_embeddings = []\n            with torch.no_grad():\n                for model in models:\n                    embedding = model(mel_spec)\n                    file_embeddings.append(embedding.cpu().numpy())\n            \n            # Average embeddings from all models\n            avg_embedding = np.mean(file_embeddings, axis=0)\n            all_embeddings.append(avg_embedding[0])\n            processed_files.append(str(audio_file))\n            labels.append(label)\n            \n        except Exception as e:\n            print(f\"Error processing {audio_file}: {e}\")\n            continue\n    \n    embeddings = np.array(all_embeddings)\n    \n    # Create visualization\n    print(\"Creating visualization...\")\n    plot_embeddings(embeddings, labels, cfg.output_path)\n    \n    print(\"Done!\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-30T17:43:24.462269Z","iopub.execute_input":"2025-04-30T17:43:24.462837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}