{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport os\nimport librosa\nimport numpy as np\nimport pandas as pd\n\nimport gc\nimport dataclasses\nfrom concurrent.futures import ThreadPoolExecutor\nfrom typing import Optional, Callable, Tuple, List\nimport traceback  # Import traceback module","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:47:52.440362Z","iopub.execute_input":"2025-05-05T23:47:52.440746Z","iopub.status.idle":"2025-05-05T23:47:52.447859Z","shell.execute_reply.started":"2025-05-05T23:47:52.440718Z","shell.execute_reply":"2025-05-05T23:47:52.446483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define file paths \ntest_data = \"/kaggle/input/birdclef-2025/test_soundscapes\"\nsubmission = \"/kaggle/input/birdclef-2025/sample_submission.csv\"\ntrain_csv = \"/kaggle/input/birdclef-2025/train.csv\"\ntaxonomy_csv = \"/kaggle/input/birdclef-2025/taxonomy.csv\"\n\ntransform: Optional[Callable] = None  # Type hint for transform\naudio_transform: Optional[Callable] = None # Type hint for audio_transform\n\n@dataclasses.dataclass\nclass AudioParam:\n    SR: int = 32_000  # Sample rate\n    NFFT: int = 2048  # Number of FFT points\n    NMEL: int = 128   # Number of Mel bands\n    FMAX: int = 16_000 # Maximum frequency\n    FMIN: int = 20   # Minimum frequency\n    HOP_LENGTH: int = NFFT // 4  # Hop length\n\naudio_param = AudioParam()\n\n# Load submission CSV to get class names\ntry:\n    sub_csv = pd.read_csv(submission)\n    idx2cls = sub_csv.columns.drop(\"row_id\").tolist()  # List of bird species (class names)\n    cls2idx = {c: i for i, c in enumerate(idx2cls)} # Class name to index mapping\nexcept FileNotFoundError as e:\n    print(f\"Error: sample_submission.csv not found! {e}\")\n    idx2cls = [] # Provide a default for testing, but the code will likely fail\n    cls2idx = {}\n\n\nDEBUG = True # Enable Debugging\nfile_names = [os.path.join(test_data, fp) for fp in os.listdir(test_data) if fp.endswith(\".ogg\")]\n\n# Use a single file for debugging.  This makes the matrix dimension calculations easier.\nif len(file_names) == 0:\n    file_names = [\n        \"/kaggle/input/birdclef-2025/train_soundscapes/H02_20230420_074000.ogg\",\n    ]\n    DEBUG = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:47:52.726437Z","iopub.execute_input":"2025-05-05T23:47:52.726854Z","iopub.status.idle":"2025-05-05T23:47:52.758708Z","shell.execute_reply.started":"2025-05-05T23:47:52.726828Z","shell.execute_reply":"2025-05-05T23:47:52.757580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#  a simpler, randomly initialized CNN model\nclass SimpleCNN(nn.Module):\n    def __init__(self, num_classes: int = 1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.flatten = nn.Flatten()\n\n        # Calculate the input size to the linear layer dynamically\n        self._to_linear = None  # Placeholder, will be calculated during the first forward pass\n        self.fc1 = nn.Linear(1, num_classes)  # Placeholder Linear layer\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        try:\n            x = self.pool1(self.relu1(self.conv1(x)))\n            x = self.pool2(self.relu2(self.conv2(x)))\n            x = self.flatten(x)\n\n            # Dynamically determine the input size of the linear layer\n            if self._to_linear is None:\n                self._to_linear = x.shape[1]\n                if self._to_linear == 0:\n                   print(\"Error: self._to_linear is zero!\")\n                   #Handle this better - e.g., skip or set a min size\n                   return torch.zeros((1, len(idx2cls)))  # or a zero tensor of the right size\n                self.fc1 = nn.Linear(self._to_linear, len(idx2cls))  # Update the linear layer\n            x = self.fc1(x)\n            return x\n        except Exception as e:\n            print(f\"Error in SimpleCNN.forward: {e}\")\n            return torch.zeros((1, len(idx2cls)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:47:53.036468Z","iopub.execute_input":"2025-05-05T23:47:53.036782Z","iopub.status.idle":"2025-05-05T23:47:53.047570Z","shell.execute_reply.started":"2025-05-05T23:47:53.036760Z","shell.execute_reply":"2025-05-05T23:47:53.046546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Instantiate the SimpleCNN model.\nmodel = SimpleCNN(num_classes=len(idx2cls))\nmodel.eval() # Set the model to evaluation mode.\n\ndef pipeline(x: np.ndarray) -> np.ndarray:\n    \"\"\"\n    Converts audio data to MFCCs (Mel-frequency cepstral coefficients).\n    \"\"\"\n    try:\n        mfccs = librosa.feature.mfcc(\n            y=x,\n            sr=audio_param.SR,\n            n_mfcc=audio_param.NMEL,  # Using the same dimension as previously defined for mels\n            n_fft=audio_param.NFFT,\n            hop_length=audio_param.HOP_LENGTH,\n            fmin=audio_param.FMIN,\n            fmax=audio_param.FMAX\n        )\n        # Normalize the MFCCs\n        mfccs_normalized = (mfccs - np.mean(mfccs)) / (np.std(mfccs) + 1e-6)\n        if np.isnan(mfccs_normalized).any():\n            print(\"Warning: NaN values detected in mfccs!\")\n            mfccs_normalized = np.nan_to_num(mfccs_normalized)  # Replace with 0\n        return mfccs_normalized[None, :, :]  # Add a channel dimension (1, height, width)\n    except Exception as e:\n        print(f\"Error in pipeline: {e}\")\n        return np.zeros((1, audio_param.NMEL, 1))  # return a zero array","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:47:56.298398Z","iopub.execute_input":"2025-05-05T23:47:56.298790Z","iopub.status.idle":"2025-05-05T23:47:56.318706Z","shell.execute_reply.started":"2025-05-05T23:47:56.298764Z","shell.execute_reply":"2025-05-05T23:47:56.317251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_submission(file_names):\n    \"\"\"\n    Generate predictions for the test files and create a submission CSV\n    formatted according to Kaggle's requirements.\n    \n    Args:\n        file_names (list): List of file paths to predict on\n    \"\"\"\n    row_id = []\n    matrix = []\n\n    # Function to process a single file for thread pooling\n    def process_file(fp):\n        try:\n            out, rid = predict(fp)\n            if len(rid) > 0:  # Only return if there are valid results\n                return out, rid\n            else:\n                print(f\"Warning: No predictions generated for file: {fp}\")\n                return None, None\n        except Exception as e:\n            print(f\"Failed to run predict for file {fp}: {e}\")\n            return None, None\n\n    # Using a ThreadPoolExecutor to parallelize the predictions\n    with ThreadPoolExecutor(max_workers=4) as executor:\n        futures = [executor.submit(process_file, fp) for fp in file_names]\n        \n        # Process results as they complete\n        for fp_idx, future in enumerate(as_completed(futures)):\n            try:\n                out, rid = future.result()\n                if rid is not None and len(rid) > 0:\n                    row_id.extend(rid)  # Extend to add all row IDs from this file\n                    matrix.extend(out)  # Extend to add all predictions from this file\n                gc.collect()  # Collect garbage after processing each file's results\n                print(f\"Finished {fp_idx+1}/{len(file_names)}\")  # Track progress\n            except Exception as e:\n                print(f\"Error processing result: {e}\")\n\n    try:\n        if len(row_id) > 0 and len(matrix) > 0:\n            # CRITICAL FIX: Correct the row_id format to match Kaggle's requirements\n            # Extract the test file basename without extension and use the timestamp\n            corrected_row_ids = []\n            for r in row_id:\n                # Original format: \"filename_start_end\" \n                # Required format: \"filename_end\" (just the end timestamp)\n                parts = r.split('_')\n                filename = parts[0]\n                end_time = parts[2]  # Get the end timestamp\n                corrected_row_id = f\"{filename}_{end_time}\"\n                corrected_row_ids.append(corrected_row_id)\n            \n            # Convert predictions to numpy array\n            matrix_np = np.array(matrix)\n            \n            # Ensure we have exactly the expected classes from the competition\n            # First, create a dataframe with just row_ids\n            submission_df = pd.DataFrame({\"row_id\": corrected_row_ids})\n            \n            # Add each class column correctly\n            for i, cls_name in enumerate(idx2cls):\n                submission_df[cls_name] = matrix_np[:, i]\n            \n            # Verify columns match expected format\n            expected_columns = [\"row_id\"] + idx2cls\n            missing_cols = set(expected_columns) - set(submission_df.columns)\n            if missing_cols:\n                print(f\"Warning: Missing columns in submission: {missing_cols}\")\n                # Add missing columns with zeros\n                for col in missing_cols:\n                    if col != \"row_id\":  # Skip row_id as it should already be there\n                        submission_df[col] = 0.0\n            \n            # Ensure columns are in the correct order\n            submission_df = submission_df[expected_columns]\n            \n            # Save to CSV\n            submission_df.to_csv('submission.csv', index=False)\n            print(\"Submission file created successfully!\")\n            print(f\"Shape: {submission_df.shape}\")\n            print(submission_df.head())\n            \n            # Validate submission format\n            print(\"\\nValidating submission format...\")\n            print(f\"Number of rows: {len(submission_df)}\")\n            print(f\"Number of columns: {len(submission_df.columns)}\")\n            print(f\"Missing values: {submission_df.isnull().sum().sum()}\")\n            \n        else:\n            print(\"Error: No valid predictions were generated.\")\n    except Exception as e:\n        print(f\"Error creating submission file: {e}\")\n        traceback.print_exc()  # Print full traceback for debugging\n\n    print(\"Finished!\")\n    gc.collect()\n\n# Updated predict function to fix row_ID format\n@torch.no_grad()\ndef predict(fp: str) -> tuple[np.ndarray, list[str]]:\n    \"\"\"\n    Predicts bird calls in a given audio file using MFCC features.\n    Args:\n        fp (str): File path of the audio file.\n    Returns:\n        Tuple[np.ndarray, List[str]]: Tuple containing the model output and the list of row IDs.\n    \"\"\"\n    try:\n        # Fix the syntax error in loading audio\n        x, sr = librosa.load(fp, sr=audio_param.SR)  # Load the audio file.\n        if x.size == 0:\n            print(f\"Warning: Audio file {fp} is empty!\")\n            return np.array([]), [] # return empty arrays\n    except Exception as e:\n        print(f\"Error loading file {fp}: {e}\")\n        return np.array([]), []\n        \n    # Number of 5-second segments\n    num_segments = int(np.floor(len(x) / audio_param.SR / 5))\n    all_outs = []\n    all_row_ids = []\n    \n    for i in range(num_segments):\n        start = i * audio_param.SR * 5\n        end = (i + 1) * audio_param.SR * 5\n        segment = x[start:end]\n        \n        # Apply audio transforms if defined\n        if 'audio_transform' in globals() and audio_transform is not None:\n            try:\n                segment = audio_transform(sample=segment, sample_rate=audio_param.SR) # Apply audio transform\n            except Exception as e:\n                print(f\"Audio Transform Failed {e}\")\n                \n        try:\n            # This will now use the updated pipeline that extracts MFCCs instead of mel spectrograms\n            segment_features = pipeline(segment)  # Convert waveform to MFCCs\n        except Exception as e:\n            print(f\"Pipeline failed {e}\")\n            continue\n            \n        # Apply image transforms if defined\n        if 'transform' in globals() and transform is not None:\n            try:\n                segment_features = transform(image=segment_features)[\"image\"] # Apply image transform\n            except Exception as e:\n                print(f\"Transform failed {e}\")\n                continue\n                \n        try:\n            # Convert to tensor and add batch dimension\n            segment_tensor = torch.from_numpy(segment_features).float().unsqueeze(0)\n            \n            # Get the model output\n            out = model(segment_tensor).sigmoid().detach().cpu().numpy()\n            all_outs.append(out[0])\n            \n            # Extract the base filename and create row ID\n            # KAGGLE FORMAT: Use just the filename and end time\n            fp_name = os.path.basename(fp).split(\".\")[0]\n            end_time = (i + 1) * 5  # End time in seconds\n            row_id = f\"{fp_name}_{end_time}\"  # Format matches Kaggle requirements: \"filename_endtime\"\n            all_row_ids.append(row_id)\n        except Exception as e:\n            print(f\"Error during processing of segment {i} in {fp}: {e}\\n{traceback.format_exc()}\")  # Print trace\n            \n    return np.array(all_outs), all_row_ids  # return all values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:49:45.882851Z","iopub.execute_input":"2025-05-05T23:49:45.883177Z","iopub.status.idle":"2025-05-05T23:49:45.902663Z","shell.execute_reply.started":"2025-05-05T23:49:45.883157Z","shell.execute_reply":"2025-05-05T23:49:45.901272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import the necessary modules\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nimport traceback\n\nrow_id = []\nmatrix = []\n\n# Function to process a single file for thread pooling\ndef process_file(fp):\n    try:\n        out, rid = predict(fp)\n        if len(rid) > 0:  # Only return if there are valid results\n            return out, rid\n        else:\n            print(f\"Warning: No predictions generated for file: {fp}\")\n            return None, None\n    except Exception as e:\n        print(f\"Failed to run predict for file {fp}: {e}\")\n        return None, None\n\n# Using a ThreadPoolExecutor to parallelize the predictions\nwith ThreadPoolExecutor(max_workers=4) as executor:\n    futures = [executor.submit(process_file, fp) for fp in file_names]\n    \n    # Process results as they complete\n    for fp_idx, future in enumerate(as_completed(futures)):\n        try:\n            out, rid = future.result()\n            if rid is not None and len(rid) > 0:\n                row_id.extend(rid)  # Extend to add all row IDs from this file\n                matrix.extend(out)  # Extend to add all predictions from this file\n            gc.collect()  # Collect garbage after processing each file's results\n            print(f\"Finished {fp_idx+1}/{len(file_names)}\")  # Track progress\n        except Exception as e:\n            print(f\"Error processing result: {e}\")\n\ntry:\n    if len(row_id) > 0 and len(matrix) > 0:\n        # Convert row_id to string type to ensure it's handled correctly\n        row_id = [str(r) for r in row_id]\n        \n        # Create DataFrame directly from dictionary\n        sub = pd.DataFrame({\n            \"row_id\": row_id,\n            **{cls_name: [pred[i] for pred in matrix] for i, cls_name in enumerate(idx2cls)}\n        })\n        \n        # Save to CSV\n        sub.to_csv('submission.csv', index=False)\n        print(\"Submission file created successfully!\")\n        print(sub.head())\n    else:\n        print(\"Error: No valid predictions were generated.\")\nexcept Exception as e:\n    print(f\"Error creating submission file: {e}\")\n    traceback.print_exc()  # Print full traceback for debugging\n\nprint(\"Finished!\")  # If you see this, then great!\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T23:49:46.255257Z","iopub.execute_input":"2025-05-05T23:49:46.255590Z","iopub.status.idle":"2025-05-05T23:49:47.404331Z","shell.execute_reply.started":"2025-05-05T23:49:46.255569Z","shell.execute_reply":"2025-05-05T23:49:47.403531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}