{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11421803,"sourceType":"datasetVersion","datasetId":7153191}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install noisereduce pedalboard\n!pip download --extra-index-url https://developer.download.nvidia.com/compute/redist/nightly nvidia-dali-nightly-cuda120\n!ls /kaggle/working |grep nvidia_dali_nightly_cuda120 |xargs pip install \nimport numpy as np\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\nimport os, gc, random \nimport pandas as pd\nimport pickle\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\nimport IPython.display as ipd\nfrom IPython.display import display, clear_output\nimport ipywidgets as widgets\nimport librosa\nimport librosa.display\nimport soundfile as sf\nimport noisereduce as nr\nfrom pedalboard import Pedalboard, Gain\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, accuracy_score, confusion_matrix\n\nfrom nvidia.dali import pipeline_def\nimport nvidia.dali.fn as fn\nimport nvidia.dali.types as types\nimport nvidia.dali as dali\n\nfrom PIL import Image\nimport imagehash\nfrom tqdm import tqdm\nfrom itertools import combinations\nfrom multiprocessing import Pool, cpu_count\nfrom collections import defaultdict\n\n# !rm -rf /kaggle/working/\nclear_output()\nprint(\"Install and Import DONE\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:16:20.445708Z","iopub.execute_input":"2025-05-01T03:16:20.446419Z","iopub.status.idle":"2025-05-01T03:17:29.251359Z","shell.execute_reply.started":"2025-05-01T03:16:20.446393Z","shell.execute_reply":"2025-05-01T03:17:29.250555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nclass Config:\n    def __init__(self, **kwargs):\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n    def update(self, **kwargs):\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n# Initialize and set basic configuration\ncfg = Config(\n    COMPARED_CLASS = '963335', # 0,1,2,3 \n    SEED=42, \n    SAMPLE_RATE=32000,\n    DATA_PATH=Path(\"/kaggle/input/birdclef-2025/train_audio\"),\n    OUTPUT_FOLDER =Path(\"/kaggle/working/\"),\n    COLOR_MAP =['inferno'],\n    OUTPUT_METADATA=\"spec_img_meta.csv\",\n    WINDOW=\"hann\",\n    NFILTER_MEL=128,\n    WINDOW_LENGTH= 1024,\n    WINDOW_STEP= 512,\n    FREQ_HIGH=14000,\n    TARGET_DURATION_S = 5,\n    TARGET_SAMPLES = 5*32000,\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    )\n# Function to seed everything to ensure reproducibility\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False # Change to true if input sizes are kept constant\n\nseed_everything(cfg.SEED)\n# Verifying changes\nprint(cfg.__dict__)\n# Device check\nprint(f\"Using device: {cfg.DEVICE}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:29.252750Z","iopub.execute_input":"2025-05-01T03:17:29.253133Z","iopub.status.idle":"2025-05-01T03:17:29.357732Z","shell.execute_reply.started":"2025-05-01T03:17:29.253114Z","shell.execute_reply":"2025-05-01T03:17:29.357153Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Meta-data","metadata":{}},{"cell_type":"code","source":"# Taking a closer look at the meta data\nmetadata_path = Path(\"/kaggle/input/birdclef-2025/train.csv\")\n\ndef split_df_into_parts(df, n_parts=1, seed=42):\n    # 1) shuffle reproducibly\n    df_shuffled = df.sample(frac=1, random_state=seed).reset_index(drop=True)\n    parts = np.array_split(df_shuffled, n_parts)\n    return parts\n    \nif metadata_path.exists():\n    train_df = pd.read_csv(metadata_path)\n    train_df = train_df.drop(columns=['url', 'license', 'common_name','collection','author','type'])\n    print(train_df.head(15))\n    print(\"\\nMetadata Columns:\", train_df.columns)\n    print(\"\\nTraining Samples:\", len(train_df))\n    print(\"\\nUnique Species:\", train_df['primary_label'].nunique())\n    print(\"\\nSecondary Species Labels(Recordist Marked):\", train_df['secondary_labels'].nunique())\n    # Key distributions\n    print(\"\\nSpecies distribution (top 10):\")\n    print(train_df['primary_label'].value_counts().head(10))\nelse:\n    print(f\"Metadata file not found at {meta_datapath}. Check path!\")\nlist_part = split_df_into_parts(train_df, n_parts=1, seed=42)\ntrain_df = list_part[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:29.358492Z","iopub.execute_input":"2025-05-01T03:17:29.358772Z","iopub.status.idle":"2025-05-01T03:17:30.987651Z","shell.execute_reply.started":"2025-05-01T03:17:29.358748Z","shell.execute_reply":"2025-05-01T03:17:30.986847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train_df[train_df['primary_label'] == cfg.COMPARED_CLASS]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:30.989308Z","iopub.execute_input":"2025-05-01T03:17:30.989532Z","iopub.status.idle":"2025-05-01T03:17:30.996183Z","shell.execute_reply.started":"2025-05-01T03:17:30.989513Z","shell.execute_reply":"2025-05-01T03:17:30.995479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scientific_name_counts = train_df['scientific_name'].value_counts()\nprint(scientific_name_counts)\noverlap_map = {name: 0.8 if count < 50 else 0.2 for name, count in scientific_name_counts.items()}\nprint(\"dataframe mới\")\ntrain_df['overlap-percent'] = train_df['scientific_name'].map(overlap_map)\nprint(train_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:30.996861Z","iopub.execute_input":"2025-05-01T03:17:30.997118Z","iopub.status.idle":"2025-05-01T03:17:31.014426Z","shell.execute_reply.started":"2025-05-01T03:17:30.997097Z","shell.execute_reply":"2025-05-01T03:17:31.013583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Analyzing audio durations...\")\ndurations = []\npbar = tqdm(train_df['filename'].tolist(), desc=\"Calculating durations\")\nfor filename in pbar:\n    file_path = cfg.DATA_PATH/filename\n    if file_path.exists():\n        try:\n            # Efficient approach to get duration with loading the whole file\n            info = sf.info(file_path)\n            durations.append(info.duration)\n        except Exception as e:\n            print(f\"Could not get info for {filename}: {e}\") #Comment / uncomment for debugging\n            durations.append(np.nan) # mark errors\n    else:\n        durations.append(np.nan)\ntrain_df[\"duration\"]= durations","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:31.015265Z","iopub.execute_input":"2025-05-01T03:17:31.015873Z","iopub.status.idle":"2025-05-01T03:17:31.088617Z","shell.execute_reply.started":"2025-05-01T03:17:31.015850Z","shell.execute_reply":"2025-05-01T03:17:31.087858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"duration_by_class = train_df.groupby('primary_label')['duration'].sum()\nduration_by_class = duration_by_class.reset_index().sort_values(by=['duration']).reset_index()\n# Hiển thị dữ liệu tổng hợp (có thể in ra để kiểm tra)\nprint(duration_by_class)\n\n# Vẽ biểu đồ cột:\nplt.figure(figsize=(10, 6))\nplt.bar(duration_by_class['primary_label'].astype(str), duration_by_class['duration'], color='skyblue')\nplt.xlabel('Primary Label')\nplt.ylabel('total time (second)')\nplt.title('Total Time Per Class (primary_label)')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:31.089424Z","iopub.execute_input":"2025-05-01T03:17:31.090016Z","iopub.status.idle":"2025-05-01T03:17:31.305706Z","shell.execute_reply.started":"2025-05-01T03:17:31.089997Z","shell.execute_reply":"2025-05-01T03:17:31.304978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:31.306527Z","iopub.execute_input":"2025-05-01T03:17:31.306736Z","iopub.status.idle":"2025-05-01T03:17:31.323199Z","shell.execute_reply.started":"2025-05-01T03:17:31.306711Z","shell.execute_reply":"2025-05-01T03:17:31.322293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing Function","metadata":{}},{"cell_type":"code","source":"# ============================\n# 1. Hàm load và tiền xử lý audio\n# ============================\ndef load_and_preprocess(audio_path, sr, origin = True):\n    \"\"\"\n    - Load file .ogg bằng librosa.\n    - Giảm nhiễu bằng noisereduce.\n    - Tăng âm lượng bằng pedalboard.\n    \"\"\"\n    samples, sr = librosa.load(audio_path, sr=sr)\n    samples_nr = nr.reduce_noise(y=samples, sr=sr)\n    samples_proc = None\n    if origin:\n        board = Pedalboard([Gain(gain_db=10)])\n        samples_proc = board(samples_nr, sr)\n    else:\n        gain_db = 10\n        gain_linear = 10**(gain_db / 20)\n        samples_proc = samples_nr*gain_linear\n    \n    return samples_proc, sr\n\n# ============================\n# 2. Hàm tạo sliding window (5 giây, bước nhảy 0.5 giây)\n# ============================\ndef get_sliding_windows(audio, sr, segment_duration=5.0, overlap_percent=0.5):\n    \"\"\"\n    Trả về danh sách các tuple (start_sample, end_sample) cho mỗi window.\n    \"\"\"\n    step = segment_duration*(1-overlap_percent)\n    seg_samples = int(segment_duration * sr)\n    step_samples = int(step * sr)\n    windows = []\n    for start in range(0, len(audio) - seg_samples + 1, step_samples):\n        end = start + seg_samples\n        windows.append((start, end))\n    return windows\n    \n# ============================\n# 3. Hàm loại bỏ đoạn có tiếng người và padding\n# ============================\ndef remove_human_voice(segment_audio, sr, window_start, human_voice_intervals, seg_duration=5.0):\n    \"\"\"\n    - segment_audio: mảng audio của 1 window (5 giây)\n    - window_start: thời gian bắt đầu (giây) của window trong file gốc.\n    - human_voice_intervals: danh sách dict {'start': ..., 'end': ...} (giây) đã phát hiện tiếng người trong file gốc.\n    \n    Hàm này sẽ loại bỏ (cut) các đoạn trùng với tiếng người, nối các đoạn còn lại lại,\n    rồi thêm padding (âm im) ở 2 đầu để độ dài về 5 giây.\n    \"\"\"\n    seg_len = len(segment_audio)\n    \n    # Lọc các interval có giao với window hiện tại\n    intervals = [iv for iv in human_voice_intervals \n                 if iv['start'] < window_start + seg_duration and iv['end'] > window_start]\n    intervals = sorted(intervals, key=lambda x: x['start'])\n    \n    # Nếu không có tiếng người trong window -> return segment ban đầu\n    if not intervals:\n        return segment_audio\n    \n    nonhuman_parts = []\n    current_sample = 0\n    # Xử lý các interval theo thứ tự tăng dần\n    for iv in intervals:\n        # Chuyển đổi thời gian của interval về thời gian tương đối trong window\n        start_relative = max(iv['start'] - window_start, 0)\n        end_relative = min(iv['end'] - window_start, seg_duration)\n        start_idx = int(start_relative * sr)\n        end_idx = int(end_relative * sr)\n        # Nếu có đoạn không có tiếng người trước interval hiện tại, lấy nó\n        if start_idx > current_sample:\n            nonhuman_parts.append(segment_audio[current_sample:start_idx])\n        # Cập nhật vị trí hiện tại (bỏ qua phần có tiếng người)\n        current_sample = end_idx\n    # Lấy phần sau interval cuối cùng\n    if current_sample < seg_len:\n        nonhuman_parts.append(np.zeros(seg_len -current_sample))\n        nonhuman_parts.append(segment_audio[current_sample:])\n    \n    if nonhuman_parts:\n        nonhuman_audio = np.concatenate(nonhuman_parts)\n    else:\n        nonhuman_audio = np.array([], dtype=segment_audio.dtype)\n    \n    # Padding thêm silence để có đủ số mẫu ban đầu (5 giây)\n    desired_length = seg_len\n    current_length = len(nonhuman_audio)\n    if current_length < desired_length:\n        pad_total = desired_length - current_length\n        pad_left = pad_total // 2\n        pad_right = pad_total - pad_left\n        nonhuman_audio = np.concatenate([\n            np.zeros(pad_left, dtype=segment_audio.dtype), \n            nonhuman_audio, \n            np.zeros(pad_right, dtype=segment_audio.dtype)\n        ])\n    else:\n        nonhuman_audio = nonhuman_audio[:desired_length]\n    \n    return nonhuman_audio\n\n# ============================\n# 4. Lưu ảnh Mel Spectrogram\n# ============================\nfrom PIL import Image\nimport matplotlib.cm as cm\n\n\ndef save_spectrogram_image(mel_spec, cfg, output_dir, output_filename, output_size=(1280, 720)):\n    # Normalize mel spectrogram to 0–1\n    mel_spec = (mel_spec - mel_spec.min()) / (mel_spec.max() - mel_spec.min() + 10**(-6))\n    # List of colormaps\n    colormaps = cfg.COLOR_MAP\n\n    for cmap_name in colormaps:\n        out_path =  cfg.OUTPUT_FOLDER /cmap_name /output_dir\n        out_path.mkdir(parents=True, exist_ok=True)\n        output_file = out_path / output_filename\n        \n        cmap = cm.get_cmap(cmap_name)\n        rgba_img = cmap(mel_spec)  # shape: (H, W, 4), values in [0, 1]\n        rgb_img = (rgba_img[:, :, :3] * 255).astype(np.uint8)\n        img = Image.fromarray(rgb_img)\n\n        # Flip vertically (to match origin='lower' behavior)\n        img = img.transpose(Image.FLIP_TOP_BOTTOM)\n        img_resized = img.resize(output_size, Image.LANCZOS)\n\n        # Save final image\n        img_resized.save(output_file)\n\n\ndef show_spectrogram(spec, title, sr, hop_length, y_axis=\"log\", x_axis=\"time\"):\n    librosa.display.specshow(\n        spec, sr=sr, y_axis=y_axis, x_axis=x_axis, hop_length=hop_length\n    )\n    plt.title(title)\n    plt.colorbar(format=\"%+2.0f dB\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:31.324198Z","iopub.execute_input":"2025-05-01T03:17:31.324603Z","iopub.status.idle":"2025-05-01T03:17:31.339618Z","shell.execute_reply.started":"2025-05-01T03:17:31.324586Z","shell.execute_reply":"2025-05-01T03:17:31.338968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# -------------------------\n# 4. DALI Pipeline: Tạo Mel Spectrogram\n# -------------------------\n\ndef compute_mel_spectrogram_gpu(audio_segment, cfg):\n    audio_data = np.array(audio_segment, dtype=np.float32)\n    @pipeline_def\n    def mel_spectrogram_pipe(nfft, window_length, window_step,sample_rate,nfilter,freq_high,device=\"gpu\"):\n        audio = types.Constant(device=device, value=audio_data)\n        spectrogram = fn.spectrogram(\n            audio,\n            device=device,\n            nfft=nfft,\n            window_length=window_length,\n            window_step=window_step,\n        )\n        mel_spectrogram = fn.mel_filter_bank(\n            spectrogram, sample_rate=cfg.SAMPLE_RATE, nfilter=nfilter, freq_high=freq_high\n        )\n        mel_spectrogram_dB = fn.to_decibels(\n            mel_spectrogram, multiplier=10.0, cutoff_db=-80\n        )\n        return mel_spectrogram_dB\n        \n    pipe = mel_spectrogram_pipe(\n        device=\"gpu\",\n        batch_size=1,\n        num_threads=3,\n        device_id=0,\n        nfft=cfg.WINDOW_LENGTH,\n        window_length=cfg.WINDOW_LENGTH,\n        window_step=cfg.WINDOW_STEP,\n        sample_rate=cfg.SAMPLE_RATE,\n        nfilter=cfg.NFILTER_MEL,\n        freq_high=cfg.FREQ_HIGH,\n    )\n    pipe.build()\n    outputs = pipe.run()\n    mel_spectrogram_dali_db = np.array(outputs[0][0].as_cpu())\n\n    return mel_spectrogram_dali_db\n\ndef compute_mel_spectrogram_cpu(audio_segment, cfg):\n    audio_data = np.array(audio_segment, dtype=np.float32)\n\n    @pipeline_def\n    def mel_spectrogram_pipe(nfft, window_length, window_step, sample_rate, nfilter, freq_high, device=\"cpu\"):\n        audio = types.Constant(device=device, value=audio_data)\n        spectrogram = fn.spectrogram(\n            audio,\n            device=device,\n            nfft=nfft,\n            window_length=window_length,\n            window_step=window_step,\n        )\n        mel_spectrogram = fn.mel_filter_bank(\n            spectrogram,\n            device=device,\n            sample_rate=sample_rate,\n            nfilter=nfilter,\n            freq_high=freq_high\n        )\n        mel_spectrogram_dB = fn.to_decibels(\n            mel_spectrogram,\n            device=device,\n            multiplier=10.0,\n            cutoff_db=-80\n        )\n        return mel_spectrogram_dB\n\n    pipe = mel_spectrogram_pipe(\n        device=\"cpu\",\n        batch_size=1,\n        num_threads=1,\n        nfft=cfg.WINDOW_LENGTH,\n        window_length=cfg.WINDOW_LENGTH,\n        window_step=cfg.WINDOW_STEP,\n        sample_rate=cfg.SAMPLE_RATE,\n        nfilter=cfg.NFILTER_MEL,\n        freq_high=cfg.FREQ_HIGH,\n    )\n\n    pipe.build()\n    outputs = pipe.run()\n    mel_spectrogram_dali_db = np.array(outputs[0][0])  # No need for .as_cpu()\n\n    return mel_spectrogram_dali_db\n# -------------------------\n# 4. librosa Pipeline: Tạo Mel Spectrogram\n# -------------------------\n\ndef compute_mel_spectrogram_librosa(\n    audio_segment: np.ndarray,\n    cfg\n) -> np.ndarray:\n    # # 1) Compute power spectrogram with STFT\n    # S = librosa.stft(\n    #     audio_segment,\n    #     n_fft=cfg.WINDOW_LENGTH,\n    #     hop_length=cfg.WINDOW_STEP,\n    #     win_length=cfg.WINDOW_LENGTH,\n    #     window='hann',\n    #     center=True,\n    #     pad_mode='reflect'\n    # )\n    # # Convert to power (magnitude squared)\n    # S_power = np.abs(S)**2\n\n    # 2) Apply Mel filter bank\n    mel_S = librosa.feature.melspectrogram(\n        y=audio_segment,\n        sr=cfg.SAMPLE_RATE,\n        n_mels=cfg.NFILTER_MEL,\n        fmax=cfg.FREQ_HIGH,\n        pad_mode=\"reflect\"\n    )\n\n    # 3) Convert to decibel scale\n    S_db = librosa.power_to_db(mel_S, ref=np.max, top_db=80.0)\n\n    return S_db\n# -------------------------\n# 4. torch audio Pipeline: Tạo Mel Spectrogram\n# -------------------------\nimport torchaudio\n\ndef compute_mel_spectrogram_torch(audio_segment, cfg):\n    # Convert to PyTorch tensor\n    waveform = torch.tensor(audio_segment, dtype=torch.float32)\n    if waveform.ndim == 1:\n        waveform = waveform.unsqueeze(0)  # Shape: (1, num_samples)\n\n    # Define MelSpectrogram transform\n    mel_spec_transform = torchaudio.transforms.MelSpectrogram(\n        sample_rate=cfg.SAMPLE_RATE,\n        n_fft=cfg.WINDOW_LENGTH,\n        win_length=cfg.WINDOW_LENGTH,\n        hop_length=cfg.WINDOW_STEP,\n        n_mels=cfg.NFILTER_MEL,\n        f_max=cfg.FREQ_HIGH,\n        power=2,\n        pad_mode = 'reflect'\n    )\n\n    # Apply MelSpectrogram\n    mel_spec = mel_spec_transform(waveform)  # Shape: (1, n_mels, time)\n\n    # Convert to decibel (log scale)\n    mel_spec_db = torchaudio.transforms.AmplitudeToDB(top_db=80)(mel_spec)\n\n    # Convert to numpy if needed\n    return mel_spec_db.squeeze(0).numpy()\n\ndef process_audio_file(audio_path, human_voice_dict, output_folder, cfg, overlap_percent):\n    # print(\"processing: \",audio_path)\n    samples_proc_origin, sr = load_and_preprocess(audio_path, cfg.SAMPLE_RATE)\n    samples_proc_new, sr = load_and_preprocess(audio_path, cfg.SAMPLE_RATE, True)\n    # Tạo sliding windows, mỗi window TARGET_DURATION_S giây, bước 0.5 giây\n    windows_origin = get_sliding_windows(samples_proc_origin, sr, cfg.TARGET_DURATION_S, overlap_percent = overlap_percent)\n    windows_new = get_sliding_windows(samples_proc_new, sr, cfg.TARGET_DURATION_S, overlap_percent = overlap_percent)\n    \n    # Lấy human voice intervals nếu tồn tại cho file này (dùng đường dẫn tương đối so với DATA_PATH)\n    human_intervals = human_voice_dict.get(audio_path, [])\n    print(\"processing nvida dali\")\n    for start_sample, end_sample in windows_origin:\n        # Tạo tên file: AudioFileName_start_end.jpg\n        window_start_time = start_sample / sr\n        base_name = os.path.splitext(Path(audio_path).name)[0]\n        output_filename = f\"{base_name}_{window_start_time:.1f}_{(window_start_time+cfg.TARGET_DURATION_S):.1f}.jpg\"\n        # =======Original Pre processing===========\n        segment_audio = samples_proc_origin[start_sample:end_sample]\n        # Loại bỏ tiếng người và padding lại thành đủ TARGET_DURATION_S giây\n        segment_clean = remove_human_voice(segment_audio, sr, window_start_time, human_intervals, cfg.TARGET_DURATION_S)\n        # Tính Mel Spectrogram sử dụng NVIDIA DALI\n        mel_spec_dali = compute_mel_spectrogram_gpu(segment_clean, cfg)\n        #Save into virisdis format\n        output_folder_dali = output_folder + \"/dali\"\n        save_spectrogram_image(mel_spec_dali, cfg, output_folder_dali, output_filename, output_size=(640, 480))\n\n    print(\"processing torchaudio\")\n    for start_sample, end_sample in windows_new:   \n        # Tạo tên file: AudioFileName_start_end.jpg\n        window_start_time = start_sample / sr\n        base_name = os.path.splitext(Path(audio_path).name)[0]\n        output_filename = f\"{base_name}_{window_start_time:.1f}_{(window_start_time+cfg.TARGET_DURATION_S):.1f}.jpg\"\n        # =======New Pre processing===========\n        segment_audio = samples_proc_new[start_sample:end_sample]\n        # Loại bỏ tiếng người và padding lại thành đủ TARGET_DURATION_S giây\n        segment_clean = remove_human_voice(segment_audio, sr, window_start_time, human_intervals, cfg.TARGET_DURATION_S)\n        # Tính Mel Spectrogram sử dụng NVIDIA DALI\n        mel_spec_librosa = compute_mel_spectrogram_cpu(segment_clean, cfg)\n        #Save into virisdis format\n        output_folder_librosa = output_folder + \"/dali_cpu\"\n        save_spectrogram_image(mel_spec_librosa, cfg, output_folder_librosa, output_filename, output_size=(640, 480))\n        \n        # save_spectrogram_image(mel_spec, str(output_file), cmap='gray')\n        # print(f\"Saved {output_filename}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:31.341520Z","iopub.execute_input":"2025-05-01T03:17:31.341693Z","iopub.status.idle":"2025-05-01T03:17:32.124035Z","shell.execute_reply.started":"2025-05-01T03:17:31.341679Z","shell.execute_reply":"2025-05-01T03:17:32.123476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nwith open(\"/kaggle/input/bc25-human-detect-sound/train_voice_data.pkl\", \"rb\") as f:\n    human_voice_dict = pickle.load(f)\n        \n    # Lặp qua từng dòng trong DataFrame và xử lý file audio\nos.makedirs(cfg.OUTPUT_FOLDER, exist_ok=True)\nerror_list = []\nfor cmap_name in cfg.COLOR_MAP:\n    path =  cfg.OUTPUT_FOLDER /cmap_name \n    path.mkdir(parents=True, exist_ok=True)\nfor idx, row in train_df.iterrows():\n    output_folder =  row['primary_label']\n    # os.makedirs(output_folder, exist_ok=True)\n    rel_path = row['filename']  # đường dẫn tương đối\n    overlap_percent = row['overlap-percent']\n    audio_path = cfg.DATA_PATH / rel_path\n    print(f\"Processing {audio_path} ...\")\n    # try:\n    process_audio_file(str(audio_path), human_voice_dict, output_folder, cfg, overlap_percent)\n    # except Exception as e:\n    #     print_out_log = f\"Error processing {audio_path}: {e}\"\n    #     error_list.append(print_out_log)\n    #     print(f\"Error processing {audio_path}: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:17:32.124712Z","iopub.execute_input":"2025-05-01T03:17:32.124981Z","iopub.status.idle":"2025-05-01T03:19:31.564977Z","shell.execute_reply.started":"2025-05-01T03:17:32.124963Z","shell.execute_reply":"2025-05-01T03:19:31.564314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\ndisplay(Image.open(\"/kaggle/working/inferno/963335/dali/CSA36372_0.0_5.0.jpg\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:19:31.565833Z","iopub.execute_input":"2025-05-01T03:19:31.566183Z","iopub.status.idle":"2025-05-01T03:19:31.597773Z","shell.execute_reply.started":"2025-05-01T03:19:31.566159Z","shell.execute_reply":"2025-05-01T03:19:31.597128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(Image.open(\"/kaggle/working/inferno/963335/dali_cpu/CSA36372_0.0_5.0.jpg\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:19:31.599087Z","iopub.execute_input":"2025-05-01T03:19:31.599278Z","iopub.status.idle":"2025-05-01T03:19:31.626035Z","shell.execute_reply.started":"2025-05-01T03:19:31.599262Z","shell.execute_reply":"2025-05-01T03:19:31.625321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check similarity","metadata":{}},{"cell_type":"code","source":"\n# Constants\nIMAGE_FOLDER_A = \"/kaggle/working/inferno/963335/dali\"\nIMAGE_FOLDER_B = \"/kaggle/working/inferno/963335/dali_cpu\"\nHASH_FUNC = imagehash.phash\nHASH_BITS = 64\nSIMILARITY_THRESHOLD = int((1 - 0.98) * HASH_BITS)  # Allow 2% difference\n\n# Get image paths by filename\ndef get_image_paths(folder):\n    return {\n        f: os.path.join(folder, f)\n        for f in os.listdir(folder)\n        if f.lower().endswith((\".png\", \".jpg\", \".jpeg\", \".bmp\"))\n    }\n\n# Compute hash from image\ndef compute_hash(image_path):\n    try:\n        with Image.open(image_path) as img:\n            return HASH_FUNC(img)\n    except:\n        return None\n\n# Compare hash distance\ndef compare_hashes(args):\n    file_name, path_a, path_b = args\n    hash_a = compute_hash(path_a)\n    hash_b = compute_hash(path_b)\n    if hash_a is None or hash_b is None:\n        return (file_name, \"ERROR\")\n    distance = hash_a - hash_b\n    return (file_name, distance)\n\n# Main\ndef main():\n    paths_a = get_image_paths(IMAGE_FOLDER_A)\n    paths_b = get_image_paths(IMAGE_FOLDER_B)\n\n    common_files = sorted(set(paths_a) & set(paths_b))\n    print(f\"Comparing {len(common_files)} common images.\")\n\n    # Prepare comparison tasks\n    tasks = [(fname, paths_a[fname], paths_b[fname]) for fname in common_files]\n\n    # Compare with multiprocessing\n    with Pool(cpu_count()) as pool:\n        results = list(tqdm(pool.imap(compare_hashes, tasks), total=len(tasks), desc=\"Comparing\"))\n\n    # Report mismatches\n    mismatches = [(f, d) for f, d in results if isinstance(d, int) and d > SIMILARITY_THRESHOLD]\n    errors = [f for f, d in results if d == \"ERROR\"]\n\n    print(f\"\\n✅ Matched: {len(common_files) - len(mismatches) - len(errors)}\")\n    print(f\"⚠️ Mismatched (>{SIMILARITY_THRESHOLD}): {len(mismatches)}\")\n    print(f\"❌ Errors: {len(errors)}\")\n\n    if mismatches:\n        print(\"\\nExamples of mismatches:\")\n        for f, d in mismatches[:10]:\n            print(f\"{f}: Distance = {d}\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-01T03:19:57.729600Z","iopub.execute_input":"2025-05-01T03:19:57.730326Z","iopub.status.idle":"2025-05-01T03:19:59.301467Z","shell.execute_reply.started":"2025-05-01T03:19:57.730303Z","shell.execute_reply":"2025-05-01T03:19:59.300535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}