{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":25954,"databundleVersionId":2091745,"sourceType":"competition"},{"sourceId":172118,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":146512,"modelId":169040}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"name":"keras_test","provenance":[]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nwarnings.filterwarnings(action='ignore')\nimport pandas as pd\nimport librosa\nimport numpy as np\nfrom pathlib import Path\nfrom os import listdir\nimport tensorflow as tf\nfrom tensorflow.keras.layers import TFSMLayer\nSAMPLE_RATE = 32000\nSIGNAL_LENGTH = 5\nSPEC_SHAPE = (48, 128)\nFMIN = 400\nFMAX = 12000\nTHRESHOLD = 0.27","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/birdclef-2021/train_metadata.csv\")\n\nLABEL_IDS = {label: label_id for label_id,label in enumerate(sorted(df_train[\"primary_label\"].unique()))}\nINV_LABEL_IDS = {val: key for key,val in LABEL_IDS.items()}\nlen(INV_LABEL_IDS)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model('/kaggle/input/resnet_34_b0_birds/keras/default/1/resnet_34_b0_birds.keras')\n\ncommon_df = pd.DataFrame()\n\nTEST_AUDIO_ROOT= \"../input/birdclef-2021/test_soundscapes/\"\nTARGET_PATH = None\nSAMPLE_SUB_PATH = \"../input/birdclef-2021/sample_submission.csv\"\n\nif not len(list(Path(TEST_AUDIO_ROOT).glob(\"*.ogg\"))):\n    TEST_AUDIO_ROOT = \"../input/birdclef-2021/train_soundscapes/\"\n    TARGET_PATH = \"../input/birdclef-2021/train_soundscape_labels.csv\"\n    SAMPLE_SUB_PATH = None\n\n\nsoundscape_files = list(listdir(TEST_AUDIO_ROOT))\ntotal_files = len(soundscape_files)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Функции для обработки данных\ndef prepare_mel_spec(chunk, sr=SAMPLE_RATE):\n    \"\"\"Подготовка мел-спектрограммы для одного фрагмента\"\"\"\n    hop_length = int(SIGNAL_LENGTH * sr / (SPEC_SHAPE[1] - 1))\n    mel_spec = librosa.feature.melspectrogram(y=chunk, \n                                              sr=sr, \n                                              n_fft=1024, \n                                              hop_length=hop_length, \n                                              n_mels=SPEC_SHAPE[0], \n                                              fmin=FMIN, \n                                              fmax=FMAX)\n    mel_spec = librosa.power_to_db(mel_spec, ref=np.max)\n    mel_spec -= mel_spec.min()\n    mel_spec /= mel_spec.max()\n    mel_spec = np.expand_dims(mel_spec, -1)  # добавляем размерность для канала\n    return mel_spec\n\ndef process_file(soundscape_path, model, threshold=THRESHOLD):\n    \"\"\"Обработка одного файла\"\"\"\n    try:\n        sig, rate = librosa.load(soundscape_path, sr=SAMPLE_RATE)\n    except Exception as e:\n        print(f\"Ошибка при загрузке файла {soundscape_path}: {e}\")\n        return pd.DataFrame(columns=[\"row_id\", \"birds\"])\n\n    data = {'row_id': [], 'birds': []}\n    sig_splits = [sig[i:i + int(SIGNAL_LENGTH * SAMPLE_RATE)] \n                  for i in range(0, len(sig), int(SIGNAL_LENGTH * SAMPLE_RATE))\n                  if len(sig[i:i + int(SIGNAL_LENGTH * SAMPLE_RATE)]) == int(SIGNAL_LENGTH * SAMPLE_RATE)]\n\n    # Подготовка спектрограмм для всех фрагментов\n    mel_specs = np.array([prepare_mel_spec(chunk) for chunk in sig_splits])\n\n    # Предсказания для всех спектрограмм сразу\n    predictions = model.predict(mel_specs)\n\n    seconds = 0\n    for p in predictions:\n        seconds += 5\n        detected_species = []\n        for idx, score in enumerate(p):\n            if score.mean() > threshold:  # Проверяем среднее значение\n                detected_species.append(INV_LABEL_IDS[idx])\n        \n        prediction = \" \".join(sorted(detected_species)) if detected_species else \"nocall\"\n        row_id = f\"{soundscape_path.split(os.sep)[-1].rsplit('_', 1)[0]}_{str(seconds)}\"\n        data['row_id'].append(row_id)\n        data['birds'].append(prediction)\n    # for p in predictions:\n    #     seconds += 5\n    #     detected_species = [INV_LABEL_IDS[idx] for idx, score in enumerate(p) if score > threshold]\n    #     prediction = \" \".join(sorted(detected_species)) if detected_species else \"nocall\"\n    #     row_id = f\"{soundscape_path.split(os.sep)[-1].rsplit('_', 1)[0]}_{str(seconds)}\"\n    #     data['row_id'].append(row_id)\n    #     data['birds'].append(prediction)\n\n    return pd.DataFrame(data)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Обработка всех файлов\niteration = 1\nfor soundscape_path in soundscape_files:\n    soundscape_path = TEST_AUDIO_ROOT + soundscape_path\n    result_df = process_file(soundscape_path, model)\n    common_df = pd.concat([common_df, result_df])\n    \n    print(f\"ITER NUMBER {iteration} OF {total_files}\")\n    iteration += 1\n    #break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SAMPLE_SUB_PATH:\n    sample_sub = pd.read_csv(SAMPLE_SUB_PATH, usecols=[\"row_id\"])\n    common_df = sample_sub.merge(common_df, on=\"row_id\", how=\"left\")\n    common_df[\"birds\"] = common_df[\"birds\"].fillna(\"nocall\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Запись результата в файл\ncommon_df.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_metrics(s_true, s_pred):\n    s_true = set(s_true.split())\n    s_pred = set(s_pred.split())\n    n, n_true, n_pred = len(s_true.intersection(s_pred)), len(s_true), len(s_pred)\n    \n    prec = n/n_pred\n    rec = n/n_true\n    f1 = 2*prec*rec/(prec + rec) if prec + rec else 0\n    \n    return {\"f1\": f1, \"prec\": prec, \"rec\": rec, \"n_true\": n_true, \"n_pred\": n_pred, \"n\": n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TARGET_PATH:\n    sub_target = pd.read_csv(TARGET_PATH)\n    sub_target = sub_target.merge(common_df, how=\"left\", on=\"row_id\")\n    \n    print(sub_target[\"birds_x\"].notnull().sum(), sub_target[\"birds_x\"].notnull().sum())\n    assert sub_target[\"birds_x\"].notnull().all()\n    assert sub_target[\"birds_y\"].notnull().all()\n    \n    df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target.birds_x, sub_target.birds_y)])\n    \n    print(df_metrics.mean())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_target.head(10)","metadata":{},"execution_count":null,"outputs":[]}]}