{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8900,"databundleVersionId":862232,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Импорт библиотек","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport librosa.display\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models, callbacks\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"Библиотеки успешно импортированы!\")\nprint(f\"Версия TensorFlow: {keras.__version__}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Настройка путей к данным","metadata":{}},{"cell_type":"code","source":"# Пути к данным на Kaggle\nDATA_DIR = '/kaggle/input/freesound-audio-tagging'\nTRAIN_DIR = os.path.join(DATA_DIR, 'audio_train')\nTEST_DIR = os.path.join(DATA_DIR, 'audio_test')\nTRAIN_CSV = os.path.join(DATA_DIR, 'train.csv')\nSAMPLE_SUBMISSION = os.path.join(DATA_DIR, 'sample_submission.csv')\n\nprint(f\"Папка train существует: {os.path.exists(TRAIN_DIR)}\")\nprint(f\"Папка test существует: {os.path.exists(TEST_DIR)}\")\nprint(f\"Файл train.csv существует: {os.path.exists(TRAIN_CSV)}\")\n\nif os.path.exists(DATA_DIR):\n    print(f\"\\nСодержимое {DATA_DIR}:\")\n    print(os.listdir(DATA_DIR))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Исследование данных","metadata":{}},{"cell_type":"code","source":"# Загрузка тренировочных данных\ntrain_df = pd.read_csv(TRAIN_CSV)\nprint(f\"Количество тренировочных примеров: {len(train_df)}\")\nprint(f\"\\nПервые несколько строк:\")\ntrain_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Анализ распределения меток\nprint(f\"Количество уникальных меток: {train_df['label'].nunique()}\")\nprint(f\"\\nРаспределение меток:\")\nlabel_counts = train_df['label'].value_counts()\nprint(label_counts)\n\n# Визуализация распределения классов\nplt.figure(figsize=(15, 6))\nlabel_counts.plot(kind='bar')\nplt.title('Распределение аудио классов')\nplt.xlabel('Метка класса')\nplt.ylabel('Количество')\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Загрузка и визуализация примера аудиофайла\nsample_file = train_df.iloc[0]['fname']\nsample_label = train_df.iloc[0]['label']\nsample_path = os.path.join(TRAIN_DIR, sample_file)\n\nprint(f\"Файл примера: {sample_file}\")\nprint(f\"Метка: {sample_label}\")\n\n# Загрузка аудио\naudio, sr = librosa.load(sample_path, sr=44100)\nprint(f\"Частота дискретизации: {sr}\")\nprint(f\"Длительность аудио: {len(audio)/sr:.2f} секунд\")\n\n# Визуализация волновой формы\nplt.figure(figsize=(12, 4))\nlibrosa.display.waveshow(audio, sr=sr)\nplt.title(f'Волновая форма: {sample_label}')\nplt.xlabel('Время')\nplt.ylabel('Амплитуда')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Извлечение признаков\n\nИзвлекаем mel-спектрограммы в качестве признаков.","metadata":{}},{"cell_type":"code","source":"# Параметры извлечения признаков\nSAMPLE_RATE = 44100\nDURATION = 2\nN_MELS = 128\nN_FFT = 2048\nHOP_LENGTH = 512\n\ndef extract_features(file_path, duration=DURATION, sr=SAMPLE_RATE):\n    try:\n        audio, sample_rate = librosa.load(file_path, sr=sr, duration=duration)\n        \n        if len(audio) < sr * duration:\n            audio = np.pad(audio, (0, sr * duration - len(audio)), mode='constant')\n        \n        # Извлечение mel-спектрограммы\n        mel_spec = librosa.feature.melspectrogram(\n            y=audio, \n            sr=sample_rate, \n            n_mels=N_MELS,\n            n_fft=N_FFT,\n            hop_length=HOP_LENGTH\n        )\n        \n        # Преобразование в логарифмическую шкалу (dB)\n        mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n        \n        return mel_spec_db\n    \n    except Exception as e:\n        print(f\"Ошибка обработки {file_path}: {e}\")\n        return None\n\n# Тестирование извлечения признаков\nsample_features = extract_features(sample_path)\nprint(f\"Размерность признаков: {sample_features.shape}\")\n\n# Визуализация mel-спектрограммы\nplt.figure(figsize=(12, 6))\nlibrosa.display.specshow(sample_features, sr=SAMPLE_RATE, hop_length=HOP_LENGTH, \n                         x_axis='time', y_axis='mel', cmap='coolwarm')\nplt.colorbar(format='%+2.0f dB')\nplt.title(f'Mel-спектрограмма: {sample_label}')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Извлечение признаков для всех тренировочных файлов\nprint(\"Извлечение признаков из тренировочных данных...\")\nprint(\"Это может занять несколько минут...\")\n\nX_train = []\ny_train = []\n\nfor idx, row in tqdm(train_df.iterrows(), total=len(train_df)):\n    file_path = os.path.join(TRAIN_DIR, row['fname'])\n    features = extract_features(file_path)\n    \n    if features is not None:\n        X_train.append(features)\n        y_train.append(row['label'])\n\nX_train = np.array(X_train)\ny_train = np.array(y_train)\n\nprint(f\"\\nРазмерность тренировочных признаков: {X_train.shape}\")\nprint(f\"Размерность тренировочных меток: {y_train.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Кодирование меток\nlabel_encoder = LabelEncoder()\ny_train_encoded = label_encoder.fit_transform(y_train)\nnum_classes = len(label_encoder.classes_)\n\nprint(f\"Количество классов: {num_classes}\")\nprint(f\"Классы: {label_encoder.classes_}\")\n\n# Преобразование в категориальный формат\ny_train_categorical = keras.utils.to_categorical(y_train_encoded, num_classes)\n\n# Изменение формы для CNN (добавление канала)\nX_train = X_train[..., np.newaxis]\nprint(f\"\\nФинальная форма тренировочных данных: {X_train.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Разделение на тренировочную и валидационную выборки\nX_train_split, X_val, y_train_split, y_val = train_test_split(\n    X_train, y_train_categorical, test_size=0.2, random_state=42, stratify=y_train_encoded\n)\n\nprint(f\"Тренировочная выборка: {X_train_split.shape}\")\nprint(f\"Валидационная выборка: {X_val.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Построение модели","metadata":{}},{"cell_type":"code","source":"def create_cnn_model(input_shape, num_classes):\n    model = models.Sequential([\n        # Первый сверточный блок\n        layers.Conv2D(32, (3, 3), activation='relu', padding='same', input_shape=input_shape),\n        layers.BatchNormalization(),\n        layers.Conv2D(32, (3, 3), activation='relu', padding='same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D((2, 2)),\n        layers.Dropout(0.25),\n        \n        # Второй сверточный блок\n        layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n        layers.BatchNormalization(),\n        layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D((2, 2)),\n        layers.Dropout(0.25),\n        \n        # Третий сверточный блок\n        layers.Conv2D(128, (3, 3), activation='relu', padding='same'),\n        layers.BatchNormalization(),\n        layers.Conv2D(128, (3, 3), activation='relu', padding='same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D((2, 2)),\n        layers.Dropout(0.25),\n        \n        # Полносвязные слои\n        layers.Flatten(),\n        layers.Dense(256, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.5),\n        layers.Dense(128, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.5),\n        layers.Dense(num_classes, activation='softmax')\n    ])\n    \n    return model\n\n# Создание модели\nmodel = create_cnn_model(X_train_split.shape[1:], num_classes)\nmodel.summary()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Компиляция модели\nmodel.compile(\n    optimizer=keras.optimizers.Adam(learning_rate=0.001),\n    loss='categorical_crossentropy',\n    metrics=['accuracy', keras.metrics.TopKCategoricalAccuracy(k=3, name='top_3_accuracy')]\n)\n\nprint(\"Модель успешно скомпилирована!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Обучение модели","metadata":{}},{"cell_type":"code","source":"# Проверка доступности GPU\nimport tensorflow as tf\nprint(\"Доступные GPU:\", tf.config.list_physical_devices('GPU'))\nprint(\"Используется GPU:\", len(tf.config.list_physical_devices('GPU')) > 0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Определение callbacks\ncheckpoint = callbacks.ModelCheckpoint(\n    'best_model.h5',\n    monitor='val_accuracy',\n    save_best_only=True,\n    mode='max',\n    verbose=1\n)\n\nearly_stopping = callbacks.EarlyStopping(\n    monitor='val_loss',\n    patience=10,\n    restore_best_weights=True,\n    verbose=1\n)\n\nreduce_lr = callbacks.ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=5,\n    min_lr=1e-7,\n    verbose=1\n)\n\n# Обучение модели\nEPOCHS = 50\nBATCH_SIZE = 32\n\nprint(\"Начало обучения...\")\nprint(f\"Эпох: {EPOCHS}, Batch size: {BATCH_SIZE}\")\n\nhistory = model.fit(\n    X_train_split, y_train_split,\n    batch_size=BATCH_SIZE,\n    epochs=EPOCHS,\n    validation_data=(X_val, y_val),\n    callbacks=[checkpoint, early_stopping, reduce_lr],\n    verbose=1\n)\n\nprint(\"\\nОбучение завершено!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Визуализация истории обучения\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\n\n# Точность\naxes[0].plot(history.history['accuracy'], label='Точность на обучении')\naxes[0].plot(history.history['val_accuracy'], label='Точность на валидации')\naxes[0].plot(history.history['top_3_accuracy'], label='Top-3 точность на обучении')\naxes[0].plot(history.history['val_top_3_accuracy'], label='Top-3 точность на валидации')\naxes[0].set_title('Точность модели')\naxes[0].set_xlabel('Эпоха')\naxes[0].set_ylabel('Точность')\naxes[0].legend()\naxes[0].grid(True)\n\n# Функция потерь\naxes[1].plot(history.history['loss'], label='Потери на обучении')\naxes[1].plot(history.history['val_loss'], label='Потери на валидации')\naxes[1].set_title('Функция потерь')\naxes[1].set_xlabel('Эпоха')\naxes[1].set_ylabel('Потери')\naxes[1].legend()\naxes[1].grid(True)\n\nplt.tight_layout()\nplt.show()\n\n# Вывод лучших результатов\nbest_val_acc = max(history.history['val_accuracy'])\nbest_val_top3 = max(history.history['val_top_3_accuracy'])\nprint(f\"\\n\" + \"=\"*50)\nprint(f\"ЛУЧШИЕ РЕЗУЛЬТАТЫ:\")\nprint(f\"=\"*50)\nprint(f\"Лучшая точность на валидации: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\")\nprint(f\"Лучшая Top-3 точность на валидации: {best_val_top3:.4f} ({best_val_top3*100:.2f}%)\")\nprint(f\"=\"*50)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Генерация предсказаний для тестовой выборки","metadata":{}},{"cell_type":"code","source":"model.load_weights('best_model.h5')\nprint(\"Лучшая модель загружена!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission = pd.read_csv(SAMPLE_SUBMISSION)\nprint(f\"Количество тестовых примеров: {len(sample_submission)}\")\nsample_submission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Извлечение признаков из тестовых данных...\")\n\nX_test = []\ntest_fnames = []\n\nfor fname in tqdm(sample_submission['fname']):\n    file_path = os.path.join(TEST_DIR, fname)\n    features = extract_features(file_path)\n    \n    if features is not None:\n        X_test.append(features)\n        test_fnames.append(fname)\n\nX_test = np.array(X_test)\nX_test = X_test[..., np.newaxis]\n\nprint(f\"\\nРазмерность тестовых признаков: {X_test.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Создание предсказаний\nprint(\"Генерация предсказаний...\")\npredictions = model.predict(X_test, batch_size=BATCH_SIZE, verbose=1)\n\n# Получение топ-3 предсказаний для каждого примера\ntop3_predictions = []\n\nfor pred in predictions:\n    # Получение индексов топ-3 предсказаний\n    top3_indices = np.argsort(pred)[-3:][::-1]\n    # Преобразование индексов в названия меток\n    top3_labels = label_encoder.inverse_transform(top3_indices)\n    # Объединение через пробелы\n    top3_predictions.append(' '.join(top3_labels))\n\nprint(f\"\\nСгенерировано {len(top3_predictions)} предсказаний\")\nprint(f\"\\nПримеры предсказаний:\")\nfor i in range(min(5, len(top3_predictions))):\n    print(f\"  {test_fnames[i]}: {top3_predictions[i]}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Создание файла submission","metadata":{}},{"cell_type":"code","source":"# Создание DataFrame для submission\nsubmission = pd.DataFrame({\n    'fname': test_fnames,\n    'label': top3_predictions\n})\n\n# Сохранение файла submission\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"✓ Файл submission создан: submission.csv\")\nprint(f\"\\nПервые 10 предсказаний:\")\nsubmission.head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Проверка формата submission\nprint(\"Проверка формата submission файла:\")\nprint(f\"  Размер submission: {submission.shape}\")\nprint(f\"  Требуемый размер: {sample_submission.shape}\")\nprint(f\"  Столбцы совпадают: {list(submission.columns) == list(sample_submission.columns)}\")\nprint(f\"  Количество строк совпадает: {len(submission) == len(sample_submission)}\")\n\nif submission.shape == sample_submission.shape:\n    print(\"\\n✓ Формат submission файла корректный!\")\nelse:\n    print(\"\\n✗ ОШИБКА: Неверный формат submission файла!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}