{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":1898.526756,"end_time":"2023-10-19T15:48:54.716185","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-10-19T15:17:16.189429","version":"2.4.0"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8900,"databundleVersionId":862232,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"902d04a9","cell_type":"markdown","source":"Загрузим CSV таблицы и определим количество уникальных классов в тестовой выборке","metadata":{"papermill":{"duration":0.003912,"end_time":"2023-10-19T15:17:19.472677","exception":false,"start_time":"2023-10-19T15:17:19.468765","status":"completed"},"tags":[]}},{"id":"2e0067e9","cell_type":"code","source":"import pandas as pd\n\n# 8304 Oblizanov Alexander\n\n\n\n# read csv data\n\ntrain = pd.read_csv(\"../input/freesound-audio-tagging/train.csv\")\n\ntest = pd.read_csv(\"../input/freesound-audio-tagging/sample_submission.csv\")\n\n\n\n# check labels\n\nunique_labels = train.label.unique()\n\nprint(\"Labels:\", unique_labels)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-11-12T19:39:31.809618Z","iopub.execute_input":"2024-11-12T19:39:31.809996Z","iopub.status.idle":"2024-11-12T19:39:32.305355Z","shell.execute_reply.started":"2024-11-12T19:39:31.809957Z","shell.execute_reply":"2024-11-12T19:39:32.304267Z"},"papermill":{"duration":0.901398,"end_time":"2023-10-19T15:17:20.377515","exception":false,"start_time":"2023-10-19T15:17:19.476117","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"398c242b","cell_type":"markdown","source":"Прочтем один из файлов, вырежем тишину и построим wave график для него с помощью библиотеки librosa. ","metadata":{"papermill":{"duration":0.003316,"end_time":"2023-10-19T15:17:20.384606","exception":false,"start_time":"2023-10-19T15:17:20.38129","status":"completed"},"tags":[]}},{"id":"6480ae8e","cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport IPython.display as ipd\n\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n\nimport librosa\n\nimport librosa.display\n\n\n\nrecord_prefix = '../input/freesound-audio-tagging/audio_train/'\n\n\n\n# read test file and show wave\n\ndata1, sr1 = librosa.load(record_prefix + train.loc[0].fname)\n\ndata1_trimmed, _ = librosa.effects.trim(data1)\n\nlibrosa.display.waveshow(data1_trimmed, sr=sr1)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:39:32.307243Z","iopub.execute_input":"2024-11-12T19:39:32.307614Z","iopub.status.idle":"2024-11-12T19:39:49.238096Z","shell.execute_reply.started":"2024-11-12T19:39:32.307548Z","shell.execute_reply":"2024-11-12T19:39:49.237192Z"},"papermill":{"duration":10.069399,"end_time":"2023-10-19T15:17:30.457383","exception":false,"start_time":"2023-10-19T15:17:20.387984","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9955d687","cell_type":"markdown","source":"Возьмем по одной записи на класс и построим графики для 16 классов, чтобы визуально оценить разницу между различными звуками.","metadata":{"papermill":{"duration":0.00366,"end_time":"2023-10-19T15:17:30.46539","exception":false,"start_time":"2023-10-19T15:17:30.46173","status":"completed"},"tags":[]}},{"id":"78ee12b9","cell_type":"code","source":"# let's take one record for each label to analyze\n\nfiles = []\n\nfor label in unique_labels:\n\n    label_files = record_prefix + train.loc[train['label'] == label].fname\n\n    files.append(label_files.iloc[0])\n\n\n\n# let's try some charts to see if they show difference that can be used by model\n\n# first, wave\n\n\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\n\nfor i in range(16):\n\n    data_i, sr_i = librosa.load(files[i])\n\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n\n    librosa.display.waveshow(data_i_trimmed, sr=sr_i, ax=ax[i % 4, int(i / 4)])\n\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:39:49.239751Z","iopub.execute_input":"2024-11-12T19:39:49.240294Z","iopub.status.idle":"2024-11-12T19:40:24.324685Z","shell.execute_reply.started":"2024-11-12T19:39:49.240248Z","shell.execute_reply":"2024-11-12T19:40:24.323495Z"},"papermill":{"duration":28.416654,"end_time":"2023-10-19T15:17:58.885714","exception":false,"start_time":"2023-10-19T15:17:30.46906","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"f0acdb0b","cell_type":"markdown","source":"Из графиков видно, что некоторые классы довольно похожи друг на друга. Попробуем другой вид графиков - mfcc","metadata":{"papermill":{"duration":0.007291,"end_time":"2023-10-19T15:17:58.900443","exception":false,"start_time":"2023-10-19T15:17:58.893152","status":"completed"},"tags":[]}},{"id":"ac4ceac6","cell_type":"code","source":"# second, mfcc\n\n\n\nrate = 44100\n\n\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\n\nfor i in range(16):\n\n    data_i, sr_i = librosa.core.load(files[i], sr=rate)\n\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n\n    data_i_trimmed = data_i_trimmed[:2*rate]\n\n    mfcc_i = librosa.feature.mfcc(y=data_i_trimmed, sr=rate, n_mfcc=64)\n\n    librosa.display.specshow(mfcc_i, x_axis='time', ax=ax[i % 4, int(i / 4)])\n\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:40:24.327021Z","iopub.execute_input":"2024-11-12T19:40:24.327448Z","iopub.status.idle":"2024-11-12T19:40:28.933916Z","shell.execute_reply.started":"2024-11-12T19:40:24.327404Z","shell.execute_reply":"2024-11-12T19:40:28.932851Z"},"papermill":{"duration":3.140339,"end_time":"2023-10-19T15:18:02.047863","exception":false,"start_time":"2023-10-19T15:17:58.907524","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4f70cc77","cell_type":"markdown","source":"В двумерном представлении различия классов более существенные. Попробуем также спектрограмму (сжатую)","metadata":{"papermill":{"duration":0.011172,"end_time":"2023-10-19T15:18:02.070673","exception":false,"start_time":"2023-10-19T15:18:02.059501","status":"completed"},"tags":[]}},{"id":"a85f8655","cell_type":"code","source":"# third, spec\n\n\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\n\nfor i in range(16):\n\n    data_i, sr_i = librosa.load(files[i])\n\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n\n    X = librosa.feature.melspectrogram(y=data_i_trimmed, sr=sr_i, n_mels=128)\n\n    Xdb = librosa.amplitude_to_db(abs(X))\n\n    librosa.display.specshow(Xdb, sr=sr_i, x_axis='time', y_axis='log', ax=ax[i % 4, int(i / 4)])\n\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:40:28.935379Z","iopub.execute_input":"2024-11-12T19:40:28.935843Z","iopub.status.idle":"2024-11-12T19:40:35.452056Z","shell.execute_reply.started":"2024-11-12T19:40:28.935794Z","shell.execute_reply":"2024-11-12T19:40:35.451015Z"},"papermill":{"duration":5.009423,"end_time":"2023-10-19T15:18:07.091088","exception":false,"start_time":"2023-10-19T15:18:02.081665","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"39a503f7","cell_type":"markdown","source":"Спектрограмма содержит наибольшее количество информации, но мел-кепстральные коэффициенты могут быть полезнее в меньшей размерности. Использование спектрограммы сильно усложнит модель ввиду большой размерности, поэтому в качестве входных данных будет использована MFCC.\n\n\n\nОптимальным типом модели для решения этой задачи является сверточная нейронная сеть. Для использования с MFCC необходимы 2D слоя свертки.","metadata":{"papermill":{"duration":0.017661,"end_time":"2023-10-19T15:18:07.126143","exception":false,"start_time":"2023-10-19T15:18:07.108482","status":"completed"},"tags":[]}},{"id":"fd4c1c69","cell_type":"code","source":"!pip install resampy\n\nimport resampy\n\ntrain.set_index(\"fname\", inplace=True)\n\ntest.set_index(\"fname\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:40:35.453395Z","iopub.execute_input":"2024-11-12T19:40:35.45376Z","iopub.status.idle":"2024-11-12T19:40:48.513253Z","shell.execute_reply.started":"2024-11-12T19:40:35.453724Z","shell.execute_reply":"2024-11-12T19:40:48.511974Z"},"papermill":{"duration":10.305795,"end_time":"2023-10-19T15:18:17.449358","exception":false,"start_time":"2023-10-19T15:18:07.143563","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"58d95746","cell_type":"code","source":"from keras.layers import (Convolution2D, GlobalAveragePooling2D, BatchNormalization, Flatten,\n\n                          GlobalMaxPool2D, MaxPool2D, concatenate, Activation, Dropout, Input, Dense)\n\nfrom keras.utils import Sequence, to_categorical\n\nfrom keras import losses, models, optimizers\n\nfrom keras.activations import relu, softmax\n\nfrom keras import backend as K\n\n\n\nclass Config(object):\n\n    def __init__(self,\n\n                 sampling_rate=16000, audio_duration=2, n_classes=41,\n\n                 use_mfcc=False, n_folds=10, learning_rate=0.0001, \n\n                 max_epochs=50, n_mfcc=20, n_mels=128):\n\n        self.sampling_rate = sampling_rate\n\n        self.audio_duration = audio_duration\n\n        self.n_classes = n_classes\n\n        self.use_mfcc = use_mfcc\n\n        self.n_mfcc = n_mfcc\n\n        self.n_folds = n_folds\n\n        self.learning_rate = learning_rate\n\n        self.max_epochs = max_epochs\n\n        self.n_mels = n_mels\n\n\n\n        self.audio_length = self.sampling_rate * self.audio_duration\n\n        if self.use_mfcc:\n\n            self.dim = (self.n_mfcc, 1 + int(np.floor(self.audio_length/512)), 1)\n\n        else:\n\n            self.dim = (self.n_mels, 1 + int(np.floor(self.audio_length/512)), 1)\n\n            \n\ndef prepare_data(df, config, data_dir):\n\n    X = np.empty(shape=(df.shape[0], config.dim[0], config.dim[1], 1))\n\n    for i, fname in enumerate(df.index):\n\n        file_path = data_dir + fname\n\n        data, _ = librosa.load(file_path, sr=config.sampling_rate)\n\n        data, _ = librosa.effects.trim(data)\n\n        if len(data) > config.audio_length:\n\n            max_offset = len(data) - config.audio_length\n\n            offset = np.random.randint(max_offset)\n\n            data = data[offset:(config.audio_length+offset)]\n\n        else:\n\n            if config.audio_length > len(data):\n\n                max_offset = config.audio_length - len(data)\n\n                offset = np.random.randint(max_offset)\n\n            else:\n\n                offset = 0\n\n            data = np.pad(data, (offset, config.audio_length - len(data) - offset), \"constant\")\n\n        if config.use_mfcc:\n\n            data = librosa.feature.mfcc(y=data, sr=config.sampling_rate, n_mfcc=config.n_mfcc)\n\n        else: \n\n            data = librosa.feature.melspectrogram(y=data, sr=config.sampling_rate, n_mels=config.n_mels)\n\n        data = np.expand_dims(data, axis=-1)\n\n        X[i,] = data\n\n    return X\n\n\n\nconfig = Config(sampling_rate=44100, audio_duration=2, n_folds=5, \n\n                learning_rate=0.001, use_mfcc=True, n_mfcc=40, n_mels=128)\n\n\n\nunique_labels = list(train.label.unique())\n\nlabel_idx = {label: i for i, label in enumerate(unique_labels)}\n\ntrain[\"label_idx\"] = train.label.apply(lambda x: label_idx[x])\n\ny_train = to_categorical(train.label_idx, num_classes=config.n_classes)","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:40:48.514884Z","iopub.execute_input":"2024-11-12T19:40:48.515296Z","iopub.status.idle":"2024-11-12T19:41:00.222136Z","shell.execute_reply.started":"2024-11-12T19:40:48.515255Z","shell.execute_reply":"2024-11-12T19:41:00.221147Z"},"papermill":{"duration":9.943182,"end_time":"2023-10-19T15:18:27.410621","exception":false,"start_time":"2023-10-19T15:18:17.467439","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"6140b6ab","cell_type":"code","source":"X_train = prepare_data(train, config, '../input/freesound-audio-tagging/audio_train/')\n\nX_test = prepare_data(test, config, '../input/freesound-audio-tagging/audio_test/')\n\n\n\nmean = np.mean(X_train, axis=0)\n\nstd = np.std(X_train, axis=0)\n\n\n\nX_train = (X_train - mean)/std\n\nX_test = (X_test - mean)/std","metadata":{"execution":{"iopub.status.busy":"2024-11-12T19:41:00.223216Z","iopub.execute_input":"2024-11-12T19:41:00.223793Z"},"papermill":{"duration":1519.677354,"end_time":"2023-10-19T15:43:47.106089","exception":false,"start_time":"2023-10-19T15:18:27.428735","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"ad7fe004","cell_type":"markdown","source":"Построим модель 2D свертки. Она будет состоять из 4 этапов свертки, которые затем будут выпрямлены и приведены к слою классификации с функцией активации softmax. Для исключения переобучения будут использованы слои Dropout. \n\nДля обучения будет использован оптимизатор Adam, функция потерь - категориальная кросс-энтропия, которая является наиболее подходящей для модели классификации.","metadata":{"papermill":{"duration":0.017386,"end_time":"2023-10-19T15:43:47.141893","exception":false,"start_time":"2023-10-19T15:43:47.124507","status":"completed"},"tags":[]}},{"id":"acd9af9e","cell_type":"code","source":"print(config.dim)","metadata":{"papermill":{"duration":0.026015,"end_time":"2023-10-19T15:43:47.185246","exception":false,"start_time":"2023-10-19T15:43:47.159231","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"02d5a652","cell_type":"code","source":"def construct_model(config):\n\n    \n\n    input_layer = Input(shape=(config.dim[0],config.dim[1],1))\n\n    \n\n    x = Convolution2D(32, (4, 4), padding=\"same\")(input_layer)\n\n    x = BatchNormalization()(x)\n\n    x = Activation(\"relu\")(x)\n\n    x = MaxPool2D()(x)\n\n    \n\n    x = Convolution2D(32, (8, 8), padding=\"same\")(x)\n\n    x = BatchNormalization()(x)\n\n    x = Activation(\"relu\")(x)\n\n    x = MaxPool2D()(x)\n\n    \n\n    x = Convolution2D(32, (16, 16), padding=\"same\")(x)\n\n    x = BatchNormalization()(x)\n\n    x = Activation(\"relu\")(x)\n\n    x = MaxPool2D()(x)\n\n    \n\n    x = Convolution2D(32, (16, 16), padding=\"same\")(x)\n\n    x = BatchNormalization()(x)\n\n    x = Activation(\"relu\")(x)\n\n    x = MaxPool2D()(x)\n\n\n\n    x = Flatten()(x)\n\n    x = Dense(64)(x)\n\n    x = BatchNormalization()(x)\n\n    x = Activation(\"relu\")(x)\n\n    \n\n    output = Dense(config.n_classes, activation=softmax)(x)\n\n    \n\n    model = models.Model(inputs=input_layer, outputs=output)\n\n    \n\n    model.compile(\n\n        optimizer=optimizers.Adam(config.learning_rate), \n\n        loss=losses.categorical_crossentropy, \n\n        metrics=['acc']\n\n    )\n\n    \n\n    return model","metadata":{"papermill":{"duration":0.027885,"end_time":"2023-10-19T15:43:47.230116","exception":false,"start_time":"2023-10-19T15:43:47.202231","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4ae45536","cell_type":"markdown","source":"Обучим модель на подготовленных данных MFCC","metadata":{"papermill":{"duration":0.016677,"end_time":"2023-10-19T15:43:47.264571","exception":false,"start_time":"2023-10-19T15:43:47.247894","status":"completed"},"tags":[]}},{"id":"11af3327","cell_type":"code","source":"print(len(X_test))\n\nprint(len(test))","metadata":{"papermill":{"duration":0.069169,"end_time":"2023-10-19T15:43:47.350517","exception":false,"start_time":"2023-10-19T15:43:47.281348","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"7728fd4c","cell_type":"code","source":"import os\n\nimport shutil\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom keras.callbacks import (EarlyStopping, LearningRateScheduler,\n\n                             ModelCheckpoint, TensorBoard, ReduceLROnPlateau)\n\n\n\nPREDICTION_FOLDER = \"predictions_2d_conv\"\n\nif not os.path.exists(PREDICTION_FOLDER):\n\n    os.mkdir(PREDICTION_FOLDER)\n\nif os.path.exists('logs/' + PREDICTION_FOLDER):\n\n    shutil.rmtree('logs/' + PREDICTION_FOLDER)\n\n\n\nskf = StratifiedKFold(n_splits=config.n_folds)\n\nfor i, (train_split, val_split) in enumerate(skf.split(X_train, train.label_idx)):\n\n    K.clear_session()\n\n    X, y, X_val, y_val = X_train[train_split], y_train[train_split], X_train[val_split], y_train[val_split]\n\n    checkpoint = ModelCheckpoint('best_%d.h5'%i, monitor='val_loss', verbose=1, save_best_only=True)\n\n    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n\n    callbacks_list = [checkpoint, early]\n\n    print(\"#\"*50)\n\n    print(\"Fold: \", i)\n\n    model = construct_model(config)\n\n    history = model.fit(X, y, validation_data=(X_val, y_val), callbacks=callbacks_list, \n\n                        batch_size=64, epochs=config.max_epochs)\n\n    model.load_weights('best_%d.h5'%i)\n\n\n\n    # Save train predictions\n\n    predictions = model.predict(X_train, batch_size=64, verbose=1)\n\n    np.save(PREDICTION_FOLDER + \"/train_predictions_%d.npy\"%i, predictions)\n\n\n\n    # Save test predictions\n\n    predictions = model.predict(X_test, batch_size=64, verbose=1)\n\n    np.save(PREDICTION_FOLDER + \"/test_predictions_%d.npy\"%i, predictions)\n\n\n\n    # Make a submission file\n\n    top_3 = np.array(unique_labels)[np.argsort(-predictions, axis=1)[:, :3]]\n\n    predicted_labels = [' '.join(list(x)) for x in top_3]\n\n    test_sub = pd.read_csv('../input/freesound-audio-tagging/sample_submission.csv')\n\n    test_sub['label'] = predicted_labels\n\n    test_sub[['label']].to_csv(PREDICTION_FOLDER + \"/predictions_%d.csv\"%i)","metadata":{"papermill":{"duration":302.56278,"end_time":"2023-10-19T15:48:49.930484","exception":false,"start_time":"2023-10-19T15:43:47.367704","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"cb119aa2","cell_type":"code","source":"pred_list = []\n\nfor i in range(config.n_folds):\n\n    pred_list.append(np.load(\"./predictions_2d_conv/test_predictions_%d.npy\"%i))\n\nprediction = np.ones_like(pred_list[0])\n\nfor pred in pred_list:\n\n    prediction = prediction*pred\n\nprediction = prediction**(1./len(pred_list))\n\n# Make a submission file\n\ntop_3 = np.array(unique_labels)[np.argsort(-prediction, axis=1)[:, :3]]\n\npredicted_labels = [' '.join(list(x)) for x in top_3]\n\ntest = pd.read_csv('../input/freesound-audio-tagging/sample_submission.csv')\n\ntest['label'] = predicted_labels\n\ntest[['fname', 'label']].to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":0.294158,"end_time":"2023-10-19T15:48:50.438487","exception":false,"start_time":"2023-10-19T15:48:50.144329","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}