{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Загрузим CSV таблицы и определим количество уникальных классов в тестовой выборке","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n# 8304 Oblizanov Alexander\n\n# read csv data\ntrain = pd.read_csv(\"../input/freesound-audio-tagging/train.csv\")\ntest = pd.read_csv(\"../input/freesound-audio-tagging/sample_submission.csv\")\n\n# check labels\nunique_labels = train.label.unique()\nprint(\"Labels:\", unique_labels)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-19T13:53:46.427393Z","iopub.execute_input":"2023-10-19T13:53:46.427754Z","iopub.status.idle":"2023-10-19T13:53:46.450911Z","shell.execute_reply.started":"2023-10-19T13:53:46.427725Z","shell.execute_reply":"2023-10-19T13:53:46.450171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Прочтем один из файлов, вырежем тишину и построим wave график для него с помощью библиотеки librosa. ","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport IPython.display as ipd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport librosa\nimport librosa.display\n\nrecord_prefix = '../input/freesound-audio-tagging/audio_train/'\n\n# read test file and show wave\ndata1, sr1 = librosa.load(record_prefix + train.loc[0].fname)\ndata1_trimmed, _ = librosa.effects.trim(data1)\nlibrosa.display.waveshow(data1_trimmed, sr=sr1)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-19T13:53:46.452613Z","iopub.execute_input":"2023-10-19T13:53:46.453238Z","iopub.status.idle":"2023-10-19T13:53:46.883279Z","shell.execute_reply.started":"2023-10-19T13:53:46.453204Z","shell.execute_reply":"2023-10-19T13:53:46.882355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Возьмем по одной записи на класс и построим графики для 16 классов, чтобы визуально оценить разницу между различными звуками.","metadata":{}},{"cell_type":"code","source":"# let's take one record for each label to analyze\nfiles = []\nfor label in unique_labels:\n    label_files = record_prefix + train.loc[train['label'] == label].fname\n    files.append(label_files.iloc[0])\n\n# let's try some charts to see if they show difference that can be used by model\n# first, wave\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\nfor i in range(16):\n    data_i, sr_i = librosa.load(files[i])\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n    librosa.display.waveshow(data_i_trimmed, sr=sr_i, ax=ax[i % 4, int(i / 4)])\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2023-10-19T13:53:46.884523Z","iopub.execute_input":"2023-10-19T13:53:46.884750Z","iopub.status.idle":"2023-10-19T13:54:15.259160Z","shell.execute_reply.started":"2023-10-19T13:53:46.884730Z","shell.execute_reply":"2023-10-19T13:54:15.258209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Из графиков видно, что некоторые классы довольно похожи друг на друга. Попробуем другой вид графиков - mfcc","metadata":{}},{"cell_type":"code","source":"# second, mfcc\n\nrate = 44100\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\nfor i in range(16):\n    data_i, sr_i = librosa.core.load(files[i], sr=rate)\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n    data_i_trimmed = data_i_trimmed[:2*rate]\n    mfcc_i = librosa.feature.mfcc(y=data_i_trimmed, sr=rate, n_mfcc=64)\n    librosa.display.specshow(mfcc_i, x_axis='time', ax=ax[i % 4, int(i / 4)])\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2023-10-19T13:54:15.261689Z","iopub.execute_input":"2023-10-19T13:54:15.262061Z","iopub.status.idle":"2023-10-19T13:54:18.163260Z","shell.execute_reply.started":"2023-10-19T13:54:15.262031Z","shell.execute_reply":"2023-10-19T13:54:18.162300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В двумерном представлении различия классов более существенные. Попробуем также спектрограмму (сжатую)","metadata":{}},{"cell_type":"code","source":"# third, spec\n\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(18, 18))\nfor i in range(16):\n    data_i, sr_i = librosa.load(files[i])\n    data_i_trimmed, _ = librosa.effects.trim(data_i)\n    X = librosa.feature.melspectrogram(y=data_i_trimmed, sr=sr_i, n_mels=128)\n    Xdb = librosa.amplitude_to_db(abs(X))\n    librosa.display.specshow(Xdb, sr=sr_i, x_axis='time', y_axis='log', ax=ax[i % 4, int(i / 4)])\n    ax[i % 4, int(i / 4)].title.set_text(unique_labels[i])","metadata":{"execution":{"iopub.status.busy":"2023-10-19T13:54:18.164458Z","iopub.execute_input":"2023-10-19T13:54:18.164792Z","iopub.status.idle":"2023-10-19T13:54:23.084009Z","shell.execute_reply.started":"2023-10-19T13:54:18.164759Z","shell.execute_reply":"2023-10-19T13:54:23.083140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Спектрограмма содержит наибольшее количество информации, но мел-кепстральные коэффициенты могут быть полезнее в меньшей размерности. Использование спектрограммы сильно усложнит модель ввиду большой размерности, поэтому в качестве входных данных будет использована MFCC.\n\nОптимальным типом модели для решения этой задачи является сверточная нейронная сеть. Для использования с MFCC необходимы 2D слоя свертки.","metadata":{}},{"cell_type":"code","source":"!pip install resampy\nimport resampy\ntrain.set_index(\"fname\", inplace=True)\ntest.set_index(\"fname\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T13:54:23.085298Z","iopub.execute_input":"2023-10-19T13:54:23.085644Z","iopub.status.idle":"2023-10-19T13:54:31.254954Z","shell.execute_reply.started":"2023-10-19T13:54:23.085615Z","shell.execute_reply":"2023-10-19T13:54:31.253731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import (Convolution2D, GlobalAveragePooling2D, BatchNormalization, Flatten,\n                          GlobalMaxPool2D, MaxPool2D, concatenate, Activation, Dropout, Input, Dense)\nfrom keras.utils import Sequence, to_categorical\nfrom keras import losses, models, optimizers\nfrom keras.activations import relu, softmax\nfrom keras import backend as K\n\nclass Config(object):\n    def __init__(self,\n                 sampling_rate=16000, audio_duration=2, n_classes=41,\n                 use_mfcc=False, n_folds=10, learning_rate=0.0001, \n                 max_epochs=50, n_mfcc=20, n_mels=128):\n        self.sampling_rate = sampling_rate\n        self.audio_duration = audio_duration\n        self.n_classes = n_classes\n        self.use_mfcc = use_mfcc\n        self.n_mfcc = n_mfcc\n        self.n_folds = n_folds\n        self.learning_rate = learning_rate\n        self.max_epochs = max_epochs\n        self.n_mels = n_mels\n\n        self.audio_length = self.sampling_rate * self.audio_duration\n        if self.use_mfcc:\n            self.dim = (self.n_mfcc, 1 + int(np.floor(self.audio_length/512)), 1)\n        else:\n            self.dim = (self.n_mels, 1 + int(np.floor(self.audio_length/512)), 1)\n            \ndef prepare_data(df, config, data_dir):\n    X = np.empty(shape=(df.shape[0], config.dim[0], config.dim[1], 1))\n    for i, fname in enumerate(df.index):\n        file_path = data_dir + fname\n        data, _ = librosa.load(file_path, sr=config.sampling_rate)\n        data, _ = librosa.effects.trim(data)\n        if len(data) > config.audio_length:\n            max_offset = len(data) - config.audio_length\n            offset = np.random.randint(max_offset)\n            data = data[offset:(config.audio_length+offset)]\n        else:\n            if config.audio_length > len(data):\n                max_offset = config.audio_length - len(data)\n                offset = np.random.randint(max_offset)\n            else:\n                offset = 0\n            data = np.pad(data, (offset, config.audio_length - len(data) - offset), \"constant\")\n        if config.use_mfcc:\n            data = librosa.feature.mfcc(y=data, sr=config.sampling_rate, n_mfcc=config.n_mfcc)\n        else: \n            data = librosa.feature.melspectrogram(y=data, sr=config.sampling_rate, n_mels=config.n_mels)\n        data = np.expand_dims(data, axis=-1)\n        X[i,] = data\n    return X\n\nconfig = Config(sampling_rate=44100, audio_duration=2, n_folds=5, \n                learning_rate=0.001, use_mfcc=True, n_mfcc=40, n_mels=128)\n\nunique_labels = list(train.label.unique())\nlabel_idx = {label: i for i, label in enumerate(unique_labels)}\ntrain[\"label_idx\"] = train.label.apply(lambda x: label_idx[x])\ny_train = to_categorical(train.label_idx, num_classes=config.n_classes)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:22:21.791866Z","iopub.execute_input":"2023-10-19T14:22:21.792728Z","iopub.status.idle":"2023-10-19T14:22:21.810450Z","shell.execute_reply.started":"2023-10-19T14:22:21.792696Z","shell.execute_reply":"2023-10-19T14:22:21.809526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = prepare_data(train, config, '../input/freesound-audio-tagging/audio_train/')\nX_test = prepare_data(test, config, '../input/freesound-audio-tagging/audio_test/')\n\nmean = np.mean(X_train, axis=0)\nstd = np.std(X_train, axis=0)\n\nX_train = (X_train - mean)/std\nX_test = (X_test - mean)/std","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Построим модель 2D свертки. Она будет состоять из 4 этапов свертки, которые затем будут выпрямлены и приведены к слою классификации с функцией активации softmax. Для исключения переобучения будут использованы слои Dropout. \nДля обучения будет использован оптимизатор Adam, функция потерь - категориальная кросс-энтропия, которая является наиболее подходящей для модели классификации.","metadata":{}},{"cell_type":"code","source":"print(config.dim)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:28:01.164185Z","iopub.execute_input":"2023-10-19T14:28:01.164937Z","iopub.status.idle":"2023-10-19T14:28:01.169541Z","shell.execute_reply.started":"2023-10-19T14:28:01.164906Z","shell.execute_reply":"2023-10-19T14:28:01.168578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def construct_model(config):\n    \n    input_layer = Input(shape=(config.dim[0],config.dim[1],1))\n    \n    x = Convolution2D(32, (4, 4), padding=\"same\")(input_layer)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    x = MaxPool2D()(x)\n    \n    x = Convolution2D(32, (8, 8), padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    x = MaxPool2D()(x)\n    \n    x = Convolution2D(32, (16, 16), padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    x = MaxPool2D()(x)\n    \n    x = Convolution2D(32, (16, 16), padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    x = MaxPool2D()(x)\n\n    x = Flatten()(x)\n    x = Dense(64)(x)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    \n    output = Dense(config.n_classes, activation=softmax)(x)\n    \n    model = models.Model(inputs=input_layer, outputs=output)\n    \n    model.compile(\n        optimizer=optimizers.Adam(config.learning_rate), \n        loss=losses.categorical_crossentropy, \n        metrics=['acc']\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:39:34.911324Z","iopub.execute_input":"2023-10-19T14:39:34.911667Z","iopub.status.idle":"2023-10-19T14:39:34.919393Z","shell.execute_reply.started":"2023-10-19T14:39:34.911641Z","shell.execute_reply":"2023-10-19T14:39:34.918304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Обучим модель на подготовленных данных MFCC","metadata":{}},{"cell_type":"code","source":"print(len(X_test))\nprint(len(test))","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:18:58.026970Z","iopub.execute_input":"2023-10-19T14:18:58.027782Z","iopub.status.idle":"2023-10-19T14:18:58.032495Z","shell.execute_reply.started":"2023-10-19T14:18:58.027751Z","shell.execute_reply":"2023-10-19T14:18:58.031215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nfrom sklearn.model_selection import StratifiedKFold\nfrom keras.callbacks import (EarlyStopping, LearningRateScheduler,\n                             ModelCheckpoint, TensorBoard, ReduceLROnPlateau)\n\nPREDICTION_FOLDER = \"predictions_2d_conv\"\nif not os.path.exists(PREDICTION_FOLDER):\n    os.mkdir(PREDICTION_FOLDER)\nif os.path.exists('logs/' + PREDICTION_FOLDER):\n    shutil.rmtree('logs/' + PREDICTION_FOLDER)\n\nskf = StratifiedKFold(n_splits=config.n_folds)\nfor i, (train_split, val_split) in enumerate(skf.split(X_train, train.label_idx)):\n    K.clear_session()\n    X, y, X_val, y_val = X_train[train_split], y_train[train_split], X_train[val_split], y_train[val_split]\n    checkpoint = ModelCheckpoint('best_%d.h5'%i, monitor='val_loss', verbose=1, save_best_only=True)\n    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n    callbacks_list = [checkpoint, early]\n    print(\"#\"*50)\n    print(\"Fold: \", i)\n    model = construct_model(config)\n    history = model.fit(X, y, validation_data=(X_val, y_val), callbacks=callbacks_list, \n                        batch_size=64, epochs=config.max_epochs)\n    model.load_weights('best_%d.h5'%i)\n\n    # Save train predictions\n    predictions = model.predict(X_train, batch_size=64, verbose=1)\n    np.save(PREDICTION_FOLDER + \"/train_predictions_%d.npy\"%i, predictions)\n\n    # Save test predictions\n    predictions = model.predict(X_test, batch_size=64, verbose=1)\n    np.save(PREDICTION_FOLDER + \"/test_predictions_%d.npy\"%i, predictions)\n\n    # Make a submission file\n    top_3 = np.array(unique_labels)[np.argsort(-predictions, axis=1)[:, :3]]\n    predicted_labels = [' '.join(list(x)) for x in top_3]\n    test_sub = pd.read_csv('../input/freesound-audio-tagging/sample_submission.csv')\n    test_sub['label'] = predicted_labels\n    test_sub[['label']].to_csv(PREDICTION_FOLDER + \"/predictions_%d.csv\"%i)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:39:41.527365Z","iopub.execute_input":"2023-10-19T14:39:41.528183Z","iopub.status.idle":"2023-10-19T14:41:52.066854Z","shell.execute_reply.started":"2023-10-19T14:39:41.528155Z","shell.execute_reply":"2023-10-19T14:41:52.065357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_list = []\nfor i in range(config.n_folds):\n    pred_list.append(np.load(\"./predictions_2d_conv/test_predictions_%d.npy\"%i))\nprediction = np.ones_like(pred_list[0])\nfor pred in pred_list:\n    prediction = prediction*pred\nprediction = prediction**(1./len(pred_list))\n# Make a submission file\ntop_3 = np.array(unique_labels)[np.argsort(-prediction, axis=1)[:, :3]]\npredicted_labels = [' '.join(list(x)) for x in top_3]\ntest = pd.read_csv('../input/freesound-audio-tagging/sample_submission.csv')\ntest['label'] = predicted_labels\ntest[['fname', 'label']].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T14:17:45.256093Z","iopub.status.idle":"2023-10-19T14:17:45.256444Z","shell.execute_reply.started":"2023-10-19T14:17:45.256274Z","shell.execute_reply":"2023-10-19T14:17:45.256289Z"},"trusted":true},"execution_count":null,"outputs":[]}]}