{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport librosa\nimport seaborn as sns\nimport matplotlib\nfrom sklearn.model_selection import train_test_split\nimport gc\nimport wave\nimport joblib\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.preprocessing as pp\nfrom tqdm import tqdm\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau,EarlyStopping\nimport os \n\n\n%matplotlib inline\nmatplotlib.style.use('ggplot')\n%config InlineBackend.figure_format = 'retina'\nsns.set(font_scale=1.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig_size=(15,8)\nsns.set(rc={'figure.figsize':fig_size})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/freesound-audio-tagging/train_post_competition.csv')\nTRAIN_FILES_PATH = '../input/freesound-audio-tagging/audio_train/'\nRANDOM_STATE=42","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby('label').describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby('label')['label'].count().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_RATE = 44100\nNUM_MFCC = 40\nN_FFT = 2048\nHOP_LENGTH = 512\nEPOCHS=50\nAUDIO_LENGTH = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_file_length(file_name):\n    file_path = TRAIN_FILES_PATH + file_name \n    return 0\n    #return  wave.open(file_path).getnframes()\n\ndf['length'] = df['fname'].apply(get_file_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, ax = plt.subplots(figsize=(16, 4))\nsns.violinplot(ax=ax, x=\"label\", y=\"length\", data=df)\nplt.xticks(rotation=90)\nplt.title('Distribution of audio frames, per label', fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COMPLETE_RUN = True\nLABELS = list(df['label'].unique())\nlabel_idx = {label: i for i, label in enumerate(LABELS)}\n\nif not COMPLETE_RUN:\n    df = df[:2000]\n\ndf[\"label_idx\"] = df['label'].apply(lambda x: label_idx[x])\nX = df['fname'].apply(lambda x: TRAIN_FILES_PATH + x).values\ny = df['label_idx'].values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_path = TRAIN_FILES_PATH + df.head(1).values.tolist()[0][0]\nfile_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data, _ = librosa.core.load(file_path, sr=SAMPLE_RATE)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"S = librosa.feature.melspectrogram(data, sr=SAMPLE_RATE, n_fft=N_FFT, hop_length=HOP_LENGTH, n_mels=NUM_MFCC)\nS.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = librosa.feature.mfcc(data, SAMPLE_RATE, n_mfcc=NUM_MFCC, n_fft=N_FFT, hop_length=HOP_LENGTH)\ndata.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_datasets(X, y, test_size, validation_size):\n    # create train, validation and test split\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size)\n    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size=validation_size)\n\n    return X_train, X_validation, X_test, y_train, y_validation, y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Audio length in seconds\ndef pre_process(file_path,\n                audio_length=AUDIO_LENGTH, \n                sample_rate=SAMPLE_RATE,\n                num_mfcc=NUM_MFCC, \n                n_fft=N_FFT, \n                hop_length=HOP_LENGTH):\n    \n    data, _ = librosa.core.load(file_path, sr=sample_rate, res_type=\"kaiser_fast\")\n    input_length = audio_length * sample_rate\n    # Random offset / Padding\n    if len(data) > input_length:\n        max_offset = len(data) - input_length\n        offset = np.random.randint(max_offset)\n        data = data[offset:(input_length+offset)]\n    else:\n        if input_length > len(data):\n            max_offset = input_length - len(data)\n            offset = np.random.randint(max_offset)\n        else:\n            offset = 0\n        data = np.pad(data, (offset, input_length - len(data) - offset), \"constant\")\n\n    data = librosa.feature.mfcc(data, sample_rate, n_mfcc=num_mfcc, n_fft=n_fft, hop_length=hop_length)\n    data = data.T\n    #data = librosa.feature.melspectrogram(data, sr=sample_rate, n_fft=n_fft, hop_length=hop_length, n_mels=num_mfcc)\n    data = data[..., np.newaxis]\n    return file_path, data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_PATH = \"./output/\"\nCACHE_PATH = \"./cache/\"\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nPROCESSING_BATCH_SIZE = 16\nimport shutil\nif os.path.exists(CACHE_PATH):\n    shutil.rmtree(CACHE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def path_to_id(file_path):\n    return file_path.split(\"/\")[-1].split(\".\")[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_object(obj,name, folder_path = OUTPUT_PATH):\n    if not os.path.exists(folder_path):\n        os.makedirs(folder_path)\n    path=folder_path + name + '.pkl'\n    joblib.dump(obj, path) \n\ndef load_object(name, folder_path = OUTPUT_PATH):\n    return joblib.load(folder_path + name + '.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_features(file_id):\n    return load_object(file_id, folder_path=CACHE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_features_from_files(data):\n    file_paths = list(set(data))\n    dataset = tf.data.Dataset.from_tensor_slices(file_paths)\n    dataset = dataset.map(lambda x: tf.numpy_function(func=pre_process,\n              inp=[x], Tout=[tf.string,tf.float32]), num_parallel_calls=AUTOTUNE)\n                    \n    # https://www.tensorflow.org/guide/data\n    for paths, images in tqdm(dataset.batch(PROCESSING_BATCH_SIZE)):\n        for index, extracted_feature in enumerate(images):\n            file_id = path_to_id(paths[index].numpy().decode(\"utf-8\"))\n            save_object(extracted_feature.numpy(),file_id, folder_path=CACHE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extract_features_from_files(X)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dim = (NUM_MFCC,1 + int(np.floor(AUDIO_LENGTH*SAMPLE_RATE/HOP_LENGTH)), 1)\n\ndim = (1 + int(np.floor(AUDIO_LENGTH*SAMPLE_RATE/HOP_LENGTH)),NUM_MFCC, 1)\n\n\n\nX_NEW = np.empty(shape=(X.shape[0], dim[0], dim[1], 1))\nfor i, path in enumerate(X):\n    if i%500 == 0:\n        print(\"created mfcc {}\".format(i))\n    X_NEW[i,] = load_image_features(path_to_id(path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get train, validation, test splits\nmean = np.mean(X_NEW, axis=0)\nstd = np.std(X_NEW, axis=0)\n\nX_NEW = (X_NEW - mean)/std\n\nX_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(X_NEW, y, 0.2, 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SHUFFLE_BUFFER_SIZE = 1000\nBATCH_SIZE = 64\nAUTOTUNE = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_addons as tfa\n@tf.function\ndef augment_cutout(image):\n    image = tf.expand_dims(image, 0)\n    image = tfa.image.random_cutout(image, (5,5), constant_values = 0)\n    return tf.squeeze(image,[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef augment_brightness(image):\n     return tf.image.random_brightness(image, max_delta=0.95, seed=RANDOM_STATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef augment_contrast(image):\n     return tf.image.random_contrast(image, lower=0.1, upper=0.9, seed=RANDOM_STATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef augment_saturation(image):\n     return tf.image.random_saturation(image, lower=0.1, upper=0.9, seed=RANDOM_STATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef translate_tf(image):\n    ratio=tf.random.uniform((2,), minval=-2, maxval=2, dtype=tf.dtypes.int32)\n    ratio=tf.cast(ratio, tf.dtypes.float32)\n    return tfa.image.translate(image, ratio,'BILINEAR')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef augment(image, label):\n    random= tf.random.uniform((1,), minval=0, maxval=5, dtype=tf.dtypes.int32)\n    if random==0:\n        image = augment_cutout(image)\n#     elif random==1:\n#         image = augment_brightness(image)\n#     elif random == 2:\n#         image = augment_contrast(image)\n#     elif random == 4:\n#         image = translate_tf(image)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_set(files, labels, is_training=False):\n    dataset = tf.data.Dataset.from_tensor_slices((files, labels))\n    if is_training:\n        dataset = dataset.shuffle(SHUFFLE_BUFFER_SIZE)\n        dataset = dataset.map(augment, num_parallel_calls=AUTOTUNE)        \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_data_set(X_train, y_train, is_training=True)\nval_dataset = get_data_set(X_validation, y_validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_file_batch, sample_label_batch = next(iter(train_dataset))\nprint(sample_file_batch.shape) \nprint(sample_label_batch.shape) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNNBlock(layers.Layer):\n    def __init__(self, \n                 out_channel, \n                 kernel_size=3, \n                 dropout=None, \n                 has_maxpool=True,\n                 maxpool_strides=None,\n                 max_pool_size = (2, 2)):\n        super(CNNBlock,self).__init__()\n        self.conv = layers.Conv2D(out_channel, kernel_size)\n        self.bn = layers.BatchNormalization()\n        self.activation = layers.Activation(\"relu\")\n        self.maxpool = None\n        self.dropout = None\n        if has_maxpool:\n            self.maxpool = layers.MaxPool2D(pool_size=max_pool_size, strides=maxpool_strides, padding=\"same\")\n        if dropout:\n            self.dropout = layers.Dropout(dropout)\n    \n    def call(self, input_tensor, training=False):\n        x = self.conv(input_tensor)\n        x = self.bn(x, training=training)\n        x = self.activation(x)\n        if self.maxpool:\n            x = self.maxpool(x)\n        if self.dropout:\n            x = self.dropout(x)\n        return x    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_uncompiled_model():\n    inputs = layers.Input(shape=(dim[0],dim[1],1))\n    model = keras.Sequential(\n         [\n          CNNBlock(32,kernel_size=(3,3),dropout=.3), \n          CNNBlock(32,kernel_size=(3, 3),dropout=.3), \n          CNNBlock(32,kernel_size= (3,3),dropout=.3), \n          layers.Flatten(), \n          layers.Dense(64),\n          layers.BatchNormalization(),\n          layers.Activation(\"relu\"),\n          layers.Dropout(0.5),\n          layers.Dense(len(LABELS))]\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_compiled_model():\n    model = get_uncompiled_model()\n    model.compile(\n        optimizer=keras.optimizers.Adam(learning_rate=0.01),\n        loss=[keras.losses.SparseCategoricalCrossentropy(from_logits=True),],\n        metrics=[\"accuracy\"],\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_compiled_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduce_lr_on_plateau = ReduceLROnPlateau(monitor='val_loss',\n                       factor=0.5,\n                       patience=3,\n                       min_lr=0.000001,\n                       verbose=1)  \nfilepath = 'model.h5'\ncheckpoint=tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True,\n        save_weights_only=True, mode='min')\n#early_stopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=8)\ncallbacks_list = [reduce_lr_on_plateau,checkpoint]\n# history = model.fit(X_train, \n#                     y_train, \n#                     batch_size=BATCH_SIZE,\n#                     validation_data=(X_validation, y_validation), \n#                     epochs=EPOCHS, \n#                     verbose=1, \n#                     callbacks=callbacks_list).history\n\nhistory = model.fit(train_dataset, \n                    epochs=EPOCHS, \n                    verbose=1,\n                    validation_data=val_dataset,\n                    callbacks=callbacks_list).history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nplt.ylabel(\"Loss (training and validation)\")\nplt.xlabel(\"Training Steps\")\nplt.ylim([0,4])\nplt.plot(history[\"loss\"])\nplt.plot(history[\"val_loss\"])\n\nplt.figure()\nplt.ylabel(\"Accuracy (training and validation)\")\nplt.xlabel(\"Training Steps\")\nplt.ylim([0,1])\nplt.plot(history[\"accuracy\"])\nplt.plot(history[\"val_accuracy\"])\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reference\n[Notebook](https://www.kaggle.com/fizzbuzz/beginner-s-guide-to-audio-data)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}