{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8900,"databundleVersionId":862232,"sourceType":"competition"},{"sourceId":1378,"sourceType":"modelInstanceVersion","modelInstanceId":1163,"modelId":162}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import Packages","metadata":{"execution":{"iopub.status.busy":"2024-11-23T19:09:58.059466Z","iopub.execute_input":"2024-11-23T19:09:58.060299Z","iopub.status.idle":"2024-11-23T19:09:58.08539Z","shell.execute_reply.started":"2024-11-23T19:09:58.060232Z","shell.execute_reply":"2024-11-23T19:09:58.084091Z"}}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport IPython\nimport IPython.display as ipd\nimport librosa\nimport librosa.display\nimport pickle\nimport joblib\nimport random\nimport cv2\nimport wave\nimport torch\nimport torchaudio\n\nfrom scipy.signal import wiener\nfrom sklearn.model_selection import train_test_split, KFold, ShuffleSplit\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import LabelEncoder\n\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.keras.utils.data_utils import Sequence\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.optimizers import AdamW\nfrom tensorflow.keras.applications import Xception\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom transformers import Wav2Vec2Processor, HubertModel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:28:34.458299Z","iopub.execute_input":"2024-12-12T06:28:34.458834Z","iopub.status.idle":"2024-12-12T06:29:03.162391Z","shell.execute_reply.started":"2024-12-12T06:28:34.458772Z","shell.execute_reply":"2024-12-12T06:29:03.160936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Training Data","metadata":{"execution":{"iopub.status.busy":"2024-11-23T19:09:58.059466Z","iopub.execute_input":"2024-11-23T19:09:58.060299Z","iopub.status.idle":"2024-11-23T19:09:58.08539Z","shell.execute_reply.started":"2024-11-23T19:09:58.060232Z","shell.execute_reply":"2024-11-23T19:09:58.084091Z"}}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/freesound-audio-tagging/train_post_competition.csv\", dtype=str)\nprint(train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.164285Z","iopub.execute_input":"2024-12-12T06:29:03.164966Z","iopub.status.idle":"2024-12-12T06:29:03.209177Z","shell.execute_reply.started":"2024-12-12T06:29:03.164927Z","shell.execute_reply":"2024-12-12T06:29:03.207857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.211515Z","iopub.execute_input":"2024-12-12T06:29:03.211943Z","iopub.status.idle":"2024-12-12T06:29:03.281801Z","shell.execute_reply.started":"2024-12-12T06:29:03.211909Z","shell.execute_reply":"2024-12-12T06:29:03.280255Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Explore the unique classes","metadata":{}},{"cell_type":"code","source":"print(\"Number of training examples=\", train.shape[0], \"  Number of classes=\", len(train.label.unique()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.284747Z","iopub.execute_input":"2024-12-12T06:29:03.285163Z","iopub.status.idle":"2024-12-12T06:29:03.297990Z","shell.execute_reply.started":"2024-12-12T06:29:03.285128Z","shell.execute_reply":"2024-12-12T06:29:03.296665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.label.unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.299487Z","iopub.execute_input":"2024-12-12T06:29:03.299861Z","iopub.status.idle":"2024-12-12T06:29:03.316887Z","shell.execute_reply.started":"2024-12-12T06:29:03.299814Z","shell.execute_reply":"2024-12-12T06:29:03.315178Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Audio Samples by category (manually verified/not verified)","metadata":{"execution":{"iopub.status.busy":"2024-11-23T19:46:39.971982Z","iopub.execute_input":"2024-11-23T19:46:39.972451Z","iopub.status.idle":"2024-11-23T19:46:56.206304Z","shell.execute_reply.started":"2024-11-23T19:46:39.972414Z","shell.execute_reply":"2024-11-23T19:46:56.204393Z"}}},{"cell_type":"code","source":"train['manually_verified'] = train['manually_verified'].astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.318760Z","iopub.execute_input":"2024-12-12T06:29:03.319136Z","iopub.status.idle":"2024-12-12T06:29:03.334685Z","shell.execute_reply.started":"2024-12-12T06:29:03.319101Z","shell.execute_reply":"2024-12-12T06:29:03.333531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"category_group = train.groupby(['label', 'manually_verified']).size().unstack(fill_value=0)\n\nsorted_category_group = category_group.reindex(category_group.sum(axis=1).sort_values().index)\n\nplt.figure(figsize=(16, 10))\nplot = sorted_category_group.plot(kind='bar', stacked=True, title=\"Number of Audio Samples per Category\", figsize=(16,10))\nplot.set_xlabel(\"Category\")\nplot.set_ylabel(\"Number of Samples\")\nplt.legend(title='Manually Verified')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:03.336050Z","iopub.execute_input":"2024-12-12T06:29:03.336515Z","iopub.status.idle":"2024-12-12T06:29:04.121992Z","shell.execute_reply.started":"2024-12-12T06:29:03.336465Z","shell.execute_reply":"2024-12-12T06:29:04.120690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"verification_counts = train['manually_verified'].value_counts()\nprint(verification_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:04.123444Z","iopub.execute_input":"2024-12-12T06:29:04.123802Z","iopub.status.idle":"2024-12-12T06:29:04.133449Z","shell.execute_reply.started":"2024-12-12T06:29:04.123768Z","shell.execute_reply":"2024-12-12T06:29:04.131722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Minimum samples per category = ', min(train.label.value_counts()))\nprint('Maximum samples per category = ', max(train.label.value_counts()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:04.135327Z","iopub.execute_input":"2024-12-12T06:29:04.135983Z","iopub.status.idle":"2024-12-12T06:29:04.170929Z","shell.execute_reply.started":"2024-12-12T06:29:04.135927Z","shell.execute_reply":"2024-12-12T06:29:04.169771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels = (train.label.value_counts() / len(train)).to_frame().sort_index().T\n\ntrain_labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:04.174220Z","iopub.execute_input":"2024-12-12T06:29:04.174627Z","iopub.status.idle":"2024-12-12T06:29:04.207804Z","shell.execute_reply.started":"2024-12-12T06:29:04.174593Z","shell.execute_reply":"2024-12-12T06:29:04.206715Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Sample File","metadata":{"execution":{"iopub.status.busy":"2024-11-23T19:47:44.394534Z","iopub.execute_input":"2024-11-23T19:47:44.39497Z","iopub.status.idle":"2024-11-23T19:47:51.468265Z","shell.execute_reply.started":"2024-11-23T19:47:44.394933Z","shell.execute_reply":"2024-11-23T19:47:51.466565Z"}}},{"cell_type":"code","source":"random_row = train.sample(n=1)\nrandom_filename = random_row['fname'].values[0]\nlabel = random_row['label'].values[0]\n\naudio_dir = '/kaggle/input/freesound-audio-tagging/audio_train'\naudio_path = f\"{audio_dir}/{random_filename}\"\nsample, sr = librosa.load(audio_path, sr=None)\n\nprint(f\"Playing audio: {random_filename}\")\nipd.display(ipd.Audio(sample, rate=sr))\n\nprint(f'Length: {len(sample)/sr:.2f}s')\nprint(f'Label: {label}')\nprint(f'Sample Rate: {sr}')\n\nfig, ax = plt.subplots(4, 1, figsize=(16, 10))\n\nlibrosa.display.waveshow(sample, sr=sr, ax=ax[0])\nax[0].set(title='Temporal Signal', xlabel='Time (s)', ylabel='Amplitude')\n\n# STFT - Short Term Fourier Transform: Computes the amplitude of the frequencies for different bands over time.\nstft_result = librosa.stft(sample)\nstft_db = librosa.amplitude_to_db(abs(stft_result))\nlibrosa.display.specshow(stft_db, sr=sr, x_axis='time', y_axis='log', ax=ax[1])\nax[1].set(title='STFT Spectrogram', xlabel='Time (s)', ylabel='Frequency (Hz)')\n\n# MFCC Mel Frequency Cepstral Coefficients: Describe the instantaneous spectral envelope shape of the speech signal.\nmfccs = librosa.feature.mfcc(y=sample, sr=sr, n_mfcc=13)\nlibrosa.display.specshow(mfccs, sr=sr, x_axis='time', ax=ax[2])\nax[2].set(title='MFCC', xlabel='Time (s)', ylabel='MFCC Coefficients')\n\n#Log-Mel spectrogram: Similar to STFT but represented in the Mel scale (log transformation of the frequency scale)\nmel_spec = librosa.feature.melspectrogram(y=sample, sr=sr)\nlog_mel_spec = librosa.power_to_db(mel_spec)\nlibrosa.display.specshow(log_mel_spec, sr=sr, x_axis='time', y_axis='mel', ax=ax[3])\nax[3].set(title='Log-Mel Spectrogram', xlabel='Time (s)', ylabel='Mel Frequency (Hz)')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:04.209506Z","iopub.execute_input":"2024-12-12T06:29:04.210605Z","iopub.status.idle":"2024-12-12T06:29:21.615183Z","shell.execute_reply.started":"2024-12-12T06:29:04.210546Z","shell.execute_reply":"2024-12-12T06:29:21.614029Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create a plot to explore the length of the files based on nframe ","metadata":{"execution":{"iopub.status.busy":"2024-11-23T19:47:44.394534Z","iopub.execute_input":"2024-11-23T19:47:44.39497Z","iopub.status.idle":"2024-11-23T19:47:51.468265Z","shell.execute_reply.started":"2024-11-23T19:47:44.394933Z","shell.execute_reply":"2024-11-23T19:47:51.466565Z"}}},{"cell_type":"code","source":"train['nframes'] = train['fname'].apply(lambda f: wave.open('../input/freesound-audio-tagging/audio_train/' + f).getnframes())\n\n_, ax = plt.subplots(figsize=(16, 4))\nsns.violinplot(ax=ax, x=\"label\", y=\"nframes\", data=train)\nplt.xticks(rotation=90)\nplt.title('Distribution of audio frames, per label', fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:29:21.616709Z","iopub.execute_input":"2024-12-12T06:29:21.617606Z","iopub.status.idle":"2024-12-12T06:31:01.875876Z","shell.execute_reply.started":"2024-12-12T06:29:21.617542Z","shell.execute_reply":"2024-12-12T06:31:01.874273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(figsize=(16,5))\ntrain.nframes.hist(bins=100)\nplt.suptitle('Frame Length Distribution in Train and Test', ha='center', fontsize='large');","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:01.877380Z","iopub.execute_input":"2024-12-12T06:31:01.877759Z","iopub.status.idle":"2024-12-12T06:31:02.256869Z","shell.execute_reply.started":"2024-12-12T06:31:01.877725Z","shell.execute_reply":"2024-12-12T06:31:02.255669Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Filter the dataframe for only manually verfified samples","metadata":{}},{"cell_type":"code","source":"### Filter for manually verified data only\n\ntrain = train[train['manually_verified'] == 1]\n\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.258784Z","iopub.execute_input":"2024-12-12T06:31:02.259233Z","iopub.status.idle":"2024-12-12T06:31:02.275243Z","shell.execute_reply.started":"2024-12-12T06:31:02.259169Z","shell.execute_reply":"2024-12-12T06:31:02.274021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.276601Z","iopub.execute_input":"2024-12-12T06:31:02.276962Z","iopub.status.idle":"2024-12-12T06:31:02.292647Z","shell.execute_reply.started":"2024-12-12T06:31:02.276927Z","shell.execute_reply":"2024-12-12T06:31:02.291227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Split into Training and Validation","metadata":{}},{"cell_type":"code","source":"train_files, test_files = train_test_split(train, test_size=0.3, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.294355Z","iopub.execute_input":"2024-12-12T06:31:02.294803Z","iopub.status.idle":"2024-12-12T06:31:02.309495Z","shell.execute_reply.started":"2024-12-12T06:31:02.294765Z","shell.execute_reply":"2024-12-12T06:31:02.307705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_files.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.311789Z","iopub.execute_input":"2024-12-12T06:31:02.312427Z","iopub.status.idle":"2024-12-12T06:31:02.325935Z","shell.execute_reply.started":"2024-12-12T06:31:02.312370Z","shell.execute_reply":"2024-12-12T06:31:02.324592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_files.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.327316Z","iopub.execute_input":"2024-12-12T06:31:02.327779Z","iopub.status.idle":"2024-12-12T06:31:02.342512Z","shell.execute_reply.started":"2024-12-12T06:31:02.327674Z","shell.execute_reply":"2024-12-12T06:31:02.340982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Training Data Augmentation (As Required)\n- Best results so far have been without data augmentation.","metadata":{}},{"cell_type":"markdown","source":"import soundfile as sf\n\naudio_dir = '/kaggle/input/freesound-audio-tagging/audio_train'\n\naudio_dir = '/kaggle/input/freesound-audio-tagging/audio_train'\naugmented_audio_dir = '/kaggle/working/augmented_audio'\n\nos.makedirs(augmented_audio_dir, exist_ok=True)\n\ndef time_stretch(y, sr):\n    return librosa.effects.time_stretch(y, rate=np.random.uniform(0.8, 1.2))\n\ndef pitch_shift(y, sr):\n    return librosa.effects.pitch_shift(y, sr=sr, n_steps=np.random.randint(-2, 3))\n\ndef add_noise(y, sr):\n    return y + 0.005 * np.random.randn(len(y))\n\ndef time_shift(y, sr):\n    return np.roll(y, shift=np.random.randint(-sr // 10, sr // 10))\n\ndef volume_scale(y, sr):\n    return y * np.random.uniform(0.7, 1.3)\n\ndef augment_audio(file_path, output_dir, file_name):\n    y, sr = librosa.load(file_path, sr=16000)\n\n    augmentations = [\n        time_stretch,\n        pitch_shift,\n        add_noise,\n        time_shift,\n        volume_scale\n    ]\n\n    augmentation = np.random.choice(augmentations)\n    y_augmented = augmentation(y, sr)\n\n    if len(y_augmented) < 5 * sr:\n        y_augmented = np.pad(y_augmented, (0, 5 * sr - len(y_augmented)))\n    else:\n        y_augmented = y_augmented[:5 * sr]\n\n    if np.max(np.abs(y_augmented)) > 0:\n        y_augmented = y_augmented / np.max(np.abs(y_augmented))\n\n    augmented_file_path = os.path.join(output_dir, file_name)\n    sf.write(augmented_file_path, y_augmented, sr)\n\n    return augmented_file_path\n\naugmented_data = []\n\nfor idx, row in train_files.iterrows():\n    file_name = row['fname']\n    file_path = os.path.join(audio_dir, file_name)\n\n    augmented_data.append({'fname': file_name, 'label': row['label'], 'path': file_path})\n\n    for aug_idx in range(2):\n        augmented_file_name = f\"{os.path.splitext(file_name)[0]}_aug_{aug_idx}.wav\"\n        augmented_file_path = augment_audio(file_path, augmented_audio_dir, augmented_file_name)\n        augmented_data.append({'fname': augmented_file_name, 'label': row['label'], 'path': augmented_file_path})\n\naugmented_train_files = pd.DataFrame(augmented_data)\n\ntrain_files = augmented_train_files\n\ntrain_audio_paths = train_files['path'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-12-03T00:56:59.181958Z","iopub.execute_input":"2024-12-03T00:56:59.182394Z","iopub.status.idle":"2024-12-03T01:00:18.566404Z","shell.execute_reply.started":"2024-12-03T00:56:59.182359Z","shell.execute_reply":"2024-12-03T01:00:18.565033Z"}}},{"cell_type":"code","source":"#train_files.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.344140Z","iopub.execute_input":"2024-12-12T06:31:02.344566Z","iopub.status.idle":"2024-12-12T06:31:02.359809Z","shell.execute_reply.started":"2024-12-12T06:31:02.344531Z","shell.execute_reply":"2024-12-12T06:31:02.358300Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Extraction\n- Use pretrained models to extract VGGish, MFCC, and HuBERT features to use for neural network\n- Also tried with LogMel Features, but best results are without","metadata":{}},{"cell_type":"code","source":"# Define Class and functions for feature extraction\naudio_dir = '/kaggle/input/freesound-audio-tagging/audio_train'\n\nvggish_model = hub.load('https://kaggle.com/models/google/vggish/frameworks/TensorFlow2/variations/vggish/versions/1')\nprocessor = Wav2Vec2Processor.from_pretrained(\"facebook/hubert-large-ls960-ft\")\nhubert_model = HubertModel.from_pretrained(\"facebook/hubert-large-ls960-ft\")\n\n# VGGish feature extraction\nclass VGGishFeatureExtractor(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        return np.array([self.extract_vggish_embeddings(file_path) for file_path in X])\n    \n    def extract_vggish_embeddings(self, file_path):\n        sample, _ = librosa.load(file_path, sr=16000)\n        sample, _ = librosa.effects.trim(sample, top_db=10)\n        if len(sample) < 5 * 16000:\n            sample = np.pad(sample, (0, 5 * 16000 - len(sample)))\n        else:\n            sample = sample[:5 * 16000]\n        if np.max(np.abs(sample)) > 0:\n            sample = sample / np.max(np.abs(sample))\n        embeddings = vggish_model(sample)\n        embeddings_flattened = embeddings.numpy().flatten()\n        return embeddings_flattened\n\n# HuBERT feature extraction\nclass HubertFeatureExtractor(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        return np.array([self.extract_features(path) for path in X])\n\n    def extract_features(self, file_path):\n        sample, sr = librosa.load(file_path, sr=16000)\n        sample, _ = librosa.effects.trim(sample, top_db=10)\n        \n        if len(sample) < 5 * 16000:\n            sample = np.pad(sample, (0, 5 * 16000 - len(sample)))\n        else:\n            sample = sample[:5 * 16000]\n\n        if np.max(np.abs(sample)) > 0:\n            sample = sample / np.max(np.abs(sample))\n\n        input_values = processor(sample, sampling_rate=16000, return_tensors=\"pt\").input_values\n        with torch.no_grad():\n            hidden_states = hubert_model(input_values).last_hidden_state\n        features = hidden_states.mean(dim=1).squeeze().numpy()\n\n        return features\n\n# MFCC feature extraction\nclass MFCCFeatureExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, n_mfcc=40, sr=16000):\n        self.n_mfcc = n_mfcc\n        self.sr = sr\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        return np.array([self.extract_features(path) for path in X])\n\n    def extract_features(self, file_path):\n        y, sr = librosa.load(file_path, sr=self.sr)\n        y, _ = librosa.effects.trim(y, top_db=10)\n\n        if len(y) < 5 * sr:\n            y = np.pad(y, (0, 5 * sr - len(y)))\n        else:\n            y = y[:5 * sr]\n\n        if np.max(np.abs(y)) > 0:\n            y = y / np.max(np.abs(y))\n\n        mfccs = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.n_mfcc)\n        return mfccs.mean(axis=1)\n\n# LogMel feature extraction\nclass LogMelFeatureExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, n_mels=64, sr=16000):\n        self.n_mels = n_mels\n        self.sr = sr\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        return np.array([self.extract_features(path) for path in X])\n\n    def extract_features(self, file_path):\n        y, sr = librosa.load(file_path, sr=self.sr)\n        y, _ = librosa.effects.trim(y, top_db=10)\n\n        if len(y) < 5 * sr:\n            y = np.pad(y, (0, 5 * sr - len(y)))\n        else:\n            y = y[:5 * sr]\n\n        if np.max(np.abs(y)) > 0:\n            y = y / np.max(np.abs(y))\n\n        mel_spectrogram = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=self.n_mels)\n        log_mel_spectrogram = librosa.power_to_db(mel_spectrogram)\n        return log_mel_spectrogram.mean(axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:02.361464Z","iopub.execute_input":"2024-12-12T06:31:02.361898Z","iopub.status.idle":"2024-12-12T06:31:12.869051Z","shell.execute_reply.started":"2024-12-12T06:31:02.361860Z","shell.execute_reply":"2024-12-12T06:31:12.867876Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Pipeline","metadata":{}},{"cell_type":"code","source":"# Create the pipeline for to apply feature extraction\nfrom sklearn.pipeline import FeatureUnion, Pipeline\n\ncombined_features = FeatureUnion([\n    (\"vggish\", VGGishFeatureExtractor()),\n    (\"mfcc\", MFCCFeatureExtractor(n_mfcc=40)),\n    (\"hubert\", HubertFeatureExtractor())\n    #(\"logmel\", LogMelFeatureExtractor(n_mels=64)) ## commented out for best results\n])\n\nfeature_pipeline = Pipeline([\n    (\"features\", combined_features)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:12.870734Z","iopub.execute_input":"2024-12-12T06:31:12.871102Z","iopub.status.idle":"2024-12-12T06:31:12.879499Z","shell.execute_reply.started":"2024-12-12T06:31:12.871067Z","shell.execute_reply":"2024-12-12T06:31:12.877915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create Training Feature Arrays","metadata":{}},{"cell_type":"code","source":"train_audio_paths = [os.path.join(audio_dir, fname) for fname in train_files['fname']]\ntest_audio_paths = [os.path.join(audio_dir, fname) for fname in test_files['fname']]\n\n#Combine all extracted features into train and test sets\nX_train_combined = feature_pipeline.fit_transform(train_audio_paths)\nX_test_combined = feature_pipeline.transform(test_audio_paths)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T06:31:12.881478Z","iopub.execute_input":"2024-12-12T06:31:12.882067Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Prepare Features for Training","metadata":{}},{"cell_type":"code","source":"X_train_combined.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test_combined.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_files.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save Features, Pipeline","metadata":{}},{"cell_type":"code","source":"#Save X_train/X_test for use in training\nnp.save('X_train_vmh_mv_strat_final.npy', X_train_combined)\nnp.save('X_test_vmh_mv_strat_final.npy', X_test_combined)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save original dataframes for creating y_tarain/y_test in the training steps\ntrain_files.to_pickle('train_mv_strat_final.pkl')\ntest_files.to_pickle('test_mv_strat_final.pkl')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save feature extraction pipleine to use for final evaluation prep\njoblib.dump(feature_pipeline, 'feature_extraction_pipeline_mv_vmh_strat_final.joblib')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}