{"cells":[{"metadata":{},"cell_type":"markdown","source":"[Official Competition Link](https://www.kaggle.com/c/freesound-audio-tagging-2019/overview)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport h5py\nimport tables\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport IPython\nimport IPython.display as ipd\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom tqdm import tqdm_notebook\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Helper functions\ndef load_train_data_df(mode='curated'):\n    # Load training data filenames and labels (raw -> multilabels are represented as a string with comma separated values)\n    csv_path = f'/kaggle/input/freesound-audio-tagging-2019/train_{mode}.csv'\n    raw_df = pd.read_csv(csv_path, index_col='fname')\n        \n    # Extract list of expected labels\n    sub = pd.read_csv('/kaggle/input/freesound-audio-tagging-2019/sample_submission.csv', index_col='fname')\n    labels_list = sub.columns.values \n\n    # Encode multi-labels in a binary vector\n    splitted_labels = [ labels.split(',') for labels in raw_df['labels'].values ]\n    encoder = MultiLabelBinarizer()\n    encoded_labels = encoder.fit_transform(splitted_labels)\n\n    # Create a new pandas Dataframe to represent training labels as binary vectors\n    labels_df = pd.DataFrame(data=encoded_labels, index=list(raw_df.index), columns=labels_list)\n    \n    return labels_df\n\ndef listen_sample(sample, sr=44100):\n    return IPython.display.display(ipd.Audio(data=sample, rate=sr))\n\ndef load_sample(path, trim=True):\n    input_data = tf.io.read_file(path)\n    sample, _ = tf.audio.decode_wav(input_data)\n    sample = sample.numpy().flatten()\n    if trim:\n        sample , _ = librosa.effects.trim(sample)\n    return sample\n\ndef compress(x, eps=1e-6):\n    x_min = np.min(x, keepdims=True)\n    x_max = np.max(x, keepdims=True)\n    x = 255 * (x - x_min)/(x_max - x_min + eps)\n    return x.astype(np.uint8)\n\ndef save_h5(savepath, X):\n    with tables.open_file(savepath, mode='w') as h5_file:\n        filters = tables.Filters(complib='zlib', complevel=1)\n        for filename, x in X.items():\n            h5_file.create_carray('/', f\"t{filename.split('.')[0]}\", obj=x, filters=filters)\n\ndef load_h5(filenames, h5_filename):\n    if not isinstance(filenames, list):\n        filenames = [filenames]\n    with h5py.File(h5_filename, mode='r') as dataset:\n        samples = [ dataset[f][()] for f in filenames ]\n        return samples\n\ndef save_features(wav_paths, savepath='train_curated_wav', format='pkl', n_splits=1):\n    \"\"\" Save compressed features (wav) into the specified format. Increase n_splits if you go out of RAM. \n    In terms of compression ratio: h5 (best) > pkl > npy.\n    In terms of read speed: pkl (best) > npy > h5.\n    \"\"\"\n    assert format in ['pkl', 'npy', 'h5'], 'Wrong format argument !'\n    \n    start_time = time.time()\n\n    for split_idx, split_paths in enumerate(np.array_split(wav_paths, n_splits)):\n        X = {}\n        for path in split_paths:\n            filename = path.split('/')[-1]\n            x = load_sample(path, trim=True)\n            x = compress(x)\n            X[filename] = x\n\n        savepath_split = savepath + ('', f'_{split_idx + 1}')[n_splits > 1]\n        if format == 'pkl':\n            save_pkl(savepath_split + '.pkl', X)\n        elif format == 'h5':\n            save_h5(savepath_split + '.h5', X)\n        elif format == 'npy':\n            save_npy(savepath_split, X)\n\n        print(f'Successfully saved .wav features in {savepath_split} ! (took {time.time() - start_time:.2f}s).')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Unzip all files (~8mn)\n!mkdir -p ../data/train/curated\n!mkdir -p ../data/train/noisy\n!mkdir -p ../data/test\n\n!unzip -q /kaggle/input/freesound-audio-tagging-2019/train_curated.zip -d ../data/train/curated/wav \n!unzip -q /kaggle/input/freesound-audio-tagging-2019/train_noisy.zip -d ../data/train/noisy/wav\n!unzip -q /kaggle/input/freesound-audio-tagging-2019/test.zip -d ../data/test/wav","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load data filenames and labels\ncurated_train_labels = load_train_data_df(mode='curated')\nnoisy_train_labels = load_train_data_df(mode='noisy')\ntest_labels = pd.read_csv('/kaggle/input/freesound-audio-tagging-2019/sample_submission.csv', index_col='fname')\n\n# Main info about the training/testing sets\nprint(f'{curated_train_labels.shape[1]} possible classes.')\nprint(f'{curated_train_labels.shape[0]} curated training samples.')\nprint(f'{noisy_train_labels.shape[0]} noisy training samples.')\nprint(f'{test_labels.shape[0]} test samples.')\n\ncurated_train_wav_paths = '../data/train/curated/wav/' + curated_train_labels.index.values\nnoisy_train_wav_paths = '../data/train/noisy/wav/' + noisy_train_labels.index.values\ntest_wav_paths = '../data/test/wav/' + test_labels.index.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsave_features(curated_train_wav_paths, savepath='../train_curated_wav', format='h5', n_splits=1) # 2mn\nsave_features(noisy_train_wav_paths, savepath='../train_noisy_wav', format='h5', n_splits=2) # 20mn\nsave_features(test_wav_paths, savepath='../test_wav', format='h5', n_splits=1) # 2mn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}