{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport  joblib\nimport numpy as np\nimport pandas as pd\nimport librosa as lb\nimport librosa.display\nimport matplotlib.pyplot as plt\n\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"DATA_ROOT = Path(\"../input/rfcx-species-audio-detection\")\nTRAIN_AUDIO_ROOT = Path(\"../input/rfcx-species-audio-detection/train\")\nTEST_AUDIO_ROOT = Path(\"../input/rfcx-species-audio-detection/test\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.DataFrame({\n    \"recording_id\": [path.stem for path in Path(TRAIN_AUDIO_ROOT).glob(\"*.flac\")],\n})\n\ndf_test = pd.DataFrame({\n    \"recording_id\": [path.stem for path in Path(TEST_AUDIO_ROOT).glob(\"*.flac\")],\n})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tools","metadata":{}},{"cell_type":"code","source":"class params:\n    \"\"\"\n    Parameters used for the audio data\n    \"\"\"\n    sr = 32000\n\n    # Melspectrogram\n    n_mels = 128\n    fmin = 20\n    fmax = sr // 2  # Shannon theorem","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_audio(record, sr=16000, root=\"\"):\n    y, _ = lb.load(\n        root.joinpath(record).with_suffix(\".flac\").as_posix(),\n        sr=sr, \n    )\n    return y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_melspec(y, params):\n    \"\"\"\n    Computes a mel-spectrogram and puts it at decibel scale\n    Arguments:\n        y {np array} -- signal\n        params {AudioParams} -- Parameters to use for the spectrogram. Expected to have the attributes sr, n_mels, f_min, f_max\n    Returns:\n        np array -- Mel-spectrogram\n    \"\"\"\n    melspec = lb.feature.melspectrogram(\n        y, sr=params.sr, n_mels=params.n_mels, fmin=params.fmin, fmax=params.fmax,\n    )\n\n    melspec = lb.power_to_db(melspec).astype(np.float32)\n    return melspec","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example","metadata":{}},{"cell_type":"code","source":"y = load_audio(df_train[\"recording_id\"][0], params.sr, TRAIN_AUDIO_ROOT)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"melspec = compute_melspec(y, params)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15, 5))\nimg = librosa.display.specshow(\n    melspec[:, :512], \n#     melspec, \n    sr=params.sr,\n    x_axis='time', \n    y_axis='linear', \n    ax=ax)\nfig.colorbar(img, ax=ax, format=\"%+2.f dB\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"melspec.npy\", melspec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time comparison","metadata":{}},{"cell_type":"code","source":"%%timeit \n\nspec = np.load(\"melspec.npy\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit \n\ny = load_audio(df_train[\"recording_id\"][0], params.sr, TRAIN_AUDIO_ROOT)\nmelspec = compute_melspec(y, params)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"x 3000 improvement ! ","metadata":{}},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"def load_and_save_train(location, record):\n    y = load_audio(record, params.sr, TRAIN_AUDIO_ROOT)\n    melspec = compute_melspec(y, params)\n\n    np.save(location + record + \".npy\", melspec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUT_TRAIN_1 = 'train1/'\nos.mkdir(OUT_TRAIN_1)\n\nOUT_TRAIN_2 = 'train2/'\nos.mkdir(OUT_TRAIN_2)\n\nOUT_TRAIN_3 = 'train3/'\nos.mkdir(OUT_TRAIN_3)\n\nOUT_TRAIN_4 = 'train4/'\nos.mkdir(OUT_TRAIN_4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.shape)\n\nchunk = (df_train.shape[0] // 4) + 1\nfor block in range(4):\n    start = block * chunk\n    stop = start + chunk\n    location = f'train{block+1}/'\n    \n    print(location,start,stop)               \n    \n    _ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(load_and_save_train)(location,record) for record in tqdm(df_train['recording_id'][start:stop].values)\n    )    \n    \n    shutil.make_archive(location, 'zip', location)\n    shutil.rmtree(location)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train['recording_id'][start:stop].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# _ = joblib.Parallel(n_jobs=8)(\n#     joblib.delayed(load_and_save_train)(OUT_TRAIN_1,record) for record in tqdm(df_train['recording_id'].values)\n# )\n\n# _ = joblib.Parallel(n_jobs=8)(\n#     joblib.delayed(load_and_save_train)(OUT_TRAIN_2,record) for record in tqdm(df_train['recording_id'].values)\n# )\n\n# _ = joblib.Parallel(n_jobs=8)(\n#     joblib.delayed(load_and_save_train)(OUT_TRAIN_3,record) for record in tqdm(df_train['recording_id'].values)\n# )\n\n# _ = joblib.Parallel(n_jobs=8)(\n#     joblib.delayed(load_and_save_train)(OUT_TRAIN_4,record) for record in tqdm(df_train['recording_id'].values)\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shutil.make_archive(OUT_TRAIN, 'zip', OUT_TRAIN)\n# shutil.rmtree(OUT_TRAIN)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test","metadata":{}},{"cell_type":"code","source":"def load_and_save_test(record):\n    y = load_audio(record, params.sr, TEST_AUDIO_ROOT)\n    melspec = compute_melspec(y, params)\n\n    np.save(OUT_TEST + record + \".npy\", melspec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUT_TEST = 'test/'\nos.mkdir(OUT_TEST)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = joblib.Parallel(n_jobs=8)(\n    joblib.delayed(load_and_save_test)(record) for record in tqdm(df_test['recording_id'].values)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive(OUT_TEST, 'zip', OUT_TEST)\nshutil.rmtree(OUT_TEST)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}