{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel uses all the cores of the CPU using multiprocessing to clip 5 audio clips. I used this create the audio clip dataset in my local system but I am not able to upload the data. \n\nI have used W&B artifacts to upload the resulting CSV file that contains the metadata. You can use the below code snippet to download that. \n\n```\nimport wandb\nrun = wandb.init()\nartifact = run.use_artifact('ayush-thakur/birdclef/audio_clips_5sec:v0', type='dataset')\nartifact_dir = artifact.download()\n```","metadata":{}},{"cell_type":"markdown","source":"# Imports and Setups","metadata":{}},{"cell_type":"code","source":"import os\nos.environ['WANDB_SILENT'] = \"true\"\nimport re\nimport gc\nimport glob\nimport wandb\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\n%matplotlib inline\n\n# Multiprocessing \nfrom multiprocessing import Pool\nfrom multiprocessing import cpu_count\n\n# Audio specific imports\nimport librosa as lb\nimport librosa.display\nimport soundfile as sf\n\n# W&B login\nwandb.login()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILES = glob.glob(\"../input/birdclef-2021/train_short_audio/*/*.ogg\")\nprint(len(TRAIN_FILES))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVE_DIR = 'birdclef_5second/'\nos.makedirs(SAVE_DIR, exist_ok=True)\n!ls {SAVE_DIR}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Audio Clips using Multiprocessing","metadata":{}},{"cell_type":"code","source":"def chunk(l, n):\n\t# loop over the list in n-sized chunks\n\tfor i in range(0, len(l), n):\n\t\t# yield the current n-sized chunk to the calling function\n\t\tyield l[i: i + n]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"procs = cpu_count()\nprint(procs)\nprocIDs = list(range(0, procs))\n# grab the paths to the input images, then determine the number\n# of images each process will handle\nnumImagesPerProc = len(TRAIN_FILES) / float(procs)\nnumImagesPerProc = int(np.ceil(numImagesPerProc))\n# chunk the image paths into N (approximately) equal sets, one\n# set of image paths for each individual process\nchunkedPaths = list(chunk(TRAIN_FILES, numImagesPerProc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clip_audio_dataset(audio_paths):\n    # Iterate over individual audio paths.\n    for audio_path in tqdm(audio_paths):\n        # Get label name\n        label = audio_path.split('/')[-2]\n        # Make dir\n        os.makedirs(SAVE_DIR+label, exist_ok=True)\n        # Load Audio \n        audio, sr = lb.load(audio_path)\n        # Get the time duration of audio\n        audio_time = len(audio)//sr\n\n        start_sample = 0\n        end_sample = sr*5 # sampling rate is number of samples per second. \n\n        for i in range(audio_time//5):\n            # Get clip\n            audio_clip = audio[start_sample:end_sample]\n            start_sample = end_sample\n            end_sample+=sr*5\n\n            # Write as .ogg file\n            file_name = audio_path.split('/')[-1].split('.')[0]\n            sf.write(f'{SAVE_DIR+label}/{file_name}_{i}.ogg', audio_clip, sr, format='ogg', subtype='vorbis')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"[INFO] launching pool using {} processes...\".format(procs))\npool = Pool(processes=procs)\npool.imap(clip_audio_dataset, chunkedPaths)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# close the pool and wait for all processes to finish\nprint(\"[INFO] waiting for processes to finish...\")\npool.close()\npool.join()\nprint(\"[INFO] multiprocessing complete\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The number of audio clip files generated: ', len(glob.glob(f\"{SAVE_DIR}*/*.ogg\")))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> This is almost 10 times the original number of recordings.a","metadata":{}},{"cell_type":"markdown","source":"# Create `train.csv` file","metadata":{}},{"cell_type":"code","source":"meta_df = pd.read_csv('train_metadata.csv')\nmeta_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUDIO_CLIPS = glob.glob(f\"{SAVE_DIR}*/*.ogg\")\nprint(f'Number of audio clips: {len(AUDIO_CLIPS)}')\nAUDIO_CLIPS[0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"procs = 12\nprint(procs)\nprocIDs = list(range(0, procs))\n# grab the paths to the input images, then determine the number\n# of images each process will handle\nnumImagesPerProc = len(AUDIO_CLIPS) / float(procs)\nnumImagesPerProc = int(np.ceil(numImagesPerProc))\n# chunk the image paths into N (approximately) equal sets, one\n# set of image paths for each individual process\nchunkedPaths = list(chunk(AUDIO_CLIPS, numImagesPerProc))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('train_clips/', exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_train_df(audio_paths):\n    # create local pandas dataframe\n    train_df = pd.DataFrame(columns=meta_df.columns)\n\n    # Iterate over individual audio paths.\n    for i, audio_path in tqdm(enumerate(audio_paths)):\n        filename = audio_path.split('/')[-1].split('.')[0].split('_')[0]\n        audio_clip_name = audio_path.split('/')[-1].split('.')[0]\n        row = meta_df.loc[meta_df['filename']==filename+'.ogg'].replace(f'{filename}.ogg', f'{audio_clip_name}.ogg')\n        train_df = train_df.append(row, ignore_index=True)      \n        \n    pid = os.getpid()\n    train_df.to_csv(f'train_clips/train_{pid}.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"[INFO] launching pool using {} processes...\".format(procs))\npool = Pool(processes=procs)\npool.imap(prepare_train_df, chunkedPaths)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunked_files = glob.glob(f\"train_clips/*.csv\")\nprint(f'Number of chunks: {len(chunked_files)}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_arr = []\nc = 0\nfor chunked_file in chunked_files:\n    tmp_df = pd.read_csv(chunked_file)\n    tmp_df = tmp_df[tmp_df.columns[1:]]\n    df_arr.append(tmp_df)\n    \ntrain_df = pd.concat(df_arr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://www.kaggle.com/shahules/bird-watch-complete-eda-fe\n# Unique eBird codes\nspecies = train_df['primary_label'].value_counts()\n\n# Make bar chart\nfig = go.Figure(data=[go.Bar(y=species.values, x=species.index)],\n                layout=go.Layout(margin=go.layout.Margin(l=0, r=0, b=10, t=50)))\n\n# Show chart\nfig.update_layout(title='Number of traning samples per species')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save as W&B Artifact","metadata":{}},{"cell_type":"code","source":"train_df.to_csv('train_clips.csv', index=False)\n\nrun = wandb.init(project='birdclef', group='Dataset Creation')\n\nraw_dataset = run.use_artifact('ayush-thakur/birdclef/train-metadata:v0', type='dataset')\n\nartifact = wandb.Artifact('audio_clips_5sec', type='dataset')\nartifact.add_file('train_clips.csv')\nrun.log_artifact(artifact)\nrun.finish()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python --version\nimport platform\nprint(platform.platform())\nprint(\"cpu cores: {0}\".format(cpu_count()))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> This notebook was run on a GCP instance with 16 CPU cores. I hope you find this kernel useful for your own experiments.","metadata":{}}]}