{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Ray Paralelization of Making Mels Spectrograms using\n\nKey takeaways:\n\n1. Library Ray seems to be outperforming in this setup.\n\n2. It is possible that one can obtain similar performance with some care.  \n\n3. Because of low overhead you do not have to think of big batches with ray.  \n\nNote: one has to be careful though as I found no good wait to check on processes finish except for the dirty method I use below of waiting for 60 seconds of no writing to disk.  Works here.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# General Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib\nmatplotlib.interactive(False)\nmatplotlib.use('Agg')\n\n# find all of the files in the directory\nimport os\nimport gc\nimport sys\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport sys\nfrom tqdm.notebook import tqdm\nimport librosa\nimport librosa.display\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport multiprocessing\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\nprintfile=None\n#open('log.txt','w')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Launching Ray\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ray\nimport ray\nray.shutdown()\nray.init(num_cpus=4,include_webui=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Various Paths and Training","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#gpath='C:/Users/karab/Documents/GitHub/birdsong-recognition/train_audio/'\n\nipath='../input/birdsong-recognition/train_audio/'\nopath= \"./\"\nopath= \"./data\"\n\nbirds = next(os.walk(ipath))[1]\nff=[] # list of all files\ncount=0;\n\nbirds = np.sort(birds)\nbirds = [b for b in birds if b[0]<='h']\n\nfor i, bird in enumerate(birds):\n    for _, _, _files in os.walk(ipath+bird):\n        count = count+1;\n        print(\"{:03} {:7} {:3} files\\t\".format(i,bird,len(_files)),end='\\t' if (i+1)%5 else '\\n',file=printfile)\n        ff.append(_files)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Core","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def saveMel(signal, directory,sr):\n    gc.enable()\n    # spectrogram parameters\n    N_FFT = 1024         # \n    HOP_SIZE = 1024      #  \n    N_MELS = 128         # h   \n    WIN_SIZE = 1024      # \n    WINDOW_TYPE = 'hann' # \n    FEATURE = 'mel'      # \n    FMIN = 1400\n\n    fig = plt.figure(1,frameon=False)\n    fig.set_size_inches(6,6)\n\n    ax = plt.Axes(fig, [0., 0., 1., 1.])\n    ax.set_axis_off()\n    fig.add_axes(ax)\n    \n    S = librosa.feature.melspectrogram(y=signal, sr=sr,\n                                        n_fft=N_FFT,\n                                        hop_length=HOP_SIZE, \n                                        n_mels=N_MELS, \n                                        htk=True, \n                                        fmin=FMIN, # higher limit ##high-pass filter freq.\n                                        fmax=sr/2) # AMPLITUDE\n    librosa.display.specshow(librosa.power_to_db(S**2,ref=np.max), fmin=FMIN) #power = S**2\n    \n    fig.savefig(directory)\n    plt.ioff()\n    #plt.show(block=False)\n    fig.clf()\n    ax.cla()\n    plt.clf()\n    plt.close('all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"size = {'desired': 5,  # [seconds]\n        'minimum': 4,  # [seconds]\n        'stride': 0,  # [seconds]\n        'name': 5  # [number of letters]\n        }  # stride should not be bigger than desired length\n\n@ray.remote\ndef worker(infile,outfile,verbose=True):\n    warnings.filterwarnings('ignore')\n    try:    \n        signal, sr = librosa.load(infile,sr=None)  # sr = sampling rate\n        step = (size['desired']-size['stride']) * sr # length of step between two cuts in seconds\n        count = 0;\n        if VERBOSITY and (sr>0): print('\\n{:03} {:12} {:>6.0f}    {} sample'.format(j, file, len(signal),sr))#, end='' if (j+1)%5 else '\\n')\n        for start, end in (zip(range(0, len(signal), step), range(size['desired']*sr, len(signal), step))):\n            count = count+1\n            os.makedirs(os.path.dirname(outfile), exist_ok=True)\n            saveMel(signal[start:end], outfile.format(count),sr) #scratch: if end-start > size['minimum']*sr else signal[-step:]\n            os.rename(outfile.format(count),outfile.format(count))\n            if VERBOSITY: print(count,end=' ')\n        #print(infile,'completed',file=printfile)\n    except:\n        print('Loading of ',infile, 'failed',file=printfile)\n        #return infile\n\n\nTEST=False\nVERBOSITY=0\n\n\nif TEST: birds = birds[:5]\ntqdm=lambda x,y,z:x\nresults = []\nprint('Number of birds: ', len(birds))\nfor i, bird in tqdm(enumerate(birds),'Main',len(birds)):#enumerate(birds):# \n    for _, _, _files in os.walk(ipath+bird):\n        infiles,outfiles=[],[]\n        if TEST: _files = _files[:5]\n        for j,file in tqdm(enumerate(_files),bird,len(_files)):#enumerate(_files):#\n            infile = os.path.join(ipath, bird, file)\n            outfile = os.path.join(opath, bird, file).replace('.mp3', '_{}.png')\n            results.append(worker.remote(infile,outfile))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dirty pool","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport glob\ncount = len(glob.glob('/kaggle/working/data/*/*'))\ntime.sleep(60)\nwhile count<len(glob.glob('/kaggle/working/data/*/*')):\n    count=len(glob.glob('/kaggle/working/data/*/*'))\n    time.sleep(60)\n    print(count)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PNGs -> tar.gz\n\nSomehow saving png files just in directory leads to not even being able to load this notebook in viewer afterwards or them not being seen so","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tarfile\nimport os\nimport shutil\nimport glob\n\nfile_name = \"data.tar.gz\"\nos.chdir('/kaggle/working/')\ntar = tarfile.open(file_name, \"w:gz\")\nfor f in glob.glob('./data/*/*'):\n    tar.add(f)\n    os.remove(f)\nshutil.rmtree('./data')\ntar.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}