{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!lscpu","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This needs to be run locally - Exceeds Allocated Memory","metadata":{}},{"cell_type":"code","source":"#-------------------------------------------\n# imports\n#---------------------------------------------\nimport librosa\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas as pd\ntqdm.pandas()\nfrom multiprocessing import Process\n#-------------------------------------------\n# proc params\n#---------------------------------------------\nnum_proc=8 # based on number of cpu's\ndata_div=500 # division of data\n#-------------------------------------------\n# helpers\n#---------------------------------------------\ndef create_dir(base,ext):\n    '''\n        creates a directory extending base\n        args:\n            base    =   base path \n            ext     =   the folder to create\n    '''\n    _path=os.path.join(base,ext)\n    if not os.path.exists(_path):\n        os.mkdir(_path)\n    return _path\n\ndef read_file(_path,sr=16000):\n    try:\n        speech,sr=librosa.load(_path,sr=sr)\n        return speech\n    except Exception as e:\n        read_file(_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_as_numpy(csv_path,data_path,iden):\n    \n    save_dir=create_dir(os.getcwd(),f\"{iden}\")\n    df=pd.read_csv(csv_path)\n    df=df[[\"path\",\"sentence\"]]\n    df[\"path\"]=df[\"path\"].progress_apply(lambda x:os.path.join(data_path,x))\n    dicts=[df[idx:idx+data_div] for idx in range(0,len(df),data_div)]\n    print(\"number of splits:\",len(dicts))\n    def _execute(idx):        \n        _df=dicts[idx]\n        _df.reset_index(drop=True,inplace=False)\n        for ridx in tqdm(range(len(_df))):\n            _path=_df.iloc[ridx,0]\n            _data=read_file(_path)\n            base=os.path.basename(_path)\n            base=base.replace(\".mp3\",\".npy\")\n            save=os.path.join(save_dir,base)\n            np.save(save,_data)\n\n    # ----------------------------------------------------------\n    def run(start,end):\n        process_list=[]\n        for idx in range(start,end):\n            p =  Process(target= _execute, args = [idx])\n            p.start()\n            process_list.append(p)\n        for process in process_list:\n            process.join()\n    #----------------------------------------------------------\n    for i in tqdm(range(0,len(dicts),num_proc)):\n        start=i\n        end=start+num_proc\n        if end>len(dicts):\n            end=len(dicts)-1\n        run(start,end)\n    df[\"path\"]=df[\"path\"].progress_apply(lambda x:os.path.basename(x).replace(\".mp3\",\".npy\"))\n    df.to_csv(f\"{iden}.csv\",index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_as_numpy(csv_path=\"../input/dlsprint/validation.csv\",\n              data_path=\"../input/dlsprint/validation_files/\",\n              iden=\"validation\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_as_numpy(csv_path=\"../input/dlsprint/train.csv\",\n              data_path=\"../input/dlsprint/train_files/\",\n              iden=\"train\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}