{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#-------------------------------\n# generall imports\n#-------------------------------\nimport os \nimport random\nimport pandas as pd \nimport librosa\nimport numpy as np  \nimport warnings\n\nfrom tqdm.auto import tqdm\nfrom glob import glob\nfrom pandarallel import pandarallel\nfrom IPython.display import display,Audio\n\npandarallel.initialize(progress_bar=True,nb_workers=8)\ntqdm.pandas()\nwarnings.filterwarnings('ignore')\n\n#-------------------------------\ndef load_data(path,sr=16000):\n    \"\"\"loads a wav\"\"\"\n    wave,_= librosa.load(path, sr=sr, mono=True)\n    return wave\n#-------------------------------\ndef load_trimmed_data(path,sr=16000):\n    \"\"\"loads a wav\"\"\"\n    wave,_= librosa.load(path, sr=sr, mono=True)\n    wave=np.trim_zeros(wave)\n    return wave\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T19:42:11.196785Z","iopub.execute_input":"2022-07-06T19:42:11.197718Z","iopub.status.idle":"2022-07-06T19:42:13.395605Z","shell.execute_reply.started":"2022-07-06T19:42:11.197591Z","shell.execute_reply":"2022-07-06T19:42:13.393526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trimming: Removing leading and ending silences","metadata":{}},{"cell_type":"code","source":"#---------------\n# data filtering\n#---------------\ndef filter_votes(x):\n    up=x[\"up_votes\"]\n    down=x[\"down_votes\"]\n    if up-down<=0:\n        return None\n    elif up==0:\n        return None\n    else:\n        return up\n# ------------------------- train data----------------------------------------\ntrain_path=\"../input/train-wavs-voted-dl-sprint/train_files_wav\"\ntrain_df=pd.read_csv(\"../input/dlsprint/train.csv\")\nprint(\"Total Data before filtering:\",len(train_df))\ntrain_df[\"up_votes\"]=train_df.progress_apply(lambda x:filter_votes(x),axis=1)\ntrain_df.dropna(subset = ['up_votes'],inplace=True)\nprint(\"Total Data after filtering:\",len(train_df))\ntrain_df[\"path\"]=train_df[\"path\"].progress_apply(lambda x:os.path.join(train_path,x).replace(\".mp3\",\".wav\"))\ntrain_df=train_df[[\"path\",\"sentence\"]]\ntrain_df.reset_index(drop=True,inplace=True)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:42:13.398546Z","iopub.execute_input":"2022-07-06T19:42:13.400106Z","iopub.status.idle":"2022-07-06T19:42:21.679959Z","shell.execute_reply.started":"2022-07-06T19:42:13.400037Z","shell.execute_reply":"2022-07-06T19:42:21.678948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inspect a random file and effect of trimming","metadata":{}},{"cell_type":"code","source":"idx=random.randint(0,len(train_df)-1)\n_path=train_df.iloc[idx,0]\naudio=load_data(_path)\nprint(\"shape of file:\",audio.shape)\ndisplay(Audio(data=audio, rate=16000))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:42:21.681299Z","iopub.execute_input":"2022-07-06T19:42:21.682005Z","iopub.status.idle":"2022-07-06T19:42:21.722738Z","shell.execute_reply.started":"2022-07-06T19:42:21.681964Z","shell.execute_reply":"2022-07-06T19:42:21.721172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio=np.trim_zeros(audio)\ndisplay(Audio(data=audio, rate=16000))\nprint(\"shape of file:\",audio.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:42:21.726224Z","iopub.execute_input":"2022-07-06T19:42:21.726849Z","iopub.status.idle":"2022-07-06T19:42:21.844301Z","shell.execute_reply.started":"2022-07-06T19:42:21.726799Z","shell.execute_reply":"2022-07-06T19:42:21.842655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### An Extreme example ","metadata":{}},{"cell_type":"code","source":"idx=6257\n_path=train_df.iloc[idx,0]\naudio=load_data(_path)\nprint(\"shape of file:\",audio.shape)\ndisplay(Audio(data=audio, rate=16000))\naudio=np.trim_zeros(audio)\ndisplay(Audio(data=audio, rate=16000))\nprint(\"shape of file:\",audio.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:46:19.947153Z","iopub.execute_input":"2022-07-06T19:46:19.947718Z","iopub.status.idle":"2022-07-06T19:46:20.071364Z","shell.execute_reply.started":"2022-07-06T19:46:19.947679Z","shell.execute_reply":"2022-07-06T19:46:20.070077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# trimmed length inspection","metadata":{}},{"cell_type":"code","source":"train_df[\"audio_len\"]=train_df[\"path\"].parallel_apply(lambda x:load_trimmed_data(x).shape[0])\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:42:21.846374Z","iopub.execute_input":"2022-07-06T19:42:21.847000Z","iopub.status.idle":"2022-07-06T19:45:05.163170Z","shell.execute_reply.started":"2022-07-06T19:42:21.846946Z","shell.execute_reply":"2022-07-06T19:45:05.161259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"max lenght of training audio(1-D numpy) :\", max(train_df[\"audio_len\"].tolist()))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:47:29.770922Z","iopub.execute_input":"2022-07-06T19:47:29.771330Z","iopub.status.idle":"2022-07-06T19:47:29.781160Z","shell.execute_reply.started":"2022-07-06T19:47:29.771286Z","shell.execute_reply":"2022-07-06T19:47:29.779329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#-----------------------------\n# validation data\n#-----------------------------\nval_path=\"../input/validation-fileswav-format/validation_files_wav/\"\nval_df=pd.read_csv(\"../input/dlsprint/validation.csv\")\nval_df=val_df[[\"path\",\"sentence\"]]\nval_df[\"path\"]=val_df[\"path\"].progress_apply(lambda x:os.path.join(val_path,x).replace(\".mp3\",\".wav\"))\nval_df[\"audio_len\"]=val_df[\"path\"].parallel_apply(lambda x:load_trimmed_data(x).shape[0])\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T19:50:02.905746Z","iopub.execute_input":"2022-07-06T19:50:02.906241Z","iopub.status.idle":"2022-07-06T20:07:21.905014Z","shell.execute_reply.started":"2022-07-06T19:50:02.906203Z","shell.execute_reply":"2022-07-06T20:07:21.902646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"max lenght of validation audio(1-D numpy) :\", max(val_df[\"audio_len\"].tolist()))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T20:24:19.410637Z","iopub.execute_input":"2022-07-06T20:24:19.412558Z","iopub.status.idle":"2022-07-06T20:24:19.424695Z","shell.execute_reply.started":"2022-07-06T20:24:19.412432Z","shell.execute_reply":"2022-07-06T20:24:19.423054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}