{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## Importhing Necessary Libraries for EDA","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport librosa\nimport librosa.display\n\nimport IPython.display as ipd # for displaying the play audio buttom\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:24.449862Z","iopub.execute_input":"2022-07-19T11:35:24.450675Z","iopub.status.idle":"2022-07-19T11:35:27.119547Z","shell.execute_reply.started":"2022-07-19T11:35:24.450569Z","shell.execute_reply":"2022-07-19T11:35:27.118313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading the .csv files","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/dlsprint/train.csv')\nval_df = pd.read_csv('../input/dlsprint/validation.csv')\nsample_submissin_df = pd.read_csv('../input/dlsprint/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:34.060407Z","iopub.execute_input":"2022-07-19T11:35:34.060813Z","iopub.status.idle":"2022-07-19T11:35:36.255308Z","shell.execute_reply.started":"2022-07-19T11:35:34.060780Z","shell.execute_reply":"2022-07-19T11:35:36.254220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Take a look at the .csv files","metadata":{}},{"cell_type":"code","source":"train_df.head(n = 3)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:45.490210Z","iopub.execute_input":"2022-07-19T11:35:45.490904Z","iopub.status.idle":"2022-07-19T11:35:45.517853Z","shell.execute_reply.started":"2022-07-19T11:35:45.490863Z","shell.execute_reply":"2022-07-19T11:35:45.516909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:48.519022Z","iopub.execute_input":"2022-07-19T11:35:48.519714Z","iopub.status.idle":"2022-07-19T11:35:48.676070Z","shell.execute_reply.started":"2022-07-19T11:35:48.519677Z","shell.execute_reply":"2022-07-19T11:35:48.674941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:48.739657Z","iopub.execute_input":"2022-07-19T11:35:48.740071Z","iopub.status.idle":"2022-07-19T11:35:48.877832Z","shell.execute_reply.started":"2022-07-19T11:35:48.740038Z","shell.execute_reply":"2022-07-19T11:35:48.876630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['down_votes'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:49.049301Z","iopub.execute_input":"2022-07-19T11:35:49.049726Z","iopub.status.idle":"2022-07-19T11:35:49.060146Z","shell.execute_reply.started":"2022-07-19T11:35:49.049682Z","shell.execute_reply":"2022-07-19T11:35:49.059013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['up_votes'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:49.329147Z","iopub.execute_input":"2022-07-19T11:35:49.329608Z","iopub.status.idle":"2022-07-19T11:35:49.339922Z","shell.execute_reply.started":"2022-07-19T11:35:49.329574Z","shell.execute_reply":"2022-07-19T11:35:49.338846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Higher the upvotes more accurate the text labels\n\nSamples with higher number of downvote than upper are considered as incorrect.","metadata":{}},{"cell_type":"markdown","source":"## Let't check how label them","metadata":{}},{"cell_type":"code","source":"train_df['locale'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:51.689688Z","iopub.execute_input":"2022-07-19T11:35:51.690086Z","iopub.status.idle":"2022-07-19T11:35:51.711166Z","shell.execute_reply.started":"2022-07-19T11:35:51.690048Z","shell.execute_reply":"2022-07-19T11:35:51.710225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So all the audio files are labled by native bengalies","metadata":{}},{"cell_type":"markdown","source":"##### Columns named  'age', 'gender', 'accents', 'locale' would be removed as they didn't carry important information for the project.","metadata":{}},{"cell_type":"code","source":"train_df = train_df.drop(['age', 'gender', 'accents', 'locale'],axis = 1)\nval_df = val_df.drop(['age', 'gender', 'accents', 'locale' , 'client_id','up_votes','down_votes'],axis = 1)\n#val_df.to_csv('new_validation.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:35:59.140931Z","iopub.execute_input":"2022-07-19T11:35:59.141325Z","iopub.status.idle":"2022-07-19T11:35:59.165199Z","shell.execute_reply.started":"2022-07-19T11:35:59.141293Z","shell.execute_reply":"2022-07-19T11:35:59.163708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:36:01.219144Z","iopub.execute_input":"2022-07-19T11:36:01.219525Z","iopub.status.idle":"2022-07-19T11:36:01.233687Z","shell.execute_reply.started":"2022-07-19T11:36:01.219494Z","shell.execute_reply":"2022-07-19T11:36:01.232390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is assumed that if upvote is greater than downvote then the label is correct.","metadata":{}},{"cell_type":"markdown","source":"# preprocessing","metadata":{}},{"cell_type":"code","source":"# librosa.load() takes the path and returns a numpy array and the sample rate of the audio.\n\nAUDIO_PATH = '../input/dlsprint/train_files/common_voice_bn_30614352.mp3'\naudio, sr = librosa.load(AUDIO_PATH)\n\nprint('Shape of the audio: ', audio.shape)\nprint('Sample rate of the audio: ', sr)\nipd.display(ipd.Audio(data=audio, rate=sr))\nprint('Audio Label :',train_df[train_df['path'] == 'common_voice_bn_30614352.mp3'].sentence)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:37:31.561434Z","iopub.execute_input":"2022-07-19T11:37:31.562519Z","iopub.status.idle":"2022-07-19T11:37:34.300138Z","shell.execute_reply.started":"2022-07-19T11:37:31.562478Z","shell.execute_reply":"2022-07-19T11:37:34.299240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## All the audio files are in .mp3 format we need to convert them into .wav format for faster processing.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nimport skimage.io\n\nimport time\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\nimport shutil\n\nfrom pydub import AudioSegment\n\nfrom joblib import Parallel, delayed\n\nimport io\nimport soundfile as sf","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:37:43.810939Z","iopub.execute_input":"2022-07-19T11:37:43.812096Z","iopub.status.idle":"2022-07-19T11:37:44.435955Z","shell.execute_reply.started":"2022-07-19T11:37:43.812050Z","shell.execute_reply":"2022-07-19T11:37:44.434805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = \"../input/dlsprint/train_files\"\nTEST_PATH = '../input/dlsprint/test_files'\nVALIDATION_PATH = '../input/dlsprint/validation_files'","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:37:48.091005Z","iopub.execute_input":"2022-07-19T11:37:48.091451Z","iopub.status.idle":"2022-07-19T11:37:48.096213Z","shell.execute_reply.started":"2022-07-19T11:37:48.091412Z","shell.execute_reply":"2022-07-19T11:37:48.095347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data filtering\ndef filter_votes(x):\n    up=x[\"up_votes\"]\n    down=x[\"down_votes\"]\n    if up-down<=0:\n        return None\n    elif up==0:\n        return None\n    else:\n        return up","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:37:52.009904Z","iopub.execute_input":"2022-07-19T11:37:52.010322Z","iopub.status.idle":"2022-07-19T11:37:52.016521Z","shell.execute_reply.started":"2022-07-19T11:37:52.010285Z","shell.execute_reply":"2022-07-19T11:37:52.015392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total Data before filtering:\",len(train_df))\ntrain_df[\"up_votes\"]=train_df.progress_apply(lambda x:filter_votes(x),axis=1)\ntrain_df.dropna(subset = ['up_votes'],inplace=True)\nprint(\"Total Data after filtering:\",len(train_df))\ntrain_audio_files=train_df[\"path\"].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:37:53.214836Z","iopub.execute_input":"2022-07-19T11:37:53.215256Z","iopub.status.idle":"2022-07-19T11:37:57.968740Z","shell.execute_reply.started":"2022-07-19T11:37:53.215220Z","shell.execute_reply":"2022-07-19T11:37:57.967572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(['client_id','up_votes','down_votes'],\n                         axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:38:24.930770Z","iopub.execute_input":"2022-07-19T11:38:24.931592Z","iopub.status.idle":"2022-07-19T11:38:24.935717Z","shell.execute_reply.started":"2022-07-19T11:38:24.931554Z","shell.execute_reply":"2022-07-19T11:38:24.934737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_df.sentence)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T10:42:36.587363Z","iopub.execute_input":"2022-07-19T10:42:36.588156Z","iopub.status.idle":"2022-07-19T10:42:36.604846Z","shell.execute_reply.started":"2022-07-19T10:42:36.588099Z","shell.execute_reply":"2022-07-19T10:42:36.603574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have to remove the special characters such as ,.?!;:।","metadata":{}},{"cell_type":"code","source":"train_df.columns = train_df.columns.str.replace('[^a-zA-Z0-9]','')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:38:35.569841Z","iopub.execute_input":"2022-07-19T11:38:35.570777Z","iopub.status.idle":"2022-07-19T11:38:35.576748Z","shell.execute_reply.started":"2022-07-19T11:38:35.570730Z","shell.execute_reply":"2022-07-19T11:38:35.575478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sentence","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:38:37.779246Z","iopub.execute_input":"2022-07-19T11:38:37.779986Z","iopub.status.idle":"2022-07-19T11:38:37.789726Z","shell.execute_reply.started":"2022-07-19T11:38:37.779949Z","shell.execute_reply":"2022-07-19T11:38:37.788542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vocab_list = list(set(vocab_train[\"vocab\"][0]) | set(vocab_test[\"vocab\"][0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_all_chars(batch):\n  all_text = \" \".join(batch[\"sentence\"])\n  vocab = list(set(all_text))\n  return {\"vocab\": [vocab], \"all_text\": [all_text]}","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:38:47.220391Z","iopub.execute_input":"2022-07-19T11:38:47.221124Z","iopub.status.idle":"2022-07-19T11:38:47.227663Z","shell.execute_reply.started":"2022-07-19T11:38:47.221088Z","shell.execute_reply":"2022-07-19T11:38:47.226409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_train = extract_all_chars(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:42:19.822120Z","iopub.execute_input":"2022-07-19T11:42:19.822583Z","iopub.status.idle":"2022-07-19T11:42:20.004759Z","shell.execute_reply.started":"2022-07-19T11:42:19.822547Z","shell.execute_reply":"2022-07-19T11:42:20.003877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_test = extract_all_chars(val_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:46:24.810108Z","iopub.execute_input":"2022-07-19T11:46:24.810552Z","iopub.status.idle":"2022-07-19T11:46:24.854755Z","shell.execute_reply.started":"2022-07-19T11:46:24.810519Z","shell.execute_reply":"2022-07-19T11:46:24.853337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_list = list(set(vocab_train[\"vocab\"][0]) | set(vocab_test[\"vocab\"][0]))","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:46:29.859756Z","iopub.execute_input":"2022-07-19T11:46:29.860344Z","iopub.status.idle":"2022-07-19T11:46:29.866552Z","shell.execute_reply.started":"2022-07-19T11:46:29.860303Z","shell.execute_reply":"2022-07-19T11:46:29.865062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_list","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:46:42.330550Z","iopub.execute_input":"2022-07-19T11:46:42.331346Z","iopub.status.idle":"2022-07-19T11:46:42.341009Z","shell.execute_reply.started":"2022-07-19T11:46:42.331308Z","shell.execute_reply":"2022-07-19T11:46:42.340072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_remove = ['!','/','॥','।','.',';','‘','…','‚',':','-','\"','—']","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:03:29.060988Z","iopub.execute_input":"2022-07-19T12:03:29.061402Z","iopub.status.idle":"2022-07-19T12:03:29.067534Z","shell.execute_reply.started":"2022-07-19T12:03:29.061369Z","shell.execute_reply":"2022-07-19T12:03:29.066291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in to_remove:\n    vocab_list.remove(x)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:03:56.510158Z","iopub.execute_input":"2022-07-19T12:03:56.510557Z","iopub.status.idle":"2022-07-19T12:03:56.515578Z","shell.execute_reply.started":"2022-07-19T12:03:56.510524Z","shell.execute_reply":"2022-07-19T12:03:56.514425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_list","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:04:03.881861Z","iopub.execute_input":"2022-07-19T12:04:03.882643Z","iopub.status.idle":"2022-07-19T12:04:03.890855Z","shell.execute_reply.started":"2022-07-19T12:04:03.882596Z","shell.execute_reply":"2022-07-19T12:04:03.890085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict = {v: k for k, v in enumerate(vocab_list)}\nvocab_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:04:22.441008Z","iopub.execute_input":"2022-07-19T12:04:22.442286Z","iopub.status.idle":"2022-07-19T12:04:22.451692Z","shell.execute_reply.started":"2022-07-19T12:04:22.442219Z","shell.execute_reply":"2022-07-19T12:04:22.450550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict[\"[UNK]\"] = len(vocab_dict)\nvocab_dict[\"[PAD]\"] = len(vocab_dict)\nlen(vocab_dict)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:06:04.521520Z","iopub.execute_input":"2022-07-19T12:06:04.521914Z","iopub.status.idle":"2022-07-19T12:06:04.529054Z","shell.execute_reply.started":"2022-07-19T12:06:04.521881Z","shell.execute_reply":"2022-07-19T12:06:04.527956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n# with open('vocab.json', 'w') as vocab_file:\n#     json.dump(vocab_dict, vocab_file)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:06:47.720839Z","iopub.execute_input":"2022-07-19T12:06:47.721910Z","iopub.status.idle":"2022-07-19T12:06:47.727518Z","shell.execute_reply.started":"2022-07-19T12:06:47.721871Z","shell.execute_reply":"2022-07-19T12:06:47.726101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In a final step, we use the json file to instantiate an object of the Wav2Vec2CTCTokenizer class.","metadata":{}},{"cell_type":"code","source":"# from transformers import Wav2Vec2CTCTokenizer\n\n# tokenizer = Wav2Vec2CTCTokenizer(\"./vocab.json\", unk_token=\"[UNK]\", pad_token=\"[PAD]\", word_delimiter_token=\"|\")","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:05.150680Z","iopub.execute_input":"2022-07-19T12:07:05.151057Z","iopub.status.idle":"2022-07-19T12:07:05.729893Z","shell.execute_reply.started":"2022-07-19T12:07:05.151017Z","shell.execute_reply":"2022-07-19T12:07:05.728845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('new_train.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a list of trian files","metadata":{}},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-07-19T08:20:27.065064Z","iopub.execute_input":"2022-07-19T08:20:27.065469Z","iopub.status.idle":"2022-07-19T08:20:27.080804Z","shell.execute_reply.started":"2022-07-19T08:20:27.065439Z","shell.execute_reply":"2022-07-19T08:20:27.079850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df","metadata":{"execution":{"iopub.status.busy":"2022-07-19T08:21:02.528423Z","iopub.execute_input":"2022-07-19T08:21:02.528825Z","iopub.status.idle":"2022-07-19T08:21:02.546832Z","shell.execute_reply.started":"2022-07-19T08:21:02.528795Z","shell.execute_reply":"2022-07-19T08:21:02.545652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list to store files\ntest_audio_files = []\n\n# Iterate directory\nfor path in os.listdir(TEST_PATH):\n    # check if current path is a file\n    if os.path.isfile(os.path.join(TEST_PATH, path)):\n        test_audio_files.append(path)\nlen(test_audio_files)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:29:31.129130Z","iopub.execute_input":"2022-07-16T16:29:31.129603Z","iopub.status.idle":"2022-07-16T16:29:42.393955Z","shell.execute_reply.started":"2022-07-16T16:29:31.129567Z","shell.execute_reply":"2022-07-16T16:29:42.392856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a list of validation files","metadata":{}},{"cell_type":"code","source":"# list to store files\nval_audio_files = []\n\n# Iterate directory\nfor path in os.listdir(VALIDATION_PATH):\n    # check if current path is a file\n    if os.path.isfile(os.path.join(VALIDATION_PATH, path)):\n        val_audio_files .append(path)\nlen(val_audio_files)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:30:32.444410Z","iopub.execute_input":"2022-07-16T16:30:32.444902Z","iopub.status.idle":"2022-07-16T16:30:43.773093Z","shell.execute_reply.started":"2022-07-16T16:30:32.444870Z","shell.execute_reply":"2022-07-16T16:30:43.771841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nnew_dir = ['new_train','new_validation','new_test']\nfor folder in new_dir:\n    if not os.path.exists(folder):\n        os.makedirs(folder)\n        \n        \"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:30:43.775797Z","iopub.execute_input":"2022-07-16T16:30:43.776907Z","iopub.status.idle":"2022-07-16T16:30:43.784015Z","shell.execute_reply.started":"2022-07-16T16:30:43.776855Z","shell.execute_reply":"2022-07-16T16:30:43.782968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"New_Train_Dir = './new_train'\nNEW_Validation_Dir = './new_validation'\nNew_Test_Dir = './new_test'","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:33:24.083264Z","iopub.execute_input":"2022-07-16T16:33:24.084711Z","iopub.status.idle":"2022-07-16T16:33:24.091347Z","shell.execute_reply.started":"2022-07-16T16:33:24.084652Z","shell.execute_reply":"2022-07-16T16:33:24.090007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting the train .mp3 files into .wav and saving it inside a new directory","metadata":{}},{"cell_type":"code","source":"def save_fn(filename):\n    \n    \n    path = f\"{TRAIN_PATH}/{filename}\"\n    save_path = f\"{New_Train_Dir}\"\n    if not os.path.exists(save_path):\n        os.makedirs(save_path, exist_ok=True)\n    \n    if os.path.exists(path):\n        try:\n            sound = AudioSegment.from_mp3(path)\n            sound = sound.set_frame_rate(16000)\n            sound.export(f\"{save_path}/{filename[:-4]}.wav\", format=\"wav\")\n        except:\n            print(path)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:19:57.472173Z","iopub.execute_input":"2022-07-16T07:19:57.472530Z","iopub.status.idle":"2022-07-16T07:19:57.480224Z","shell.execute_reply.started":"2022-07-16T07:19:57.472502Z","shell.execute_reply":"2022-07-16T07:19:57.479175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart = time.time()\n\nParallel(n_jobs=8, backend=\"multiprocessing\")(\n    delayed(save_fn)(filename) for filename in tqdm(train_audio_files)\n)\n\nend = time.time()\nprint(\"total time to process: {x} seconds\".format(x=end-start))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T07:20:01.002541Z","iopub.execute_input":"2022-07-16T07:20:01.002994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting the validation .mp3 files into .wav and saving it inside a new directory","metadata":{}},{"cell_type":"code","source":"def val_save_fn(filename):\n    \n    \n    path = f\"{VALIDATION_PATH}/{filename}\"\n    save_path = f\"{NEW_Validation_Dir}\"\n    if not os.path.exists(save_path):\n        os.makedirs(save_path, exist_ok=True)\n    \n    if os.path.exists(path):\n        try:\n            sound = AudioSegment.from_mp3(path)\n            sound = sound.set_frame_rate(16000)\n            sound.export(f\"{save_path}/{filename[:-4]}.wav\", format=\"wav\")\n        except:\n            print(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\n\nParallel(n_jobs=8, backend=\"multiprocessing\")(\n    delayed(val_save_fn)(filename) for filename in tqdm(val_audio_files)\n)\n\nend = time.time()\nprint(\"total time to process: {x} seconds\".format(x=end-start))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting the test .mp3 files into .wav and saving it inside a new directory","metadata":{}},{"cell_type":"code","source":"def test_save_fn(filename):\n    \n    \n    path = f\"{TEST_PATH}/{filename}\"\n    save_path = f\"{New_Test_Dir}\"\n    if not os.path.exists(save_path):\n        os.makedirs(save_path, exist_ok=True)\n    \n    if os.path.exists(path):\n        try:\n            sound = AudioSegment.from_mp3(path)\n            sound = sound.set_frame_rate(16000)\n            sound.export(f\"{save_path}/{filename[:-4]}.wav\", format=\"wav\")\n        except:\n            print(path)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:33:03.803783Z","iopub.execute_input":"2022-07-16T16:33:03.804147Z","iopub.status.idle":"2022-07-16T16:33:03.811388Z","shell.execute_reply.started":"2022-07-16T16:33:03.804120Z","shell.execute_reply":"2022-07-16T16:33:03.810391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\n\nParallel(n_jobs=8, backend=\"multiprocessing\")(\n    delayed(test_save_fn)(filename) for filename in tqdm(test_audio_files)\n)\n\nend = time.time()\nprint(\"total time to process: {x} seconds\".format(x=end-start))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:33:30.044574Z","iopub.execute_input":"2022-07-16T16:33:30.044987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}