{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":73047,"databundleVersionId":8149390,"sourceType":"competition"},{"sourceId":6707460,"sourceType":"datasetVersion","datasetId":3865741}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1> EDA </h1>","metadata":{}},{"cell_type":"markdown","source":"<h3>\nDataset Information:\n\n* The recordings are 16kHz *.wav audio files\n* The data are coming from 373 different speakers from 10 regions in Bangladesh\n* 10 regions : rangpur, kishoreganj, narail, chittagong, narsingdi, tangail, habiganj, barishal, sylhet, sandwip\n    </h3>","metadata":{}},{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install noisereduce","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:12.358898Z","iopub.execute_input":"2024-04-24T23:21:12.359797Z","iopub.status.idle":"2024-04-24T23:21:26.563895Z","shell.execute_reply.started":"2024-04-24T23:21:12.359760Z","shell.execute_reply":"2024-04-24T23:21:26.562647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport nltk\nimport re\nfrom nltk.corpus import words\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nfrom pydub import AudioSegment\nimport IPython.display as ipd\nfrom collections import Counter\nimport os\nimport librosa\nimport time\nfrom multiprocessing import Pool\nimport seaborn as sns\nimport random\nfrom tqdm import tqdm\nimport librosa.display\nimport noisereduce as nr","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:26.566012Z","iopub.execute_input":"2024-04-24T23:21:26.566339Z","iopub.status.idle":"2024-04-24T23:21:35.873361Z","shell.execute_reply.started":"2024-04-24T23:21:26.566310Z","shell.execute_reply":"2024-04-24T23:21:35.872455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Test Size","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/ben10/ben10/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/ben10/sample_submission.csv\")\nprint(\"Train dataframe : \")\ndisplay(df.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:35.874918Z","iopub.execute_input":"2024-04-24T23:21:35.875510Z","iopub.status.idle":"2024-04-24T23:21:36.130122Z","shell.execute_reply.started":"2024-04-24T23:21:35.875466Z","shell.execute_reply":"2024-04-24T23:21:36.129068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test dataframe : \")\ndisplay(test.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:36.132523Z","iopub.execute_input":"2024-04-24T23:21:36.132888Z","iopub.status.idle":"2024-04-24T23:21:36.143077Z","shell.execute_reply.started":"2024-04-24T23:21:36.132857Z","shell.execute_reply":"2024-04-24T23:21:36.142054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total samples in Train set: \",df.shape[0])\nprint(\"Total samples in test set: \",len(os.listdir(\"/kaggle/input/ben10/ben10/16_kHz_valid_audio\")))","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:36.144276Z","iopub.execute_input":"2024-04-24T23:21:36.144572Z","iopub.status.idle":"2024-04-24T23:21:36.384879Z","shell.execute_reply.started":"2024-04-24T23:21:36.144524Z","shell.execute_reply":"2024-04-24T23:21:36.383839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seconds_to_hours(seconds):\n    hours = seconds / 3600\n    return hours","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:21:36.385968Z","iopub.execute_input":"2024-04-24T23:21:36.386342Z","iopub.status.idle":"2024-04-24T23:21:36.391243Z","shell.execute_reply.started":"2024-04-24T23:21:36.386308Z","shell.execute_reply":"2024-04-24T23:21:36.390300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport librosa\n\ndef get_total_audio_duration(folder_path):\n    total_duration = 0\n    for filename in os.listdir(folder_path):\n        if filename.endswith(('.wav')):\n            file_path = os.path.join(folder_path, filename)\n            try:\n                audio_duration = librosa.get_duration(path=file_path)\n                total_duration += audio_duration\n            except Exception as e:\n                print(f\"Error processing {filename}: {e}\")\n    return total_duration\n\nfolder_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\ntotal_duration = get_total_audio_duration(folder_path)\nhours = seconds_to_hours(total_duration)\nprint(f\"Total duration of Train audio files: {total_duration} seconds which is {hours} Hours\")\n\nTest_dataset = \"/kaggle/input/ben10/ben10/16_kHz_valid_audio\"\ntest_total_duration = get_total_audio_duration(Test_dataset)\ntest_hours = seconds_to_hours(test_total_duration)\nprint(f\"Total duration of Test audio files: {total_duration} seconds which is {test_hours} Hours\")","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:29.900029Z","iopub.execute_input":"2024-04-24T23:22:29.901012Z","iopub.status.idle":"2024-04-24T23:23:29.001268Z","shell.execute_reply.started":"2024-04-24T23:22:29.900977Z","shell.execute_reply":"2024-04-24T23:23:29.000176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Dataset Duration: ~ 63 Hours 8 Minutes 42 Seconds\n### Test Dataset Duration: ~ 8 Hours 1 Minute 11 Seconds","metadata":{}},{"cell_type":"code","source":"train_dir = \"/kaggle/input/ben10/ben10/16_kHz_train_audio/\"\npaths = [train_dir+path for path in os.listdir(train_dir)]\n#print(\"First two path : \",paths[:2])\n\ndef get_duration(file):\n    try:\n        # Load audio file\n        y, sr = librosa.load(file, sr=None)\n        # Calculate duration\n        duration = librosa.get_duration(y=y, sr=sr)\n        return duration\n    except Exception as e:\n        return file, None\n\ndef get_durations_parallel(files):\n    with Pool() as pool:\n        results = pool.map(get_duration, files)\n    return results\n\n\n#Getting durations here. ALso checking how much time does it take to laod all the audios\nstart = time.time()\ndurations_train = get_durations_parallel(paths)\nprint(\"Total time taken in seconds : \",time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:23:29.003568Z","iopub.execute_input":"2024-04-24T23:23:29.003976Z","iopub.status.idle":"2024-04-24T23:24:19.285572Z","shell.execute_reply.started":"2024-04-24T23:23:29.003941Z","shell.execute_reply":"2024-04-24T23:24:19.284185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(durations_train,bins = [i for i in range(0,31,3)])\nplt.xticks(np.arange(0, 35, step=3))\nplt.xlabel('Duration (s)')\nplt.ylabel('Frequency')\nplt.title('Train audio file durations')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:19.287396Z","iopub.execute_input":"2024-04-24T23:24:19.288464Z","iopub.status.idle":"2024-04-24T23:24:19.699063Z","shell.execute_reply.started":"2024-04-24T23:24:19.288404Z","shell.execute_reply":"2024-04-24T23:24:19.698161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dir = \"/kaggle/input/ben10/ben10/16_kHz_valid_audio/\"\npaths = [valid_dir+path for path in os.listdir(valid_dir)]\n#print(\"First two path : \",paths[:2])\n\ndef get_duration(file):\n    try:\n        # Load audio file\n        y, sr = librosa.load(file, sr=None)\n        # Calculate duration\n        duration = librosa.get_duration(y=y, sr=sr)\n        return duration\n    except Exception as e:\n        return file, None\n\ndef get_durations_parallel(files):\n    with Pool() as pool:\n        results = pool.map(get_duration, files)\n    return results\n\n\n#Getting durations here. ALso checking how much time does it take to laod all the audios\nstart = time.time()\ndurations_valid = get_durations_parallel(paths)\nprint(\"Total time taken in seconds : \",time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:19.701911Z","iopub.execute_input":"2024-04-24T23:24:19.702599Z","iopub.status.idle":"2024-04-24T23:24:25.658517Z","shell.execute_reply.started":"2024-04-24T23:24:19.702570Z","shell.execute_reply":"2024-04-24T23:24:25.657216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(durations_valid,bins = [i for i in range(0,31,3)])\nplt.xticks(np.arange(0, 35, step=3))\nplt.xlabel('Duration (s)')\nplt.ylabel('Frequency')\nplt.title('Valid audio file durations')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:25.660127Z","iopub.execute_input":"2024-04-24T23:24:25.660483Z","iopub.status.idle":"2024-04-24T23:24:25.986190Z","shell.execute_reply.started":"2024-04-24T23:24:25.660454Z","shell.execute_reply":"2024-04-24T23:24:25.985060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Region wise samples","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef extract_regions(path):\n    unwanted_strs = [\"train_\", \"valid_\", \".wav\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"0\", \"(\", \")\", \" \"]\n    for i in unwanted_strs:\n        path = path.replace(i, \"\")\n    return path\n\ndf[\"region\"] = df[\"file_name\"].apply(lambda x: extract_regions(x))\ntest[\"region\"] = test[\"id\"].apply(lambda x: extract_regions(x))\n\n# Counting unique regions and their occurrences\ndf_region_counts = df['region'].value_counts().sort_values()\ntest_region_counts = test['region'].value_counts().sort_values()\n\n# Creating a color map for each region\ncolors = plt.cm.tab20.colors[:len(df_region_counts)]\n\n# Plotting the distributions\nfig, axes = plt.subplots(1, 2, figsize=(12, 6))\n\ndf_region_counts.plot(kind='barh', ax=axes[0], color=colors)\naxes[0].set_title('Sample Distribution of Regions in Train set')\naxes[0].set_xlabel('Count')\n\ntest_region_counts.plot(kind='barh', ax=axes[1], color=colors)\naxes[1].set_title('Sample Distribution of Regions in Test Set')\naxes[1].set_xlabel('Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:25.987284Z","iopub.execute_input":"2024-04-24T23:24:25.987583Z","iopub.status.idle":"2024-04-24T23:24:26.717828Z","shell.execute_reply.started":"2024-04-24T23:24:25.987540Z","shell.execute_reply":"2024-04-24T23:24:26.716883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### The distribution of samples in the training set resembles that of the test set.\n##### Sylhet boasts the highest number of samples, while Barishal shows the fewest.\n","metadata":{}},{"cell_type":"markdown","source":"### Total Number of samples by Region","metadata":{}},{"cell_type":"code","source":"region_counts = df.groupby('region').size()\nprint('Train Sample Count by',region_counts)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:26.719198Z","iopub.execute_input":"2024-04-24T23:24:26.719656Z","iopub.status.idle":"2024-04-24T23:24:26.732443Z","shell.execute_reply.started":"2024-04-24T23:24:26.719618Z","shell.execute_reply":"2024-04-24T23:24:26.731410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_region_counts = test.groupby('region').size()\nprint('Test Sample Count by',test_region_counts)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:26.733866Z","iopub.execute_input":"2024-04-24T23:24:26.734288Z","iopub.status.idle":"2024-04-24T23:24:26.742015Z","shell.execute_reply.started":"2024-04-24T23:24:26.734255Z","shell.execute_reply":"2024-04-24T23:24:26.741098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Listening to Region Wise Audio","metadata":{}},{"cell_type":"code","source":"regions = ['rangpur', 'kishoreganj', 'narail', 'chittagong', 'narsingdi', 'tangail', 'habiganj', 'barishal', 'sylhet', 'sandwip']","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:26.743390Z","iopub.execute_input":"2024-04-24T23:24:26.743746Z","iopub.status.idle":"2024-04-24T23:24:26.751338Z","shell.execute_reply.started":"2024-04-24T23:24:26.743721Z","shell.execute_reply":"2024-04-24T23:24:26.750581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\nfor idx in df.sample(10).index:\n    \n    file_path = os.path.join(root_path, df['file_name'].iloc[idx])\n    text = df['transcripts'].iloc[idx]\n    region = df['region'].iloc[idx]\n    print(f'Region: {str.capitalize(region)}')\n    display(AudioSegment.from_file(file_path))\n    print(f\"Original transcription : {text}\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:26.754885Z","iopub.execute_input":"2024-04-24T23:24:26.755161Z","iopub.status.idle":"2024-04-24T23:24:28.420905Z","shell.execute_reply.started":"2024-04-24T23:24:26.755138Z","shell.execute_reply":"2024-04-24T23:24:28.419690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\ndef plot_sound_characteristics(region_df):\n    for idx, row in region_df.iterrows():\n        file_path = os.path.join(root_path, row['file_name'])\n        text = row['transcripts']\n        region = row['region']\n        \n        print(f'Region: {region.capitalize()}')\n        audio = AudioSegment.from_file(file_path)\n        \n        samples, sample_rate = librosa.load(file_path, sr=None)\n        \n        time = np.arange(0, len(samples)) / sample_rate\n        \n        plt.figure(figsize=(12, 6))\n        plt.subplot(2, 1, 1)\n        plt.plot(time, samples)\n        plt.title('Waveform (Amplitude)')\n        plt.xlabel('Time (s)')\n        plt.ylabel('Amplitude')\n        \n        plt.subplot(2, 1, 2)\n        n_fft = 2048  \n        freqs = np.fft.rfftfreq(n_fft, d=1/sample_rate)\n        spectrum = np.abs(np.fft.rfft(samples, n=n_fft))\n        plt.plot(freqs, spectrum)\n        plt.title('Frequency Spectrum')\n        plt.xlabel('Frequency (Hz)')\n        plt.ylabel('Magnitude')\n        plt.xlim(0, 5000) \n        \n        plt.tight_layout()\n        plt.show()\n        \n        print(f\"Original transcription: {text}\\n\\n\")\n\n\ngrouped_by_region = df.groupby('region')\n\n\nfor region, region_df in grouped_by_region:\n    plot_sound_characteristics(region_df.sample(1))  \n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:28.422145Z","iopub.execute_input":"2024-04-24T23:24:28.422438Z","iopub.status.idle":"2024-04-24T23:24:35.290713Z","shell.execute_reply.started":"2024-04-24T23:24:28.422413Z","shell.execute_reply":"2024-04-24T23:24:35.289774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vocabulary List","metadata":{}},{"cell_type":"code","source":"chars = {}\nfor sen in tqdm(df.transcripts):\n    for j in sen:\n        try:\n            chars[j] += 1\n        except:\n            chars[j] = 1","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:35.291931Z","iopub.execute_input":"2024-04-24T23:24:35.292219Z","iopub.status.idle":"2024-04-24T23:24:36.111442Z","shell.execute_reply.started":"2024-04-24T23:24:35.292194Z","shell.execute_reply":"2024-04-24T23:24:36.110528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(chars.items())","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.112816Z","iopub.execute_input":"2024-04-24T23:24:36.113431Z","iopub.status.idle":"2024-04-24T23:24:36.126035Z","shell.execute_reply.started":"2024-04-24T23:24:36.113397Z","shell.execute_reply":"2024-04-24T23:24:36.125112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(chars.items())","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.127357Z","iopub.execute_input":"2024-04-24T23:24:36.127749Z","iopub.status.idle":"2024-04-24T23:24:36.138845Z","shell.execute_reply.started":"2024-04-24T23:24:36.127716Z","shell.execute_reply":"2024-04-24T23:24:36.137906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h4> Total 112 unique vocabs including the wrong ones. We have to clean this dataset.","metadata":{}},{"cell_type":"markdown","source":"# Train Sentence Length Distribution","metadata":{}},{"cell_type":"code","source":"df['sen_len'] = df['transcripts'].apply(lambda x: len(x.split()))\n\nplt.figure(figsize=(8, 6))\nsns.histplot(data = df, x = 'sen_len' ,bins=[i for i in range(0,151,10)])\n# sns.histplot(data = df, x = 'sen_len', bins=10)\nplt.xticks(np.arange(0, 150, step=10))\nplt.xlabel('Sentence Length')\nplt.title('Train sentence length distributions')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.140125Z","iopub.execute_input":"2024-04-24T23:24:36.140443Z","iopub.status.idle":"2024-04-24T23:24:36.525028Z","shell.execute_reply.started":"2024-04-24T23:24:36.140420Z","shell.execute_reply":"2024-04-24T23:24:36.523025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['sen_len']==0] #sentence length 0","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.527067Z","iopub.execute_input":"2024-04-24T23:24:36.527483Z","iopub.status.idle":"2024-04-24T23:24:36.538865Z","shell.execute_reply.started":"2024-04-24T23:24:36.527441Z","shell.execute_reply":"2024-04-24T23:24:36.537904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['sen_len']<5] #sentence length less than 5","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.540133Z","iopub.execute_input":"2024-04-24T23:24:36.540427Z","iopub.status.idle":"2024-04-24T23:24:36.557689Z","shell.execute_reply.started":"2024-04-24T23:24:36.540402Z","shell.execute_reply":"2024-04-24T23:24:36.556447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 222 samples has sentence length less than 5","metadata":{}},{"cell_type":"code","source":"sample = df[df['sen_len']>60]\nsample","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.558944Z","iopub.execute_input":"2024-04-24T23:24:36.559669Z","iopub.status.idle":"2024-04-24T23:24:36.573500Z","shell.execute_reply.started":"2024-04-24T23:24:36.559641Z","shell.execute_reply":"2024-04-24T23:24:36.572595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 856 samples have sentence length greater than 60","metadata":{}},{"cell_type":"code","source":"idx = [random.randint(0,len(sample)) for _ in range(5)]\nfor i in sample.transcripts.iloc[idx].tolist():\n    print(\"Sample sentence : \",i,\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:36.574680Z","iopub.execute_input":"2024-04-24T23:24:36.575007Z","iopub.status.idle":"2024-04-24T23:24:36.581694Z","shell.execute_reply.started":"2024-04-24T23:24:36.574983Z","shell.execute_reply":"2024-04-24T23:24:36.580773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word Frequency","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom nltk.tokenize import word_tokenize\n\n# Assuming you have imported your DataFrame as 'df'\n\n# Combine all transcripts into a single string\ntranscripts_combined = ' '.join(df['transcripts'])\n\n# Tokenize the combined transcript into words\nwords = word_tokenize(transcripts_combined)\n\n# Count the frequency of each word\nword_freq = {}\nfor word in words:\n    if word in word_freq:\n        word_freq[word] += 1\n    else:\n        word_freq[word] = 1\n\n# Convert the dictionary into a DataFrame for easier manipulation if needed\nword_freq_df = pd.DataFrame(list(word_freq.items()), columns=['Word', 'Frequency'])\n\n# Sort the DataFrame by frequency in descending order\nword_freq_df = word_freq_df.sort_values(by='Frequency', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:24:52.164180Z","iopub.execute_input":"2024-04-24T23:24:52.164570Z","iopub.status.idle":"2024-04-24T23:24:58.852162Z","shell.execute_reply.started":"2024-04-24T23:24:52.164529Z","shell.execute_reply":"2024-04-24T23:24:58.851335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_freq = pd.DataFrame(word_freq_df, columns=['Word', 'Frequency'])\n\n# Print the full DataFrame\nprint(word_freq)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:03.509661Z","iopub.execute_input":"2024-04-24T23:25:03.510020Z","iopub.status.idle":"2024-04-24T23:25:03.519540Z","shell.execute_reply.started":"2024-04-24T23:25:03.509996Z","shell.execute_reply":"2024-04-24T23:25:03.518419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Are Sentences Repeated?","metadata":{}},{"cell_type":"code","source":"print(\"Total sentences :\",len(df))\nprint(\"Total unique sentences : \",df.transcripts.nunique())\nprint(\"Percentage of unique sentences ; \",df.transcripts.nunique()/len(df))","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:05.933340Z","iopub.execute_input":"2024-04-24T23:25:05.934022Z","iopub.status.idle":"2024-04-24T23:25:05.991090Z","shell.execute_reply.started":"2024-04-24T23:25:05.933988Z","shell.execute_reply":"2024-04-24T23:25:05.990089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Most frequent Samples in the Train Data \\n\")\ndf.transcripts.value_counts()[:10]","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:13.544336Z","iopub.execute_input":"2024-04-24T23:25:13.544757Z","iopub.status.idle":"2024-04-24T23:25:13.569267Z","shell.execute_reply.started":"2024-04-24T23:25:13.544725Z","shell.execute_reply":"2024-04-24T23:25:13.567920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h4>Though there irrelevant samples which are completely blank, we can see that there are some repeated sentences as well</h4>","metadata":{}},{"cell_type":"markdown","source":"# MFCC","metadata":{}},{"cell_type":"markdown","source":"#### MFCC is a feature extraction technique widely used in speech and audio processing. MFCCs are used to represent the spectral characteristics of sound in a way that is well-suited for various machine learning tasks, such as speech recognition and music analysis.In simpler terms, MFCCs are a set of coefficients that capture the shape of the power spectrum of a sound signal.\n\n#### MFCCs are particularly useful because they emphasize features of the audio signal that are important for human speech perception while discarding less relevant information. This makes them effective for tasks like speaker recognition, emotion detection, and speech-to-text conversion.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport librosa\nimport matplotlib.pyplot as plt\nfrom pydub import AudioSegment\n\n# Set root path to the directory containing audio files\nroot_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\n# Define a function to compute MFCC for each region\ndef compute_mfcc_for_region(region_df):\n    for idx, row in region_df.iterrows():\n        file_path = os.path.join(root_path, row['file_name'])\n        text = row['transcripts']\n        region = row['region']\n        \n        print(f'Region: {region.capitalize()}')\n        audio = AudioSegment.from_file(file_path)\n        \n        # Load audio file using librosa\n        samples, sample_rate = librosa.load(file_path, sr=None)\n        \n        # Compute MFCC\n        mfcc = librosa.feature.mfcc(y=samples, sr=sample_rate, n_mfcc=13)\n        \n        # Display MFCC plot\n        plt.figure(figsize=(10, 4))\n        librosa.display.specshow(mfcc, x_axis='time')\n        plt.colorbar()\n        plt.title(f'MFCC for {row[\"file_name\"]}')\n        plt.tight_layout()\n        plt.show()\n        \n        print(f\"Original transcription: {text}\\n\\n\")\n\n# Assuming df is your DataFrame containing sample information\n# Assuming 'region' column exists in df\n\n# Group DataFrame by 'region'\ngrouped_by_region = df.groupby('region')\n\n# Iterate over each region and compute MFCC\nfor region, region_df in grouped_by_region:\n    compute_mfcc_for_region(region_df.sample(1))  # Sample 10 entries from each region for demonstration\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:15.944867Z","iopub.execute_input":"2024-04-24T23:25:15.945697Z","iopub.status.idle":"2024-04-24T23:25:20.953874Z","shell.execute_reply.started":"2024-04-24T23:25:15.945662Z","shell.execute_reply":"2024-04-24T23:25:20.952813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Denoiser","metadata":{}},{"cell_type":"markdown","source":"#### Sometimes our models were transcribing even the slighest words uttered by people. So to secure a good accuracy score we had to denoise the audio.","metadata":{}},{"cell_type":"code","source":"# Set root path to the directory containing audio files\nroot_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\n# Define a function to denoise and plot audio\ndef denoise_and_plot(region_df):\n    for idx, row in region_df.iterrows():\n        file_path = os.path.join(root_path, row['file_name'])\n        text = row['transcripts']\n        region = row['region']\n        \n        print(f'Region: {region.capitalize()}')\n        \n        # Load audio file using librosa\n        samples, sample_rate = librosa.load(file_path, sr=None)\n        \n        # Perform noise reduction using spectral subtraction\n        reduced_noise = nr.reduce_noise(y=samples, sr=sample_rate)\n        \n        # Compute time array for waveform plot\n        time = np.arange(0, len(samples)) / sample_rate\n        \n        # Plot original and denoised waveform\n        plt.figure(figsize=(12, 6))\n        plt.subplot(2, 1, 1)\n        plt.plot(time, samples, label='Original')\n        plt.title('Original Waveform')\n        plt.xlabel('Time (s)')\n        plt.ylabel('Amplitude')\n        plt.legend()\n        \n        plt.subplot(2, 1, 2)\n        plt.plot(time, reduced_noise, label='Denoised', color='orange')\n        plt.title('Denoised Waveform')\n        plt.xlabel('Time (s)')\n        plt.ylabel('Amplitude')\n        plt.legend()\n        \n        plt.tight_layout()\n        plt.show()\n        \n        print(f\"Original transcription: {text}\\n\\n\")\n\n# Assuming df is your DataFrame containing sample information\n# Assuming 'region' column exists in df\n\n# Group DataFrame by 'region'\ngrouped_by_region = df.groupby('region')\n\n# Iterate over each region and denoise audio\nfor region, region_df in grouped_by_region:\n    denoise_and_plot(region_df.sample(1))  # Sample 1 entry from each region for demonstration\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:23.350310Z","iopub.execute_input":"2024-04-24T23:25:23.350842Z","iopub.status.idle":"2024-04-24T23:25:37.446622Z","shell.execute_reply.started":"2024-04-24T23:25:23.350811Z","shell.execute_reply":"2024-04-24T23:25:37.445592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove English Words","metadata":{}},{"cell_type":"code","source":"# Set root path to the directory containing audio files\nroot_path = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\n# Define a function to check if a transcript contains English letters\ndef contains_english_letters(transcript):\n    # Regular expression pattern to match English letters\n    pattern = re.compile('[a-zA-Z]')\n    # Search for English letters in the transcript\n    if pattern.search(transcript):\n        return True\n    return False\n\n# Define a function to remove English letters from a transcript\ndef remove_english_letters(transcript):\n    # Regular expression pattern to match English letters\n    pattern = re.compile('[a-zA-Z]')\n    # Replace English letters with an empty string\n    clean_transcript = pattern.sub('', transcript)\n    return clean_transcript\n\n# Iterate over samples and remove English letters from transcripts\nfor idx, row in df.iterrows():\n    file_path = os.path.join(root_path, row['file_name'])\n    text = row['transcripts']\n    region = row['region']\n    \n    if contains_english_letters(text):\n        print(f\"Region: {region.capitalize()}\")\n        print(f\"Original Transcription: {text}\")\n        \n        # Remove English letters from transcript\n        cleaned_text = remove_english_letters(text)\n        print(f\"Cleaned Transcription: {cleaned_text}\")\n        \n        display(AudioSegment.from_file(file_path))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:25:40.590253Z","iopub.execute_input":"2024-04-24T23:25:40.591144Z","iopub.status.idle":"2024-04-24T23:25:42.427129Z","shell.execute_reply.started":"2024-04-24T23:25:40.591110Z","shell.execute_reply":"2024-04-24T23:25:42.426008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.150800Z","iopub.status.idle":"2024-04-24T23:22:27.151160Z","shell.execute_reply.started":"2024-04-24T23:22:27.150980Z","shell.execute_reply":"2024-04-24T23:22:27.150995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAINING STARTS FRO HERE","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nfrom tqdm import tqdm  # Import tqdm\n\n# Path to the directory containing the audio files\naudio_dir = \"/kaggle/input/ben10/ben10/16_kHz_train_audio\"\n\n# Directory to save the separated audio files\nsave_dir = \"dataset_based_on_region/train/\"\n# Seperate train.csv based on region\n# Get the list of audio files\naudio_files = [f for f in os.listdir(audio_dir) if f.endswith(\".wav\")]\n\n# Use tqdm to iterate through each audio file with a progress bar\nfor filename in tqdm(audio_files, desc=\"Copying audio files\"):\n    # Extract the region name from the filename\n    region_name = filename.split(\" (\")[0]  # Split by \" (\" and take the first part\n\n    # Create a directory for the region if it doesn't exist in the save directory\n    region_dir = os.path.join(save_dir, region_name)\n    os.makedirs(region_dir, exist_ok=True)\n\n    # Copy the audio file to the corresponding directory in the save directory\n    source_path = os.path.join(audio_dir, filename)\n    destination_path = os.path.join(region_dir, filename)\n    shutil.copy(source_path, destination_path)  # Copy the file\n\nprint(\"Separation completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.152476Z","iopub.status.idle":"2024-04-24T23:22:27.152874Z","shell.execute_reply.started":"2024-04-24T23:22:27.152704Z","shell.execute_reply":"2024-04-24T23:22:27.152719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seperate train.csv based on region","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport csv\n\n# Read the CSV file\ncsv_file = \"/kaggle/input/ben10/ben10/train.csv\"\ndf = pd.read_csv(csv_file)\n\n# Create a directory to save the separated CSV files\noutput_dir = \"dataset_based_on_region/separated_train_csv/\"\nos.makedirs(output_dir, exist_ok=True)\n\n# Iterate through each row in the DataFrame to write headers and data to separate CSV files\nfor index, row in df.iterrows():\n    # Extract file name and transcripts from the current row\n    file_name = row[\"file_name\"]\n    transcripts = row[\"transcripts\"]\n    \n    # Remove newline characters from transcripts\n    transcripts = transcripts.replace(\"\\n\", \" \")\n    \n    # Enclose transcripts in double quotes and escape any double quotes within the text\n    transcripts = '\"' + transcripts.replace('\"', '\"\"') + '\"'\n    \n    # Extract the region name from the file name\n    region_name = file_name.split(\"_\")[1].split(\" (\")[0]\n    \n    # Create a CSV file for the region if it doesn't exist\n    region_csv_file = os.path.join(output_dir, f\"{region_name}_train.csv\")\n    if not os.path.exists(region_csv_file):\n        with open(region_csv_file, \"w\", newline='', encoding='utf-8') as f:\n            # Initialize CSV writer\n            writer = csv.writer(f, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n            # Write header to the CSV file\n            writer.writerow([\"file_name\", \"transcripts\"])\n    \n    # Append the row data to the CSV file for the region\n    with open(region_csv_file, \"a\", newline='', encoding='utf-8') as f:\n        # Initialize CSV writer\n        writer = csv.writer(f, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n        # Write the row data to the CSV file\n        writer.writerow([file_name, transcripts])\n\nprint(\"Separation completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.154172Z","iopub.status.idle":"2024-04-24T23:22:27.154508Z","shell.execute_reply.started":"2024-04-24T23:22:27.154349Z","shell.execute_reply":"2024-04-24T23:22:27.154362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# You have to run the above code only once.\n#### After that you have to only run the code below","metadata":{}},{"cell_type":"code","source":"import torch\ntorch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.155693Z","iopub.status.idle":"2024-04-24T23:22:27.156016Z","shell.execute_reply.started":"2024-04-24T23:22:27.155860Z","shell.execute_reply":"2024-04-24T23:22:27.155873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading all necessary libraries here","metadata":{}},{"cell_type":"code","source":"import os\n\nimport pandas as pd\n\nimport librosa\nimport librosa.display\n\nimport numpy as np\n\nimport IPython.display as ipd\n\nimport matplotlib.pyplot as plt\n\nimport random\n\nfrom collections import Counter\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torchaudio\n\nfrom dataclasses import dataclass\nfrom typing import Any, Dict, List, Union\nfrom datasets import DatasetDict\nfrom datasets import Dataset as DS\n\nfrom transformers import (\n    WhisperFeatureExtractor,\n    WhisperTokenizer,\n    WhisperProcessor,\n    WhisperForConditionalGeneration,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n    TrainerCallback,\n    TrainingArguments,\n    TrainerState,\n    TrainerControl,\n    EarlyStoppingCallback,\n    pipeline\n)\n\nfrom torchmetrics.text import WordErrorRate, CharErrorRate","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.157454Z","iopub.status.idle":"2024-04-24T23:22:27.157826Z","shell.execute_reply.started":"2024-04-24T23:22:27.157650Z","shell.execute_reply":"2024-04-24T23:22:27.157665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For every region first change the value of train_region_name, then run the code to train seperate model for each of the regions","metadata":{}},{"cell_type":"code","source":"train_region_name = 'sylhet'\nBASE_DIR = '/kaggle/working/dataset_based_on_region'\ntrain_data_dir = f\"{BASE_DIR}/train/train_{train_region_name}/\"\ndata_path = f\"{BASE_DIR}/separated_train_csv/{train_region_name}_train.csv\"\n#data_path = \"dataset/ben10/train.csv\"\nprint(data_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.159103Z","iopub.status.idle":"2024-04-24T23:22:27.159435Z","shell.execute_reply.started":"2024-04-24T23:22:27.159277Z","shell.execute_reply":"2024-04-24T23:22:27.159290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split2path = {\n    \"train\": train_data_dir,\n    \"test\": test_data_dir,\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.160496Z","iopub.status.idle":"2024-04-24T23:22:27.160866Z","shell.execute_reply.started":"2024-04-24T23:22:27.160694Z","shell.execute_reply":"2024-04-24T23:22:27.160709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Here using data=pd.concat([data, data]) to enrich the dataset, By doing this we were getting less WER as this helps the model to place words at correct position.","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(data_path)\nprint(len(data))\ndata=pd.concat([data, data])\nprint(len(data))\ndata.sample(10)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.162025Z","iopub.status.idle":"2024-04-24T23:22:27.162328Z","shell.execute_reply.started":"2024-04-24T23:22:27.162178Z","shell.execute_reply":"2024-04-24T23:22:27.162191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This code defines functions to extract information like split and district from filenames, then it processes a dataset by mapping filenames to their corresponding paths, replacing newline characters in transcripts, and adding columns for split, district, file path, and cleaned transcripts.","metadata":{}},{"cell_type":"code","source":"def extract_split(filename):\n    filename_ = filename.split(\"_\")\n    split = filename_[0]\n    return split\n\ndef extract_district(filename):\n    filename_ = filename.split(\" \")[0]\n    district = filename_.split(\"_\")[1]\n    return district\n\ndef beautify_dataset(data):\n    splits = []\n    districts = []\n    newpaths = []\n    transcripts = []\n    \n    for i in range(len(data)):\n        filename, transcript = data.iloc[i]\n        split = extract_split(filename)\n        district = extract_district(filename)\n        dir_path = split2path[split]\n        composed_path = f\"{dir_path}{filename}\"\n        \n        if os.path.exists(composed_path) == False:\n            print(f\"{composed_path} does not exist.\")\n            continue\n        \n        # replace any newline characters\n        transcript = transcript.replace(\"\\n\", \" \")\n        transcript = \" \".join(transcript.split())\n        \n        splits.append(split)\n        districts.append(district)\n        newpaths.append(composed_path)\n        transcripts.append(transcript)\n    \n    data['file_path'] = newpaths\n    data['district'] = districts\n    data['split'] = splits\n    data['transcripts'] = transcripts\n    \n#     data.drop(columns=['file_name'], inplace=True)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.163689Z","iopub.status.idle":"2024-04-24T23:22:27.164052Z","shell.execute_reply.started":"2024-04-24T23:22:27.163882Z","shell.execute_reply":"2024-04-24T23:22:27.163896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = beautify_dataset(data)\ndata.sample(20)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.165358Z","iopub.status.idle":"2024-04-24T23:22:27.165839Z","shell.execute_reply.started":"2024-04-24T23:22:27.165599Z","shell.execute_reply":"2024-04-24T23:22:27.165619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[data[\"transcripts\"] == \"<>\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.167847Z","iopub.status.idle":"2024-04-24T23:22:27.168327Z","shell.execute_reply.started":"2024-04-24T23:22:27.168078Z","shell.execute_reply":"2024-04-24T23:22:27.168098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[data[\"transcripts\"] == \"\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.169394Z","iopub.status.idle":"2024-04-24T23:22:27.169878Z","shell.execute_reply.started":"2024-04-24T23:22:27.169634Z","shell.execute_reply":"2024-04-24T23:22:27.169653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[data[\"transcripts\"] == \"..\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.172094Z","iopub.status.idle":"2024-04-24T23:22:27.172423Z","shell.execute_reply.started":"2024-04-24T23:22:27.172265Z","shell.execute_reply":"2024-04-24T23:22:27.172278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing all kind of null data from training set","metadata":{}},{"cell_type":"code","source":"# print(list(data[data['transcripts'] == ''].index))\ndata.drop(data[data['transcripts'] == ''].index, inplace=True)\n      \n# print(list(data[data['transcripts'] == '<>'].index))\ndata.drop(data[data['transcripts'] == \"<>\"].index, inplace=True)\n      \n# print(list(data[data['transcripts'] == '..'].index))\ndata.drop(data[data['transcripts'] == \"..\"].index, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.173502Z","iopub.status.idle":"2024-04-24T23:22:27.173852Z","shell.execute_reply.started":"2024-04-24T23:22:27.173682Z","shell.execute_reply":"2024-04-24T23:22:27.173696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing punctuation marks from training set. Here we are not removing '-' '*' '=' '+' ',' ':' ';' '!' as they were attached to the words and if we infer them seperately then we were getting higher WER. so we only remove \"।()[]{}\\,?_~\" these punctuations.","metadata":{}},{"cell_type":"code","source":"import string\nimport re \n\n# Bangla punctuation marks to remove\nbangla_punctuation = \"।()[]{}\\,?_~\"\n\n# Function to remove punctuation marks\ndef remove_punctuation(text):\n    translator = str.maketrans('', '', bangla_punctuation)\n    text = text.translate(translator)\n    # Remove single dots not part of a sequence of continuous dots\n    text = re.sub(r'(?<!\\.)\\.(?!\\.)', '', text)\n    return text\n\n# Apply the function to remove punctuation marks\ndata[\"transcripts\"] = data[\"transcripts\"].apply(remove_punctuation)\n\ndata.head(20)\ndata['transcripts'].to_csv(\"dots.csv\",index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.175480Z","iopub.status.idle":"2024-04-24T23:22:27.175853Z","shell.execute_reply.started":"2024-04-24T23:22:27.175685Z","shell.execute_reply":"2024-04-24T23:22:27.175700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Removing every english characters as we have found characters like \"IELTS\" etc","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport re\n\n# Assuming 'data' is your DataFrame with the 'transcripts' column\n# Replace 'data' with the actual name of your DataFrame\n\n# Define a function to remove English characters from a string using regular expressions\ndef remove_english_characters(text):\n    # Regular expression to match English characters (both lowercase and uppercase)\n    pattern = re.compile(\"[a-zA-Z]\")\n    # Replace English characters with an empty string\n    return re.sub(pattern, \"\", text)\n\n# Apply the function to the 'transcripts' column\ndata['transcripts'] = data['transcripts'].apply(remove_english_characters)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.177696Z","iopub.status.idle":"2024-04-24T23:22:27.178189Z","shell.execute_reply.started":"2024-04-24T23:22:27.177941Z","shell.execute_reply":"2024-04-24T23:22:27.177960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This line removes leading and trailing whitespace from each element in the \"transcripts\" column of the DataFrame.","metadata":{}},{"cell_type":"code","source":"data[\"transcripts\"] = data[\"transcripts\"].str.strip()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.180089Z","iopub.status.idle":"2024-04-24T23:22:27.180590Z","shell.execute_reply.started":"2024-04-24T23:22:27.180321Z","shell.execute_reply":"2024-04-24T23:22:27.180340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We used the pretrained model of TUGSTUGI (https://www.kaggle.com/competitions/bengaliai-speech/discussion/447961) from Bengali.AI Speech Recognition competion's leaderboard top team and FineTune this model for each region. As this model is already trained on huge dataset so this was giving us less WER, which is very good","metadata":{}},{"cell_type":"code","source":"TASK = \"transcribe\"\nMODEL_NAME = \"/kaggle/input/bengali-ai-asr-submission/bengali-whisper-medium\"\n","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.182914Z","iopub.status.idle":"2024-04-24T23:22:27.183537Z","shell.execute_reply.started":"2024-04-24T23:22:27.183195Z","shell.execute_reply":"2024-04-24T23:22:27.183216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading required feature_extractor, tokenizer, processor","metadata":{}},{"cell_type":"code","source":"feature_extractor = WhisperFeatureExtractor.from_pretrained(MODEL_NAME)\ntokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME, language='bn', task=TASK)\nprocessor = WhisperProcessor.from_pretrained(MODEL_NAME, language='bn', task=TASK)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.193034Z","iopub.status.idle":"2024-04-24T23:22:27.193410Z","shell.execute_reply.started":"2024-04-24T23:22:27.193238Z","shell.execute_reply":"2024-04-24T23:22:27.193252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This code initializes an empty string, encodes it using a tokenizer to obtain token IDs, and then decodes the token IDs back into a string, resulting in an empty string.","metadata":{}},{"cell_type":"code","source":"ids = tokenizer.encode(\"\")\nids","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.194401Z","iopub.status.idle":"2024-04-24T23:22:27.194762Z","shell.execute_reply.started":"2024-04-24T23:22:27.194598Z","shell.execute_reply":"2024-04-24T23:22:27.194613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.decode(ids)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.196126Z","iopub.status.idle":"2024-04-24T23:22:27.196650Z","shell.execute_reply.started":"2024-04-24T23:22:27.196365Z","shell.execute_reply":"2024-04-24T23:22:27.196383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This `DataCollator` class is designed for speech sequence-to-sequence tasks, where it prepares batches by padding input audio features and tokenized label sequences separately. It ensures correct padding for both inputs and labels, replaces padding tokens in labels with a placeholder (-100), and removes the beginning-of-sequence (BOS) token if it's present in the labels.","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass DataCollatorSpeechSeq2SeqWithPadding:\n    processor: Any\n\n    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n        # split inputs and labels since they have to be of different lengths and need different padding methods\n        # first treat the audio inputs by simply returning torch tensors\n        input_features = [{\"input_features\": feature[\"input_features\"]} for feature in features]\n        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n\n        # get the tokenized label sequences\n        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n        # pad the labels to max length\n        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n\n        # replace padding with -100 to ignore loss correctly\n        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n\n        # if bos token is appended in previous tokenization step,\n        # cut bos token here as it's append later anyways\n        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n            labels = labels[:, 1:]\n\n        batch[\"labels\"] = labels\n        \n        torch.cuda.empty_cache()\n\n        return batch","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.198513Z","iopub.status.idle":"2024-04-24T23:22:27.199028Z","shell.execute_reply.started":"2024-04-24T23:22:27.198780Z","shell.execute_reply":"2024-04-24T23:22:27.198800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.200891Z","iopub.status.idle":"2024-04-24T23:22:27.201247Z","shell.execute_reply.started":"2024-04-24T23:22:27.201062Z","shell.execute_reply":"2024-04-24T23:22:27.201076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# These functions process a dataset for speech-to-text tasks. `prepare_dataset` loads audio files using librosa, extracts features, and tokenizes transcripts. `filter_inputs` and `filter_labels` ensure the validity of input audio and label sequences by filtering out zero-length inputs or empty label sequences.train_df = data[data[\"split\"] == \"train\"]","metadata":{}},{"cell_type":"code","source":"import librosa\n\ndef prepare_dataset(example):\n    audio_path = example[\"file_path\"]\n    \n    # load the audio using librosa or torch audio (as you wish)\n    audio, sr = librosa.load(audio_path, sr=16_000)\n    \n    example[\"input_features\"] = feature_extractor(audio, sampling_rate=sr).input_features[0]\n    \n    example[\"labels\"] = tokenizer(f\"{example['transcripts']}\", max_length=448, padding=True, truncation=True).input_ids\n    \n    return example\n\n\ndef filter_inputs(input_audio):\n    \"\"\"filter inputs with zero input length\"\"\"\n    return 0 < len(input_audio)\n\n\ndef filter_labels(input_labels):\n    \"\"\"filter empty label sequences\"\"\"\n    return 0 < len(input_labels)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.202247Z","iopub.status.idle":"2024-04-24T23:22:27.202565Z","shell.execute_reply.started":"2024-04-24T23:22:27.202396Z","shell.execute_reply":"2024-04-24T23:22:27.202408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = data[data[\"split\"] == \"train\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.203838Z","iopub.status.idle":"2024-04-24T23:22:27.204187Z","shell.execute_reply.started":"2024-04-24T23:22:27.204011Z","shell.execute_reply":"2024-04-24T23:22:27.204026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n    adjust test size accordingly.\n\"\"\"\ntrain_df, eval_df = train_test_split(train_df, test_size=0.00001, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.205169Z","iopub.status.idle":"2024-04-24T23:22:27.205501Z","shell.execute_reply.started":"2024-04-24T23:22:27.205333Z","shell.execute_reply":"2024-04-24T23:22:27.205347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df), len(eval_df)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.207434Z","iopub.status.idle":"2024-04-24T23:22:27.207839Z","shell.execute_reply.started":"2024-04-24T23:22:27.207649Z","shell.execute_reply":"2024-04-24T23:22:27.207664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This code creates a `DatasetDict` named `ben_reg_voice_ds` containing two splits (`train` and `eval`), each represented by a `Dataset` object initialized from Pandas DataFrames `train_df` and `eval_df`, respectively.","metadata":{}},{"cell_type":"code","source":"ben_reg_voice_ds = DatasetDict()\n\ntrain_split = DS.from_pandas(train_df)\neval_split = DS.from_pandas(eval_df)\n\nds_splits = DatasetDict({\n    'train': train_split,\n    'eval': eval_split\n})","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.209174Z","iopub.status.idle":"2024-04-24T23:22:27.209526Z","shell.execute_reply.started":"2024-04-24T23:22:27.209355Z","shell.execute_reply":"2024-04-24T23:22:27.209370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_splits = ds_splits.remove_columns([\"split\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.210916Z","iopub.status.idle":"2024-04-24T23:22:27.211276Z","shell.execute_reply.started":"2024-04-24T23:22:27.211105Z","shell.execute_reply":"2024-04-24T23:22:27.211119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(ds_splits)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.212436Z","iopub.status.idle":"2024-04-24T23:22:27.212804Z","shell.execute_reply.started":"2024-04-24T23:22:27.212636Z","shell.execute_reply":"2024-04-24T23:22:27.212650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.object = object","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.214016Z","iopub.status.idle":"2024-04-24T23:22:27.214380Z","shell.execute_reply.started":"2024-04-24T23:22:27.214211Z","shell.execute_reply":"2024-04-24T23:22:27.214225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spliting train test set. Here we are using all train set for training only one audio was used for validation.","metadata":{}},{"cell_type":"code","source":"ds_splits = ds_splits.map(prepare_dataset, remove_columns=ds_splits.column_names[\"train\"], num_proc=1 \n                        # open for multithreadding\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.216128Z","iopub.status.idle":"2024-04-24T23:22:27.216474Z","shell.execute_reply.started":"2024-04-24T23:22:27.216306Z","shell.execute_reply":"2024-04-24T23:22:27.216320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ds_splits[\"train\"]), len(ds_splits[\"eval\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.217934Z","iopub.status.idle":"2024-04-24T23:22:27.218262Z","shell.execute_reply.started":"2024-04-24T23:22:27.218102Z","shell.execute_reply":"2024-04-24T23:22:27.218115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cer = CharErrorRate()\nwer = WordErrorRate()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.219396Z","iopub.status.idle":"2024-04-24T23:22:27.219778Z","shell.execute_reply.started":"2024-04-24T23:22:27.219605Z","shell.execute_reply":"2024-04-24T23:22:27.219620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This function calculates Word Error Rate (WER) and Character Error Rate (CER) metrics for evaluating speech recognition models. It computes WER and CER from predicted and label token IDs, then returns a dictionary containing the computed metrics. Optionally, it prints WER, CER, predicted transcription, and ground truth transcription for the first example during evaluation.","metadata":{}},{"cell_type":"code","source":"def compute_metrics(pred):\n    pred_ids = pred.predictions\n    label_ids = pred.label_ids\n\n    label_ids[label_ids == -100] = tokenizer.pad_token_id\n\n    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)\n    label_str = tokenizer.batch_decode(label_ids, skip_special_tokens=True)\n\n    wer_res = wer(pred_str, label_str)\n    cer_res = cer(pred_str, label_str)\n    \n    \"\"\"\n        uncomment the next 3 lines if you want to see how the examples look like during eval \n    \"\"\"\n    print(\"WER:\",wer_res,\"| CER:\", cer_res) # to show up during running logs\n    print(\"Pred:\",pred_str[0])\n    print(\"Label:\",label_str[0])\n    \n    return {\"wer\": wer_res, \"cer\": cer_res}","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.220884Z","iopub.status.idle":"2024-04-24T23:22:27.221210Z","shell.execute_reply.started":"2024-04-24T23:22:27.221050Z","shell.execute_reply":"2024-04-24T23:22:27.221063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading model","metadata":{}},{"cell_type":"code","source":"model = WhisperForConditionalGeneration.from_pretrained(MODEL_NAME, device_map=\"auto\")\nmodel.config.forced_decoder_ids = tokenizer.get_decoder_prompt_ids(language=\"bn\", task=\"transcribe\")","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.222189Z","iopub.status.idle":"2024-04-24T23:22:27.222490Z","shell.execute_reply.started":"2024-04-24T23:22:27.222341Z","shell.execute_reply":"2024-04-24T23:22:27.222353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.generation_config)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.223927Z","iopub.status.idle":"2024-04-24T23:22:27.224241Z","shell.execute_reply.started":"2024-04-24T23:22:27.224089Z","shell.execute_reply":"2024-04-24T23:22:27.224101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## This is the model_id where our trained model will be saved","metadata":{}},{"cell_type":"code","source":"model_id = f\"whisper-medium-reg-ben/region_based/train_1_no_pretrained_25_epochs_{train_region_name}\"","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.225218Z","iopub.status.idle":"2024-04-24T23:22:27.225598Z","shell.execute_reply.started":"2024-04-24T23:22:27.225398Z","shell.execute_reply":"2024-04-24T23:22:27.225412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# These `training_args` specify the training configurations for a sequence-to-sequence model, including settings for batch size, learning rate, evaluation strategy, checkpointing, mixed precision training (FP16), and logging. The model is trained for 25 epochs with a per-device batch size of 16 for training and 8 for evaluation. Learning Rate 1e-5 and weight decay 0.0025","metadata":{}},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir=model_id,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=8,\n    gradient_accumulation_steps=1,\n    gradient_checkpointing=True,\n    fp16=True,\n    learning_rate=1e-5, \n    weight_decay=0.0025,\n    warmup_steps=200,\n    num_train_epochs=25,\n    evaluation_strategy=\"steps\", # or \"epochs\"\n    predict_with_generate=True,\n#     generation_max_length=448,\n    save_steps=1000,\n    eval_steps=1000,\n    logging_steps=100,\n    save_total_limit=5,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"wer\",\n    greater_is_better=False,\n    push_to_hub=False,\n    report_to=\"none\",\n    remove_unused_columns=False,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.227610Z","iopub.status.idle":"2024-04-24T23:22:27.227988Z","shell.execute_reply.started":"2024-04-24T23:22:27.227820Z","shell.execute_reply":"2024-04-24T23:22:27.227834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.generation_config.language = \"bn\"\n#model.generation_config.task = \"transcribe\"\n\n\nmodel.generation_config.forced_decoder_ids = None\nmodel.config.suppress_tokens = [] # added later","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.229644Z","iopub.status.idle":"2024-04-24T23:22:27.230013Z","shell.execute_reply.started":"2024-04-24T23:22:27.229832Z","shell.execute_reply":"2024-04-24T23:22:27.229848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.config.task_to_id = {\n    \"transcribe\": 50359,\n    \"translate\": 50358\n  }","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.231510Z","iopub.status.idle":"2024-04-24T23:22:27.231908Z","shell.execute_reply.started":"2024-04-24T23:22:27.231715Z","shell.execute_reply":"2024-04-24T23:22:27.231730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This `Seq2SeqTrainer` instance is set up with the specified training arguments, the sequence-to-sequence model, training and evaluation datasets (`train_dataset` and `eval_dataset`), a data collator, and a tokenizer. It also specifies a function (`compute_metrics`) for computing evaluation metrics during training.","metadata":{}},{"cell_type":"code","source":"trainer = Seq2SeqTrainer(\n    args=training_args,\n    model=model,\n    train_dataset=ds_splits[\"train\"],\n    eval_dataset=ds_splits[\"eval\"],\n    data_collator=data_collator,\n    tokenizer=processor.feature_extractor,\n    compute_metrics=compute_metrics,\n#     callbacks=[EarlyStoppingCallback(2, 1.0)]\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.233178Z","iopub.status.idle":"2024-04-24T23:22:27.233573Z","shell.execute_reply.started":"2024-04-24T23:22:27.233363Z","shell.execute_reply":"2024-04-24T23:22:27.233378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model and then save them to given path","metadata":{}},{"cell_type":"code","source":"trainer.train()\n\n# to use the high-level pipeline, ensure both the processor outputs and model outputs exist in the same dir\ntrainer.save_model(training_args.output_dir)\nprocessor.save_pretrained(training_args.output_dir)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.234779Z","iopub.status.idle":"2024-04-24T23:22:27.235165Z","shell.execute_reply.started":"2024-04-24T23:22:27.234971Z","shell.execute_reply":"2024-04-24T23:22:27.234987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving logs file","metadata":{}},{"cell_type":"code","source":"out_logs = pd.DataFrame(trainer.state.log_history)\nout_logs.to_csv(\"logs.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.236402Z","iopub.status.idle":"2024-04-24T23:22:27.236781Z","shell.execute_reply.started":"2024-04-24T23:22:27.236601Z","shell.execute_reply":"2024-04-24T23:22:27.236616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n \ndel ds_splits\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.238141Z","iopub.status.idle":"2024-04-24T23:22:27.238479Z","shell.execute_reply.started":"2024-04-24T23:22:27.238311Z","shell.execute_reply":"2024-04-24T23:22:27.238324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clear gpu","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-04-24T23:22:27.239998Z","iopub.status.idle":"2024-04-24T23:22:27.240372Z","shell.execute_reply.started":"2024-04-24T23:22:27.240192Z","shell.execute_reply":"2024-04-24T23:22:27.240208Z"},"trusted":true},"execution_count":null,"outputs":[]}]}