{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -q git+https://github.com/csebuetnlp/normalizer","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:13:02.071675Z","iopub.execute_input":"2023-08-08T21:13:02.071961Z","iopub.status.idle":"2023-08-08T21:13:24.260710Z","shell.execute_reply.started":"2023-08-08T21:13:02.071935Z","shell.execute_reply":"2023-08-08T21:13:24.259442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VectorizeChar:\n    def __init__(self, max_len=50,unique_chars={}):\n        self.unique_chars=unique_chars\n        self.vocab = (\n            [\"-\", \"#\", \"<\", \">\"]\n            + list(self.unique_chars)\n            # + [chr(i + 96) for i in range(1, 27)]\n            # + [\" \", \".\", \",\", \"?\"]\n        )\n        \n        self.max_len=len(self.vocab)\n        self.char_to_idx = {}\n        for i, ch in enumerate(self.vocab):\n            self.char_to_idx[ch] = i\n\n    def __call__(self, text):\n        # text = text.lower()\n        text = text[: self.max_len - 2]\n        text = \"<\" + text + \">\"\n        pad_len = self.max_len - len(text)\n        return [self.char_to_idx.get(ch, 1) for ch in text] + [0] * pad_len\n\n    def get_vocabulary(self):\n        return self.vocab","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:32:11.347424Z","iopub.execute_input":"2023-08-08T21:32:11.347801Z","iopub.status.idle":"2023-08-08T21:32:11.360180Z","shell.execute_reply.started":"2023-08-08T21:32:11.347771Z","shell.execute_reply":"2023-08-08T21:32:11.359126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = VectorizeChar()\nvectorizer.vocab","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:32:17.858328Z","iopub.execute_input":"2023-08-08T21:32:17.858704Z","iopub.status.idle":"2023-08-08T21:32:17.864894Z","shell.execute_reply.started":"2023-08-08T21:32:17.858674Z","shell.execute_reply":"2023-08-08T21:32:17.863992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport librosa\nimport pandas as pd\nimport plotly.graph_objects as go\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport re\nfrom torch.utils.data import Dataset, DataLoader\nimport csv\nfrom normalizer import normalize\n\nFILTER_CHARS = [\n    '\"', '%', \"'\", ',', '-', '.', '/', '\\x93', '\\x94', '\\u200c', '\\u200d', '‘',\n    '’', '“', '”', '…', '!', ':', ';', '?', 'œ', '।', '–', '||', '‚', '—', '॥', '৷'\n]\n\ndef clean(text):\n    '''Clean text'''\n    for c in FILTER_CHARS:\n        text = text.replace(c, '')\n    return text\n\nclass SpeechRecognitionDataset(Dataset):\n    def __init__(self, audio_folder, csv_file, max_target_len, mode=\"train\",max_examples=None):\n        self.audio_folder = audio_folder\n        self.max_target_len = max_target_len\n        self.mode = mode\n        self.data_list = []\n        self.unique_chars = set()\n        self.unique_sentences = set()\n        with open(csv_file, newline='', encoding='utf-8') as csvfile:\n            reader = csv.DictReader(csvfile)\n            for row in reader:\n                sentence = clean(row[\"sentence\"])\n                for char in sentence:\n                    self.unique_chars.add(char)\n\n        with open(csv_file, newline='', encoding='utf-8') as csvfile:\n            reader = csv.DictReader(csvfile)\n            for row in reader:\n                num_examples = len(self.data_list)\n                if max_examples is not None and num_examples >= max_examples:\n                    break\n                sentence = clean(row[\"sentence\"])\n                row[\"sentence\"]=clean(row[\"sentence\"])\n                if len(sentence) <= self.max_target_len and sentence not in self.unique_sentences:\n                    self.unique_sentences.add(sentence)\n                    if clean(row[\"split\"]) == \"train\" and mode == \"train\":\n                        self.data_list.append(row)\n                    elif clean(row[\"split\"]) == \"valid\" and mode == \"test\":\n                        self.data_list.append(row)\n\n#                 for char in sentence:\n#                     self.unique_chars.add(char)\n                #num_examples += 1\n                    \n        self.vectorizer = VectorizeChar(unique_chars=self.unique_chars, max_len=max_target_len)\n\n    def __len__(self):\n        return len(self.data_list)\n\n    def visualize_sample(self, idx):\n        audio_file_path = os.path.join(self.audio_folder, self.data_list[idx][\"id\"] + \".mp3\")\n        audio, sr = librosa.load(audio_file_path, sr=None)\n\n        plt.figure(figsize=(10, 4))\n\n        # Plot the waveform\n        plt.subplot(2, 1, 1)\n        librosa.display.waveshow(audio, sr=sr)\n        plt.title(\"Waveform\")\n\n        # Plot the spectrogram\n        plt.subplot(2, 1, 2)\n        S = librosa.feature.melspectrogram(y=audio, sr=sr)\n        S_dB = librosa.power_to_db(S, ref=np.max)\n        librosa.display.specshow(S_dB, x_axis=\"time\", y_axis=\"mel\")\n        plt.colorbar(format=\"%+2.0f dB\")\n        plt.title(\"Spectrogram\")\n\n        # Add the annotation text\n        plt.text(0, 0, self.data_list[idx][\"id\"], fontsize=12, color='white', va='center', ha='left', backgroundcolor='black')\n\n        plt.tight_layout()\n        plt.show()\n\n    def visualize_sample_3d_interactive(self, idx):\n        audio_file_path = os.path.join(self.audio_folder, self.data_list[idx][\"id\"] + \".mp3\")\n        audio, sr = librosa.load(audio_file_path, sr=None)\n        S = librosa.feature.melspectrogram(y=audio, sr=sr)\n        S_dB = librosa.power_to_db(S, ref=np.max)\n\n        t, f = np.meshgrid(np.linspace(0, librosa.get_duration(y=audio, sr=sr), S_dB.shape[1]),\n                           np.linspace(0, sr/2, S_dB.shape[0]))\n\n        # Create an interactive 3D surface plot using Plotly\n        fig = go.Figure(data=[go.Surface(z=S_dB, x=t, y=f)])\n        fig.update_layout(title='Spectrogram (3D Interactive)',\n                          scene=dict(xaxis_title='Time (s)',\n                                     yaxis_title='Frequency (Hz)',\n                                     zaxis_title='dB'))\n        fig.show()\n        \n    def __getitem__(self, idx):\n        audio_file_path = os.path.join(self.audio_folder, self.data_list[idx][\"id\"] + \".mp3\")\n        audio, sr = librosa.load(audio_file_path, sr=16000)\n        stft = librosa.stft(audio, n_fft=256, hop_length=80, win_length=200)\n        magnitude_spectrogram = np.abs(stft) ** 0.5\n        pad_len = 1700\n        num_frames = magnitude_spectrogram.shape[1]\n        if num_frames < pad_len:\n            # Calculate the required padding\n            pad_width = pad_len - num_frames\n            # Pad the spectrogram with zeros along the time axis (columns)\n            magnitude_spectrogram = np.pad(magnitude_spectrogram, ((0, 0), (0, pad_width)), mode='constant')\n        else:\n            # Trim the spectrogram if it has more frames than required\n            magnitude_spectrogram = magnitude_spectrogram[:, :pad_len]\n\n        # Tokenize the annotation and convert to numeric representation\n        tokenized_annotation = self.vectorizer(self.data_list[idx][\"sentence\"])\n\n        return [torch.tensor(magnitude_spectrogram, dtype=torch.float32).permute(1, 0), torch.tensor(tokenized_annotation, dtype=torch.long)]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:32:29.246400Z","iopub.execute_input":"2023-08-08T21:32:29.246777Z","iopub.status.idle":"2023-08-08T21:32:29.275050Z","shell.execute_reply.started":"2023-08-08T21:32:29.246745Z","shell.execute_reply":"2023-08-08T21:32:29.273850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio_folder = \"/kaggle/input/bengaliai-speech/train_mp3s\"\ncsv_file = \"/kaggle/input/bengaliai-speech/train.csv\"\ndataset_train = SpeechRecognitionDataset(audio_folder, csv_file,max_target_len=50,mode=\"train\",max_examples=500)\ndataset_test = SpeechRecognitionDataset(audio_folder, csv_file,max_target_len=50,mode=\"test\",max_examples=10)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:32:48.971301Z","iopub.execute_input":"2023-08-08T21:32:48.971669Z","iopub.status.idle":"2023-08-08T21:33:25.806270Z","shell.execute_reply.started":"2023-08-08T21:32:48.971640Z","shell.execute_reply":"2023-08-08T21:33:25.805111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_train.vectorizer.char_to_idx","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:33:37.436387Z","iopub.execute_input":"2023-08-08T21:33:37.436775Z","iopub.status.idle":"2023-08-08T21:33:37.446670Z","shell.execute_reply.started":"2023-08-08T21:33:37.436745Z","shell.execute_reply":"2023-08-08T21:33:37.445683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dataset_train)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:33:46.521370Z","iopub.execute_input":"2023-08-08T21:33:46.521742Z","iopub.status.idle":"2023-08-08T21:33:46.527796Z","shell.execute_reply.started":"2023-08-08T21:33:46.521710Z","shell.execute_reply":"2023-08-08T21:33:46.526826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_train.visualize_sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:33:53.217058Z","iopub.execute_input":"2023-08-08T21:33:53.217450Z","iopub.status.idle":"2023-08-08T21:33:54.127872Z","shell.execute_reply.started":"2023-08-08T21:33:53.217420Z","shell.execute_reply":"2023-08-08T21:33:54.126959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_train.visualize_sample_3d_interactive(0)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:03.834756Z","iopub.execute_input":"2023-08-08T21:34:03.835147Z","iopub.status.idle":"2023-08-08T21:34:03.885495Z","shell.execute_reply.started":"2023-08-08T21:34:03.835112Z","shell.execute_reply":"2023-08-08T21:34:03.884306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**plot statistique**","metadata":{"execution":{"iopub.status.busy":"2023-07-31T22:38:03.154562Z","iopub.execute_input":"2023-07-31T22:38:03.154955Z","iopub.status.idle":"2023-07-31T22:38:03.162278Z","shell.execute_reply.started":"2023-07-31T22:38:03.154923Z","shell.execute_reply":"2023-07-31T22:38:03.160944Z"}}},{"cell_type":"code","source":"#train_data_loader = DataLoader(dataset_train, batch_size=1024, shuffle=True)\n#test_data_loader = DataLoader(dataset_test, batch_size=1024, shuffle=True)\n\n#imputs = next(enumerate(train_data_loader))","metadata":{"execution":{"iopub.status.busy":"2023-08-01T19:13:17.299724Z","iopub.execute_input":"2023-08-01T19:13:17.300149Z","iopub.status.idle":"2023-08-01T19:13:34.535901Z","shell.execute_reply.started":"2023-08-01T19:13:17.300117Z","shell.execute_reply":"2023-08-01T19:13:34.534803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchmetrics.text import CharErrorRate\n\nclass DisplayOutputsPyTorch:\n    def __init__(self, dataloader, idx_to_token, target_start_token_idx=27, target_end_token_idx=28):\n        \"\"\"Displays a batch of outputs after every epoch\n\n        Args:\n            dataloader: A PyTorch DataLoader for the validation dataset\n            idx_to_token: A List containing the vocabulary tokens corresponding to their indices\n            target_start_token_idx: A start token index in the target vocabulary\n            target_end_token_idx: An end token index in the target vocabulary\n        \"\"\"\n        self.dataloader = dataloader\n        self.target_start_token_idx = target_start_token_idx\n        self.target_end_token_idx = target_end_token_idx\n        self.idx_to_char = idx_to_token\n        self.cer= CharErrorRate()\n\n\n    def on_epoch_end(self, epoch, model):\n        if epoch % 50 != 0:\n            return\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        model.eval()  # Set the model to evaluation mode\n        with torch.no_grad():\n            \n            total_cer = 0.0\n            total_samples = 0\n            for batch in self.dataloader:\n                \n                \n                source, target = batch\n                \n                bs = source.size(0)\n                source = source.to(device)\n                preds = model.generate(source, self.target_start_token_idx)\n                preds = preds.cpu().numpy()\n\n                for i in range(bs):\n                    target_text = \"\".join([self.idx_to_char[_] for _ in target[i]])\n                    prediction = \"\"\n                    for idx in preds[i]:\n                        prediction += self.idx_to_char[idx.item()]\n                        if idx == self.target_end_token_idx:\n                            break\n                \n                    target_text=target_text.replace('-', '')\n                    cer = CharErrorRate()\n                    batch_cer=cer(target_text,prediction)\n                    print(f\"target:     {target_text}\")\n                    print(f\"prediction: {prediction}\")\n                    print(f\"Validation CER: {batch_cer:.4f}\")\n        model.train()  \n","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:16.703269Z","iopub.execute_input":"2023-08-08T21:34:16.703647Z","iopub.status.idle":"2023-08-08T21:34:16.719000Z","shell.execute_reply.started":"2023-08-08T21:34:16.703614Z","shell.execute_reply":"2023-08-08T21:34:16.717980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass TokenEmbedding(nn.Module):\n    def __init__(self, num_vocab=1000, maxlen=100, num_hid=64):\n        super(TokenEmbedding, self).__init__()\n        self.emb = nn.Embedding(num_vocab, num_hid)\n        self.pos_emb = nn.Embedding(maxlen, num_hid)\n\n    def forward(self, x):\n        maxlen = x.size(-1)\n        x = self.emb(x)\n        positions = torch.arange(0, maxlen, dtype=torch.long, device=x.device)\n        positions = self.pos_emb(positions)\n        return x + positions\n\n\nclass SpeechFeatureEmbedding(nn.Module):\n    def __init__(self, num_hid=64, maxlen=100):\n        super(SpeechFeatureEmbedding, self).__init__()\n        self.conv1 = nn.Conv1d(129, num_hid, kernel_size=11, stride=2, padding=5)\n        self.conv2 = nn.Conv1d(num_hid, num_hid, kernel_size=11, stride=2, padding=5)\n        self.conv3 = nn.Conv1d(num_hid, num_hid, kernel_size=11, stride=2, padding=5)\n        self.pos_emb = nn.Embedding(maxlen, num_hid)\n\n    def forward(self, x):\n        x = x.transpose(1, 2)  # Convert input to [batch, features, seq_len]\n        x = self.conv1(x)\n        x = self.conv2(x)\n        return self.conv3(x).transpose(1, 2)  # Convert back to [batch, seq_len, features]\n\n\nclass TransformerEncoder(nn.Module):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, rate=0.1):\n        super(TransformerEncoder, self).__init__()\n        self.att = nn.MultiheadAttention(embed_dim, num_heads)\n        self.ffn = nn.Sequential(\n            nn.Linear(embed_dim, feed_forward_dim),\n            nn.ReLU(),\n            nn.Linear(feed_forward_dim, embed_dim)\n        )\n        self.layernorm1 = nn.LayerNorm(embed_dim, eps=1e-6)\n        self.layernorm2 = nn.LayerNorm(embed_dim, eps=1e-6)\n        self.dropout1 = nn.Dropout(rate)\n        self.dropout2 = nn.Dropout(rate)\n\n    def forward(self, inputs, mask=None):\n        attn_output, _ = self.att(inputs, inputs, inputs, attn_mask=mask)\n        attn_output = self.dropout1(attn_output)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output)\n        return self.layernorm2(out1 + ffn_output)\n\nclass TransformerDecoder(nn.Module):\n    def __init__(self, embed_dim, num_heads, feed_forward_dim, dropout_rate=0.1):\n        super(TransformerDecoder, self).__init__()\n        self.layernorm1 = nn.LayerNorm(embed_dim, eps=1e-6)\n        self.layernorm2 = nn.LayerNorm(embed_dim, eps=1e-6)\n        self.layernorm3 = nn.LayerNorm(embed_dim, eps=1e-6)\n        self.self_att = nn.MultiheadAttention(embed_dim, num_heads,batch_first=True)\n        self.enc_att = nn.MultiheadAttention(embed_dim, num_heads,batch_first=True)\n        self.self_dropout = nn.Dropout(0.5)\n        self.enc_dropout = nn.Dropout(0.1)\n        self.ffn_dropout = nn.Dropout(0.1)\n        self.ffn = nn.Sequential(\n            nn.Linear(embed_dim, feed_forward_dim),\n            nn.ReLU(),\n            nn.Linear(feed_forward_dim, embed_dim)\n        )\n\n    def causal_attention_mask(self, batch_size, n_dest, n_src, device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")):\n        \n        i = torch.arange(n_dest).unsqueeze(1)\n        j = torch.arange(n_src)\n        mask = i >= j - n_src + n_dest\n        mask = mask.to(device)\n        mask = mask.unsqueeze(0)\n        mult = torch.cat([torch.tensor([batch_size]).to(device), torch.tensor([1, 1]).to(device)], dim=0)\n        return mask.repeat(mult)\n\n    def forward(self, enc_out, target, src_mask=None, target_mask=None):\n        target_att, _ = self.self_att(target, target, target, attn_mask=target_mask)\n        target_att = self.self_dropout(target_att)\n        target_norm = self.layernorm1(target + target_att)\n        enc_out_att, _ = self.enc_att(target_norm, enc_out, enc_out, attn_mask=src_mask)\n        enc_out_att = self.enc_dropout(enc_out_att)\n        enc_out_norm = self.layernorm2(enc_out_att + target_norm)\n        ffn_out = self.ffn(enc_out_norm)\n        ffn_out = self.ffn_dropout(ffn_out)\n        return self.layernorm3(enc_out_norm + ffn_out)\n\n\n\nclass Transformer(nn.Module):\n    def __init__(\n        self,\n        num_hid=64,\n        num_head=2,\n        num_feed_forward=128,\n        source_maxlen=100,\n        target_maxlen=100,\n        num_layers_enc=4,\n        num_layers_dec=1,\n        num_classes=10,\n    ):\n        super(Transformer, self).__init__()\n        self.loss_metric = nn.CrossEntropyLoss()\n        self.num_layers_enc = num_layers_enc\n        self.num_layers_dec = num_layers_dec\n        self.target_maxlen = target_maxlen\n        self.num_classes = num_classes\n\n        self.enc_input = SpeechFeatureEmbedding(num_hid=num_hid, maxlen=source_maxlen)\n        self.dec_input = TokenEmbedding(num_vocab=num_classes, maxlen=target_maxlen, num_hid=num_hid)\n\n        self.encoder = nn.Sequential(\n            self.enc_input,\n            *[TransformerEncoder(num_hid, num_head, num_feed_forward) for _ in range(num_layers_enc)]\n        )\n\n        for i in range(num_layers_dec):\n            setattr(self, f\"dec_layer_{i}\", TransformerDecoder(num_hid, num_head, num_feed_forward))\n\n        self.classifier = nn.Linear(num_hid, num_classes)\n\n    def decode(self, enc_out, target, src_mask=None):\n        y = self.dec_input(target)\n        for i in range(self.num_layers_dec):\n            y = getattr(self, f\"dec_layer_{i}\")(enc_out, y, src_mask=src_mask, target_mask=None)\n        return y\n\n    def forward(self, inputs, src_mask=None):\n        source = inputs[0]\n        target = inputs[1]\n        x = self.encoder(source)\n        y = self.decode(x, target, src_mask=src_mask)\n        return self.classifier(y)\n\n    def generate(self, source, target_start_token_idx):\n        bs = source.size(0)\n        enc = self.encoder(source)\n        dec_input = torch.ones((bs, 1), dtype=torch.long, device=source.device) * target_start_token_idx\n        dec_logits = []\n        for i in range(self.target_maxlen - 1):\n            dec_out = self.decode(enc, dec_input, src_mask=None)\n            logits = self.classifier(dec_out)\n            logits = torch.argmax(logits, dim=-1)\n            last_logit = logits[:, -1].unsqueeze(1)\n            dec_logits.append(last_logit)\n            dec_input = torch.cat([dec_input, last_logit], dim=-1)\n        return dec_input\n","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:23.815045Z","iopub.execute_input":"2023-08-08T21:34:23.815601Z","iopub.status.idle":"2023-08-08T21:34:23.850881Z","shell.execute_reply.started":"2023-08-08T21:34:23.815566Z","shell.execute_reply":"2023-08-08T21:34:23.849597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport torch.optim.lr_scheduler as lr_scheduler\nimport torch.optim as optim\n\nclass CustomSchedule:\n    def __init__(\n        self,\n        init_lr=0.00001,\n        lr_after_warmup=0.001,\n        final_lr=0.00001,\n        warmup_epochs=15,\n        decay_epochs=85,\n        steps_per_epoch=203,\n    ):\n        self.init_lr = init_lr\n        self.lr_after_warmup = lr_after_warmup\n        self.final_lr = final_lr\n        self.warmup_epochs = warmup_epochs\n        self.decay_epochs = decay_epochs\n        self.steps_per_epoch = steps_per_epoch\n\n    def calculate_lr(self, epoch):\n        \"\"\" linear warm up - linear decay \"\"\"\n        warmup_lr = self.init_lr + ((self.lr_after_warmup - self.init_lr) / (self.warmup_epochs - 1)) * epoch\n        decay_lr = max(self.final_lr, self.lr_after_warmup - (epoch - self.warmup_epochs) * (self.lr_after_warmup - self.final_lr) / self.decay_epochs)\n        lr = min(warmup_lr, decay_lr)\n        return lr\nmax_target_len=50","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:31.627740Z","iopub.execute_input":"2023-08-08T21:34:31.628149Z","iopub.status.idle":"2023-08-08T21:34:31.637279Z","shell.execute_reply.started":"2023-08-08T21:34:31.628113Z","shell.execute_reply":"2023-08-08T21:34:31.636180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Transformer(\n    num_hid=128,\n    num_head=2,\n    num_feed_forward=256,\n    target_maxlen=70,\n    source_maxlen=70,\n    num_layers_enc=1,\n    num_layers_dec=1,\n    num_classes=len(dataset_train.vectorizer.get_vocabulary()) # 75 # 48 # 67 # 108,\n)\n\n# optimizer = optim.Adam(model.parameters(), lr=learning_rate.init_lr) ","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:37.337280Z","iopub.execute_input":"2023-08-08T21:34:37.337650Z","iopub.status.idle":"2023-08-08T21:34:37.356439Z","shell.execute_reply.started":"2023-08-08T21:34:37.337619Z","shell.execute_reply":"2023-08-08T21:34:37.355516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_train.vectorizer.char_to_idx==dataset_test.vectorizer.char_to_idx","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:41.466397Z","iopub.execute_input":"2023-08-08T21:34:41.466770Z","iopub.status.idle":"2023-08-08T21:34:41.474110Z","shell.execute_reply.started":"2023-08-08T21:34:41.466739Z","shell.execute_reply":"2023-08-08T21:34:41.473052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss(reduction='mean', label_smoothing=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:44.137748Z","iopub.execute_input":"2023-08-08T21:34:44.138355Z","iopub.status.idle":"2023-08-08T21:34:44.143621Z","shell.execute_reply.started":"2023-08-08T21:34:44.138319Z","shell.execute_reply":"2023-08-08T21:34:44.142307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef train_and_validate_model(model, train_dataset, test_dataset, num_epochs, batch_size, learning_rate,checkpoint_dir):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model.to(device)\n    best_accuracy = 0.0\n    # Create data loaders\n    train_loader = DataLoader(train_dataset, batch_size=512, shuffle=True)\n    test_loader = DataLoader(test_dataset, batch_size=10, shuffle=False)\n\n    # Define the loss function\n    loss_fn = nn.CrossEntropyLoss(reduction='mean', label_smoothing=0.1)\n    learning_rate = CustomSchedule(\n        init_lr=learning_rate,\n        lr_after_warmup=0.001,\n        final_lr=0.00001,\n        warmup_epochs=15,\n        decay_epochs=20,\n        steps_per_epoch=len(train_loader),\n    )\n    # Define the optimizer\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate.init_lr) \n\n    # Training loop\n    for epoch in range(1, num_epochs + 1):\n        model.train()\n        total_loss = 0\n        ####\n        correct = 0\n        total_samples = 0\n        #######\n        with tqdm(total=len(train_loader), desc=f\"Epoch {epoch}/{num_epochs}\", unit=\"batch\") as pbar:\n            for batch in train_loader:\n                spectrogram, target = batch\n                spectrogram, target = spectrogram.to(device), target.to(device)\n\n                # Forward pass\n                output = model([spectrogram, target])\n\n                # Compute the loss\n                loss = loss_fn(output, target)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                _, predicted = torch.max(output, dim=1)\n                total_samples += target.size(0)*target.size(1)\n                correct += (predicted == target).sum().item()\n                total_loss += loss.item()\n                pbar.set_postfix({\"Loss\": total_loss / (pbar.n + 1)})  \n                pbar.update(1)\n\n        average_loss = total_loss / len(train_loader)\n        ####\n        accuracy = correct / total_samples * 100\n        ####\n        print(f\"Epoch {epoch}/{num_epochs} - Training Loss: {average_loss:.4f} - Training accuracy: {accuracy:.4f}\")\n\n        # Validation loop\n        model.eval()\n        correct = 0\n        total_samples = 0\n\n        with torch.no_grad():\n            for batch in test_loader:\n                spectrogram, target = batch\n                spectrogram, target = spectrogram.to(device), target.to(device)\n\n                # Forward pass\n                output = model([spectrogram, target])\n\n                # Compute accuracy\n                _, predicted = torch.max(output, dim=1)\n                total_samples += target.size(0)*target.size(1)\n                correct += (predicted == target).sum().item()\n        idx_to_token=dataset_train.vectorizer.get_vocabulary()\n        display_callback = DisplayOutputsPyTorch(test_loader, idx_to_token, target_start_token_idx=2, target_end_token_idx=3)\n        display_callback.on_epoch_end(epoch, model)        \n        accuracy = correct / total_samples * 100\n        print(f\"Epoch {epoch}/{num_epochs} - Validation Accuracy: {accuracy:.2f}%\")\n        \n        # Save checkpoint if accuracy is the best so far\n        if accuracy > best_accuracy:\n            best_accuracy = accuracy\n            checkpoint_filename = os.path.join(checkpoint_dir, \"best_checkpoint.pth\")\n            checkpoint = {\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'loss': average_loss,\n                'accuracy': accuracy,\n            }\n            torch.save(checkpoint, checkpoint_filename)\n\n    print(\"Training and validation completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:47.652605Z","iopub.execute_input":"2023-08-08T21:34:47.653011Z","iopub.status.idle":"2023-08-08T21:34:47.672014Z","shell.execute_reply.started":"2023-08-08T21:34:47.652968Z","shell.execute_reply":"2023-08-08T21:34:47.670927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 600\nbatch_size = 1024\nlearning_rate = 0.00001\nbatch_size=10\ncheckpoint_dir=\"/kaggle/working/\"\ntrain_and_validate_model(model, dataset_train, dataset_test, num_epochs, batch_size, learning_rate,checkpoint_dir)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T21:34:55.323967Z","iopub.execute_input":"2023-08-08T21:34:55.324692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#111111 : perform max_sample without effect the vocab num caracter\n#a\n#MFCCs or stft\n#add stat about dataset\n#thinking about wer as loss function torchmetrics.text","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for epoch in range(2):\n#     for batch in train_data_loader:\n#         src,target=batch\n#         model_output=model(batch)\n#         loss=loss_fn(model_output,target)\n#         optimizer.zero_grad()\n#         loss.backward()       # Compute gradients\n#         optimizer.step()      # Update model parameters\n\n#         # Print or log the loss for monitoring training progress\n#         print(f\"Epoch {epoch + 1}, Batch Loss: {loss.item()}\")\n#         print(f\"Learning Rate: {optimizer.param_groups[0]['lr']}\")","metadata":{"execution":{"iopub.status.busy":"2023-08-01T19:16:12.749043Z","iopub.execute_input":"2023-08-01T19:16:12.749518Z","iopub.status.idle":"2023-08-01T20:09:05.467547Z","shell.execute_reply.started":"2023-08-01T19:16:12.749481Z","shell.execute_reply":"2023-08-01T20:09:05.465764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchmetrics.text import CharErrorRate\npreds = [\"যথারীতি সেখানেও সাফল্যের স্বাক্ষর রাখলেন সিদ্দিক\"]\ntarget= [\"ীজীঅঅোঢ#়অোলখশঙৃলঅোঊশোাচোলঘজঝঁঅোলাোজঋ্জঃউঊনৌোজৎ৵ঝওঘজঞৈোোঊনোোোলাঊশথঈোঊ\"]\ncer = CharErrorRate()\ncer(preds, target)","metadata":{"execution":{"iopub.status.busy":"2023-08-08T14:39:59.552938Z","iopub.execute_input":"2023-08-08T14:39:59.553403Z","iopub.status.idle":"2023-08-08T14:39:59.568096Z","shell.execute_reply.started":"2023-08-08T14:39:59.553369Z","shell.execute_reply":"2023-08-08T14:39:59.567090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import torch\n# import librosa\n# import pandas as pd\n# import plotly.graph_objects as go\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import re\n# from torch.utils.data import Dataset, DataLoader\n\n# class SpeechRecognitionDataset(Dataset):\n#     def __init__(self, audio_folder, csv_file,vectorizer):\n#         self.audio_folder = audio_folder\n#         self.annotations_df = pd.read_csv(csv_file)\n#         self.vectorizer=vectorizer\n\n#     def __len__(self):\n#         return len(self.annotations_df)\n\n#     def __getitem__(self, idx):\n#         audio_filename = self.annotations_df.iloc[idx, 0]\n#         annotation = self.annotations_df.iloc[idx, 1]\n\n#         audio_file_path = os.path.join(self.audio_folder, audio_filename + \".mp3\")\n#         audio, sr = librosa.load(audio_file_path, sr=16000)\n#         mfccs = self.extract_mfcc(audio)\n\n#         # Tokenize the annotation and convert to numeric representation\n#         # Tokenize the annotation into individual characters\n#         tokenized_annotation = self.vectorizer(annotation)\n\n#         return torch.tensor(mfccs, dtype=torch.float32), torch.tensor(tokenized_annotation, dtype=torch.long)\n\n    \n\n#     def extract_mfcc(self, audio):\n#         sr = 16000  # Adjust this value as per your audio sample rate requirements\n#         n_fft = 256\n#         hop_length = 80\n#         win_length = 200\n#         stft = librosa.core.stft(audio, n_fft=n_fft, hop_length=hop_length, win_length=win_length)\n#         x = librosa.amplitude_to_db(np.abs(stft))  # Convert to dB scale (Optional)\n\n#         # Perform padding if necessary\n#         pad_len = 1700\n#         num_frames = x.shape[1]\n#         if num_frames < pad_len:\n#             x = np.pad(x, ((0, 0), (0, pad_len - num_frames)))\n#         else:\n#             x = x[:, :pad_len]\n#         return x\n\n\n#     def visualize_sample(self, idx):\n#         audio_filename = self.annotations_df.iloc[idx, 0]\n#         annotation = self.annotations_df.iloc[idx, 1]\n\n#         audio_file_path = os.path.join(self.audio_folder, audio_filename + \".mp3\")\n#         audio, sr = librosa.load(audio_file_path, sr=None)\n\n#         plt.figure(figsize=(10, 4))\n\n#         # Plot the waveform\n#         plt.subplot(2, 1, 1)\n#         librosa.display.waveshow(audio, sr=sr)\n#         plt.title(\"Waveform\")\n\n#         # Plot the spectrogram\n#         plt.subplot(2, 1, 2)\n#         S = librosa.feature.melspectrogram(y=audio, sr=sr)\n#         S_dB = librosa.power_to_db(S, ref=np.max)\n#         librosa.display.specshow(S_dB, x_axis=\"time\", y_axis=\"mel\")\n#         plt.colorbar(format=\"%+2.0f dB\")\n#         plt.title(\"Spectrogram\")\n\n#         # Add the annotation text\n#         plt.text(0, 0, audio_filename, fontsize=12, color='white', va='center', ha='left', backgroundcolor='black')\n\n#         plt.tight_layout()\n#         plt.show()\n\n#     def visualize_sample_3d_interactive(self, idx):\n#         audio_filename = self.annotations_df.iloc[idx, 0]\n#         annotation = self.annotations_df.iloc[idx, 1]\n\n#         audio_file_path = os.path.join(self.audio_folder, audio_filename + \".mp3\")\n#         audio, sr = librosa.load(audio_file_path, sr=None)\n#         S = librosa.feature.melspectrogram(y=audio, sr=sr)\n#         S_dB = librosa.power_to_db(S, ref=np.max)\n\n#         t, f = np.meshgrid(np.linspace(0, librosa.get_duration(y=audio, sr=sr), S_dB.shape[1]),\n#                            np.linspace(0, sr/2, S_dB.shape[0]))\n\n#         # Create an interactive 3D surface plot using plotly\n#         fig = go.Figure(data=[go.Surface(z=S_dB, x=t, y=f)])\n#         fig.update_layout(title='Spectrogram (3D Interactive)',\n#                           scene=dict(xaxis_title='Time (s)',\n#                                      yaxis_title='Frequency (Hz)',\n#                                      zaxis_title='dB'))\n#         fig.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-27T10:02:02.246182Z","iopub.execute_input":"2023-07-27T10:02:02.246590Z","iopub.status.idle":"2023-07-27T10:02:02.273016Z","shell.execute_reply.started":"2023-07-27T10:02:02.246553Z","shell.execute_reply":"2023-07-27T10:02:02.271513Z"},"trusted":true},"execution_count":null,"outputs":[]}]}