{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torchsummaryX wandb --quiet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-23T14:57:45.045674Z","iopub.execute_input":"2023-09-23T14:57:45.045962Z","iopub.status.idle":"2023-09-23T14:57:59.820638Z","shell.execute_reply.started":"2023-09-23T14:57:45.045938Z","shell.execute_reply":"2023-09-23T14:57:59.81938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom torchsummaryX import summary\nimport sklearn\nimport gc\nimport zipfile\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport os\nimport datetime\nimport wandb\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(\"Device: \", device)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T14:57:59.823004Z","iopub.execute_input":"2023-09-23T14:57:59.823705Z","iopub.status.idle":"2023-09-23T14:58:04.250766Z","shell.execute_reply.started":"2023-09-23T14:57:59.823665Z","shell.execute_reply":"2023-09-23T14:58:04.24974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### PHONEME LIST\nPHONEMES = [\n            '[SIL]',   'AA',    'AE',    'AH',    'AO',    'AW',    'AY',\n            'B',     'CH',    'D',     'DH',    'EH',    'ER',    'EY',\n            'F',     'G',     'HH',    'IH',    'IY',    'JH',    'K',\n            'L',     'M',     'N',     'NG',    'OW',    'OY',    'P',\n            'R',     'S',     'SH',    'T',     'TH',    'UH',    'UW',\n            'V',     'W',     'Y',     'Z',     'ZH',    '[SOS]', '[EOS]']","metadata":{"execution":{"iopub.status.busy":"2023-09-23T14:58:04.252468Z","iopub.execute_input":"2023-09-23T14:58:04.252868Z","iopub.status.idle":"2023-09-23T14:58:04.25973Z","shell.execute_reply.started":"2023-09-23T14:58:04.252834Z","shell.execute_reply":"2023-09-23T14:58:04.258756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset class to load train and validation data\nclass AudioDataset(torch.utils.data.Dataset):\n\n    def __init__(self, root, phonemes = PHONEMES, context=0, partition= \"train-clean-100\"): # Feel free to add more arguments\n\n        self.context    = context\n        self.phonemes   = phonemes\n        # TODO: MFCC directory - use partition to acces train/dev directories from kaggle data using root\n        self.mfcc_dir       = root + '/' + partition + '/mfcc/'\n        # TODO: Transcripts directory - use partition to acces train/dev directories from kaggle data using root\n        self.transcript_dir = root + '/' + partition + '/transcript/'\n\n        # TODO: List files in sefl.mfcc_dir using os.listdir in sorted order\n        mfcc_names          = sorted(os.listdir(self.mfcc_dir))\n        # TODO: List files in self.transcript_dir using os.listdir in sorted order\n        transcript_names    = sorted(os.listdir(self.transcript_dir))\n\n        # Making sure that we have the same no. of mfcc and transcripts\n        total_timestamps = 0\n        assert len(mfcc_names) == len(transcript_names)\n        for mfcc in mfcc_names:\n            total_timestamps += len(np.load(self.mfcc_dir + mfcc))\n        self.length = total_timestamps\n        print(total_timestamps)\n        \n        print(\"HERE\")\n        self.mfccs, self.transcripts = np.zeros((2*context+total_timestamps, 28), dtype=np.float16), np.zeros((total_timestamps), dtype=np.uint8) \n        #self.mfccs, self.transcripts = [], []\n        # TODO: Iterate through mfccs and transcripts\n        current_index = context\n        for i in range(len(mfcc_names)):\n        #   Load a single mfcc\n            mfcc        = np.load(self.mfcc_dir + mfcc_names[i])\n            mfcc_mean = np.mean(mfcc, axis = 0)\n            mfcc_stddev =  np.std(mfcc, axis = 0)\n            cepstral_norm = (mfcc - mfcc_mean)/mfcc_stddev\n        #   Do Cepstral Normalization of mfcc (explained in writeup)\n        #   Load the corresponding transcript\n            transcript  = np.load(self.transcript_dir + transcript_names[i]) # Remove [SOS] and [EOS] from the transcript\n            transcript = transcript[1: -1]\n            # (Is there an efficient way to do this without traversing through the transcript?)\n            # Note that SOS will always be in the starting and EOS at end, as the name suggests.\n        #   Append each mfcc to self.mfcc, transcript to self.transcript\n            self.mfccs[current_index: current_index + len(cepstral_norm)] = cepstral_norm.astype(np.float16)\n            self.transcripts[current_index - context: current_index + len(transcript) - context] = np.array(list(map(lambda x : self.phonemes.index(x), transcript)))\n            current_index += len(cepstral_norm)\n            #self.mfccs.append(cepstral_norm)\n            #self.transcripts.append(transcript)\n        print(current_index)\n        # NOTE:\n        # Each mfcc is of shape T1 x 28, T2 x 28, ...\n        # Each transcript is of shape (T1+2), (T2+2),... before removing [SOS] and [EOS]\n\n        # TODO: Concatenate all mfccs in self.mfccs such that\n        # the final shape is T x 28 (Where T = T1 + T2 + ...)\n        #self.mfccs          = np.concatenate(self.mfccs, axis = 0)\n\n        # TODO: Concatenate all transcripts in self.transcripts such that\n        # the final shape is (T,) meaning, each time step has one phoneme output\n        #self.transcripts    = np.concatenate(self.transcripts, axis = 0)\n        # Hint: Use numpy to concatenate\n\n        # Length of the dataset is now the length of concatenated mfccs/transcripts\n\n        # Take some time to think about what we have done.\n        # self.mfcc is an array of the format (Frames x Features).\n        # Our goal is to recognize phonemes of each frame\n        # We can introduce context by padding zeros on top and bottom of self.mfcc\n        #self.mfccs = np.concatenate((np.zeros((context, 28)), self.mfccs, np.zeros((context, 28))), axis = 0) # TODO\n\n        # The available phonemes in the transcript are of string data type\n        # But the neural network cannot predict strings as such.\n        # Hence, we map these phonemes to integers\n\n        # TODO: Map the phonemes to their corresponding list indexes in self.phonemes\n        #def phoneme_to_index(p):\n        #  return self.phonemes.index(p)\n        #self.transcripts = np.array(list(map(lambda x : self.phonemes.index(x), self.transcripts)))\n        # Now, if an element in self.transcript is 0, it means that it is 'SIL' (as per the above example)\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, ind):\n\n        # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n        frames = self.mfccs[ind: ind + 2*self.context+1]\n        # After slicing, you get an array of shape 2*context+1 x 28. But our MLP needs 1d data and not 2d.\n        frames = frames.flatten() # TODO: Flatten to get 1d data\n\n        frames      = torch.FloatTensor(frames) # Convert to tensors\n        phonemes    = torch.tensor(self.transcripts[ind])\n\n        return frames, phonemes\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T14:58:04.263466Z","iopub.execute_input":"2023-09-23T14:58:04.263795Z","iopub.status.idle":"2023-09-23T14:58:04.280882Z","shell.execute_reply.started":"2023-09-23T14:58:04.26377Z","shell.execute_reply":"2023-09-23T14:58:04.279908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'epochs'        : 25,\n    'batch_size'    : 1024,\n    'context'       : 32,\n    'init_lr'       : 1e-3,\n    'architecture'  : 'medium',\n    'model'         : 'diamond'\n    # Add more as you need them - e.g dropout values, weight decay, scheduler parameters\n}","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:26:26.869797Z","iopub.execute_input":"2023-09-23T15:26:26.87064Z","iopub.status.idle":"2023-09-23T15:26:26.877034Z","shell.execute_reply.started":"2023-09-23T15:26:26.870614Z","shell.execute_reply":"2023-09-23T15:26:26.8761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AudioTestDataset(torch.utils.data.Dataset):\n    # TODO: Create a test dataset class similar to the previous class but you dont have transcripts for this\n    # Imp: Read the mfccs in sorted order, do NOT shuffle the data here or in your dataloader.\n    def __init__(self, root, phonemes = PHONEMES, context=0, partition= \"train-clean-100\"): # Feel free to add more arguments\n\n        self.context    = context\n        self.phonemes   = phonemes\n        # TODO: MFCC directory - use partition to acces train/dev directories from kaggle data using root\n        self.mfcc_dir       = root + '/' + partition + '/mfcc/'\n        # TODO: Transcripts directory - use partition to acces train/dev directories from kaggle data using root\n        #self.transcript_dir = root + '/' + partition + '/transcript/'\n\n        # TODO: List files in sefl.mfcc_dir using os.listdir in sorted order\n        mfcc_names          = sorted(os.listdir(self.mfcc_dir))\n        # TODO: List files in self.transcript_dir using os.listdir in sorted order\n       #transcript_names    = sorted(os.listdir(self.transcript_dir))\n\n        # Making sure that we have the same no. of mfcc and transcripts\n        total_timestamps = 0\n        #assert len(mfcc_names) == len(transcript_names)\n        for mfcc in mfcc_names:\n            total_timestamps += len(np.load(self.mfcc_dir + mfcc))\n        self.length = total_timestamps\n        print(total_timestamps)\n        \n        \n        self.mfccs = np.zeros((2*context+total_timestamps, 28), dtype=np.float16)\n        #self.mfccs, self.transcripts = [], []\n        # TODO: Iterate through mfccs and transcripts\n        current_index = context\n        for i in range(len(mfcc_names)):\n        #   Load a single mfcc\n            mfcc        = np.load(self.mfcc_dir + mfcc_names[i])\n            mfcc_mean = np.mean(mfcc, axis = 0)\n            mfcc_stddev =  np.std(mfcc, axis = 0)\n            cepstral_norm = (mfcc - mfcc_mean)/mfcc_stddev\n        #   Do Cepstral Normalization of mfcc (explained in writeup)\n        #   Load the corresponding transcript\n            #transcript  = np.load(self.transcript_dir + transcript_names[i]) # Remove [SOS] and [EOS] from the transcript\n            #transcript = transcript[1: -1]\n            # (Is there an efficient way to do this without traversing through the transcript?)\n            # Note that SOS will always be in the starting and EOS at end, as the name suggests.\n        #   Append each mfcc to self.mfcc, transcript to self.transcript\n            self.mfccs[current_index: current_index + len(cepstral_norm)] = cepstral_norm.astype(np.float16)\n            #self.transcripts[current_index - context: current_index + len(transcript) - context] = np.array(list(map(lambda x : self.phonemes.index(x), transcript)))\n            current_index += len(cepstral_norm)\n            #self.mfccs.append(cepstral_norm)\n            #self.transcripts.append(transcript)\n        print(current_index)\n        # NOTE:\n        # Each mfcc is of shape T1 x 28, T2 x 28, ...\n        # Each transcript is of shape (T1+2), (T2+2),... before removing [SOS] and [EOS]\n\n        # TODO: Concatenate all mfccs in self.mfccs such that\n        # the final shape is T x 28 (Where T = T1 + T2 + ...)\n        #self.mfccs          = np.concatenate(self.mfccs, axis = 0)\n\n        # TODO: Concatenate all transcripts in self.transcripts such that\n        # the final shape is (T,) meaning, each time step has one phoneme output\n        #self.transcripts    = np.concatenate(self.transcripts, axis = 0)\n        # Hint: Use numpy to concatenate\n\n        # Length of the dataset is now the length of concatenated mfccs/transcripts\n\n        # Take some time to think about what we have done.\n        # self.mfcc is an array of the format (Frames x Features).\n        # Our goal is to recognize phonemes of each frame\n        # We can introduce context by padding zeros on top and bottom of self.mfcc\n        #self.mfccs = np.concatenate((np.zeros((context, 28)), self.mfccs, np.zeros((context, 28))), axis = 0) # TODO\n\n        # The available phonemes in the transcript are of string data type\n        # But the neural network cannot predict strings as such.\n        # Hence, we map these phonemes to integers\n\n        # TODO: Map the phonemes to their corresponding list indexes in self.phonemes\n        #def phoneme_to_index(p):\n        #  return self.phonemes.index(p)\n        #self.transcripts = np.array(list(map(lambda x : self.phonemes.index(x), self.transcripts)))\n        # Now, if an element in self.transcript is 0, it means that it is 'SIL' (as per the above example)\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, ind):\n\n        # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n        frames = self.mfccs[ind: ind + 2*self.context+1]\n        # After slicing, you get an array of shape 2*context+1 x 28. But our MLP needs 1d data and not 2d.\n        frames = frames.flatten() # TODO: Flatten to get 1d data\n\n        frames      = torch.FloatTensor(frames) # Convert to tensors\n        \n\n        return frames\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:26:32.960117Z","iopub.execute_input":"2023-09-23T15:26:32.960634Z","iopub.status.idle":"2023-09-23T15:26:32.977263Z","shell.execute_reply.started":"2023-09-23T15:26:32.960603Z","shell.execute_reply":"2023-09-23T15:26:32.97629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TODO: Create a dataset object using the AudioDataset class for the training data\ntrain_data = AudioDataset(\"/kaggle/input/11785-hw1p2-f23/11-785-f23-hw1p2\", context = config['context'])\n\n# TODO: Create a dataset object using the AudioDataset class for the validation data\nval_data = AudioDataset(\"/kaggle/input/11785-hw1p2-f23/11-785-f23-hw1p2\", context = config['context'], partition = \"dev-clean\")\n\n# TODO: Create a dataset object using the AudioTestDataset class for the test data\ntest_data = AudioTestDataset(\"/kaggle/input/11785-hw1p2-f23/11-785-f23-hw1p2\", context = config['context'])","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:26:37.066928Z","iopub.execute_input":"2023-09-23T15:26:37.067641Z","iopub.status.idle":"2023-09-23T15:32:10.033815Z","shell.execute_reply.started":"2023-09-23T15:26:37.067608Z","shell.execute_reply":"2023-09-23T15:32:10.032762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define dataloaders for train, val and test datasets\n# Dataloaders will yield a batch of frames and phonemes of given batch_size at every iteration\n# We shuffle train dataloader but not val & test dataloader. Why?\n\ntrain_loader = torch.utils.data.DataLoader(\n    dataset     = train_data,\n    num_workers = 4,\n    batch_size  = config['batch_size'],\n    pin_memory  = True,\n    shuffle     = True\n)\n\nval_loader = torch.utils.data.DataLoader(\n    dataset     = val_data,\n    num_workers = 2,\n    batch_size  = config['batch_size'],\n    pin_memory  = True,\n    shuffle     = False\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    dataset     = test_data,\n    num_workers = 2,\n    batch_size  = config['batch_size'],\n    pin_memory  = True,\n    shuffle     = False\n)\n\n\nprint(\"Batch size     : \", config['batch_size'])\nprint(\"Context        : \", config['context'])\nprint(\"Input size     : \", (2*config['context']+1)*28)\nprint(\"Output symbols : \", len(PHONEMES)-2)\n\nprint(\"Train dataset samples = {}, batches = {}\".format(train_data.__len__(), len(train_loader)))\nprint(\"Validation dataset samples = {}, batches = {}\".format(val_data.__len__(), len(val_loader)))\nprint(\"Test dataset samples = {}, batches = {}\".format(test_data.__len__(), len(test_loader)))","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:10.036049Z","iopub.execute_input":"2023-09-23T15:32:10.036428Z","iopub.status.idle":"2023-09-23T15:32:10.052915Z","shell.execute_reply.started":"2023-09-23T15:32:10.036391Z","shell.execute_reply":"2023-09-23T15:32:10.051412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing code to check if your data loaders are working\n\nfor i, data in enumerate(train_loader):\n    frames, phoneme = data\n    print(frames.shape, phoneme.shape)\n    \n    break","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:10.054849Z","iopub.execute_input":"2023-09-23T15:32:10.055306Z","iopub.status.idle":"2023-09-23T15:32:20.448458Z","shell.execute_reply.started":"2023-09-23T15:32:10.055272Z","shell.execute_reply":"2023-09-23T15:32:20.445504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This architecture will make you cross the very low cutoff\n# However, you need to run a lot of experiments to cross the medium or high cutoff\n#1024,2048,3072,2048,2048,1024,1024,out\nclass Network(torch.nn.Module):\n\n    def __init__(self, input_size, output_size):\n\n        super(Network, self).__init__()\n\n        self.model = torch.nn.Sequential(\n            \n            torch.nn.Linear(input_size, 1024),\n            torch.nn.BatchNorm1d(1024),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(1024,2048),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(2048, 3072),\n            torch.nn.BatchNorm1d(3072),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(3072, 2048),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(2048, 2048),\n            torch.nn.BatchNorm1d(2048),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(2048, 1024),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n           \n            torch.nn.Linear(1024, 1024),\n            torch.nn.BatchNorm1d(2048),\n            torch.nn.GELU(),\n            torch.nn.Dropout(p=0.2),\n            \n            torch.nn.Linear(1024, output_size)\n        )\n\n    def forward(self, x):\n        out = self.model(x)\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:20.459984Z","iopub.execute_input":"2023-09-23T15:32:20.463874Z","iopub.status.idle":"2023-09-23T15:32:20.497184Z","shell.execute_reply.started":"2023-09-23T15:32:20.463827Z","shell.execute_reply":"2023-09-23T15:32:20.494624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_SIZE  = (2*config['context'] + 1) * 28 # Why is this the case?\nmodel       = Network(INPUT_SIZE, 40).to(device)\n#summary(model, frames.to(device))\n# Check number of parameters of your network\n# Remember, you are limited to 25 million parameters for HW1 (including ensembles)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:20.50601Z","iopub.execute_input":"2023-09-23T15:32:20.510707Z","iopub.status.idle":"2023-09-23T15:32:20.941041Z","shell.execute_reply.started":"2023-09-23T15:32:20.510625Z","shell.execute_reply":"2023-09-23T15:32:20.939998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = torch.nn.CrossEntropyLoss() # Defining Loss function.\n# We use CE because the task is multi-class classification\n\noptimizer = torch.optim.Adam(model.parameters(), lr= config['init_lr']) #Defining Optimizer\n# Recommended : Define Scheduler for Learning Rate,\n# including but not limited to StepLR, MultiStepLR, CosineAnnealingLR, ReduceLROnPlateau, etc.\n# You can refer to Pytorch documentation for more information on how to use them.\n\n# Is your training time very high?\n# Look into mixed precision training if your GPU (Tesla T4, V100, etc) can make use of it\n# Refer - https://pytorch.org/docs/stable/notes/amp_examples.html\nscheduler = ReduceLROnPlateau(optimizer, factor = 0.5, patience = 0)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:33.058073Z","iopub.execute_input":"2023-09-23T15:32:33.05846Z","iopub.status.idle":"2023-09-23T15:32:33.065248Z","shell.execute_reply.started":"2023-09-23T15:32:33.058429Z","shell.execute_reply":"2023-09-23T15:32:33.064204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:35.829036Z","iopub.execute_input":"2023-09-23T15:32:35.829863Z","iopub.status.idle":"2023-09-23T15:32:36.081899Z","shell.execute_reply.started":"2023-09-23T15:32:35.82983Z","shell.execute_reply":"2023-09-23T15:32:36.080956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, dataloader, optimizer, criterion):\n\n    model.train()\n    tloss, tacc = 0, 0 # Monitoring loss and accuracy\n    batch_bar   = tqdm(total=len(train_loader), dynamic_ncols=True, leave=False, position=0, desc='Train')\n\n    for i, (frames, phonemes) in enumerate(dataloader):\n\n        ### Initialize Gradients\n        optimizer.zero_grad()\n\n        ### Move Data to Device (Ideally GPU)\n        frames      = frames.to(device)\n        phonemes    = phonemes.to(device)\n\n        ### Forward Propagation\n        logits  = model(frames)\n\n        ### Loss Calculation\n        loss    = criterion(logits, phonemes)\n\n        ### Backward Propagation\n        loss.backward()\n\n        ### Gradient Descent\n        optimizer.step()\n\n        tloss   += loss.item()\n        tacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n\n        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(tloss / (i + 1))),\n                              acc=\"{:.04f}%\".format(float(tacc*100 / (i + 1))))\n        batch_bar.update()\n\n        ### Release memory\n        del frames, phonemes, logits\n        torch.cuda.empty_cache()\n\n    batch_bar.close()\n    tloss   /= len(train_loader)\n    tacc    /= len(train_loader)\n\n    return tloss, tacc","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:37.321607Z","iopub.execute_input":"2023-09-23T15:32:37.32197Z","iopub.status.idle":"2023-09-23T15:32:37.333555Z","shell.execute_reply.started":"2023-09-23T15:32:37.321941Z","shell.execute_reply":"2023-09-23T15:32:37.332218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval(model, dataloader):\n\n    model.eval() # set model in evaluation mode\n    vloss, vacc = 0, 0 # Monitoring loss and accuracy\n    batch_bar   = tqdm(total=len(val_loader), dynamic_ncols=True, position=0, leave=False, desc='Val')\n\n    for i, (frames, phonemes) in enumerate(dataloader):\n\n        ### Move data to device (ideally GPU)\n        frames      = frames.to(device)\n        phonemes    = phonemes.to(device)\n\n        # makes sure that there are no gradients computed as we are not training the model now\n        with torch.inference_mode():\n            ### Forward Propagation\n            logits  = model(frames)\n            ### Loss Calculation\n            loss    = criterion(logits, phonemes)\n\n        vloss   += loss.item()\n        vacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n\n        # Do you think we need loss.backward() and optimizer.step() here?\n\n        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(vloss / (i + 1))),\n                              acc=\"{:.04f}%\".format(float(vacc*100 / (i + 1))))\n        batch_bar.update()\n\n        ### Release memory\n        del frames, phonemes, logits\n        torch.cuda.empty_cache()\n\n    batch_bar.close()\n    vloss   /= len(val_loader)\n    vacc    /= len(val_loader)\n\n    return vloss, vacc","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:39.696858Z","iopub.execute_input":"2023-09-23T15:32:39.697924Z","iopub.status.idle":"2023-09-23T15:32:39.70925Z","shell.execute_reply.started":"2023-09-23T15:32:39.697883Z","shell.execute_reply":"2023-09-23T15:32:39.708301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.login(key=\"b78f5a0a316b8f09525ad5c671b527f4b95c1695\")\nrun = wandb.init(\n    name    = \"final\", ### Wandb creates random run names if you skip this field, we recommend you give useful names\n    reinit  = True, ### Allows reinitalizing runs when you re-run this cell\n    #id     = \"y28t31uz\", ### Insert specific run id here if you want to resume a previous run\n    #resume = \"must\", ### You need this to resume previous runs, but comment out reinit = True when using this\n    project = \"hw1p2\", ### Project should be created in your wandb account\n    config  = config ### Wandb Config for your run\n)\nmodel_arch  = str(model)\n\n### Save it in a txt file\narch_file   = open(\"diamond.txt\", \"w\")\nfile_write  = arch_file.write(model_arch)\narch_file.close()\n\n### log it in your wandb run with wandb.save()\nwandb.save('diamond.txt')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterate over number of epochs to train and evaluate your model\ntorch.cuda.empty_cache()\nwandb.watch(model, log=\"all\")\ngc.collect()\nbest_acc = 0\nfor epoch in range(config['epochs']):\n\n    print(\"\\nEpoch {}/{}\".format(epoch+1, config['epochs']))\n\n    curr_lr                 = float(optimizer.param_groups[0]['lr'])\n    train_loss, train_acc   = train(model, train_loader, optimizer, criterion)\n    val_loss, val_acc       = eval(model, val_loader)\n    scheduler.step(val_loss)\n    print(\"\\tTrain Acc {:.04f}%\\tTrain Loss {:.04f}\\t Learning Rate {:.07f}\".format(train_acc*100, train_loss, curr_lr))\n    print(\"\\tVal Acc {:.04f}%\\tVal Loss {:.04f}\".format(val_acc*100, val_loss))\n    wandb.log({'train_acc': train_acc*100, 'train_loss': train_loss,\n               'val_acc': val_acc*100, 'valid_loss': val_loss, 'lr': curr_lr})\n    if train_acc > best_acc:\n        best_acc = train_acc\n\n        # Saving the model and optimizer states\n\n        torch.save({\n              'model_state_dict': model.state_dict(),\n              'optimizer_state_dict': optimizer.state_dict()\n              }, \"Model\")\n\n        # Creating Artifact\n\n        model_artifact = wandb.Artifact(config['model'], type='model')\n\n        # Adding model file to Artifact\n\n        model_artifact.add_file(\"Model\")\n\n        # Saving Artifact to WandB\n\n        run.log_artifact(model_artifact)\n### Finish your wandb run\nrun.finish()   \n    ### Log metrics at each epoch in your run\n    # Optionally, you can log at each batch inside train/eval functions\n    # (explore wandb documentation/wandb recitation)\n    ### Highly Recommended: Save checkpoint in drive and/or wandb if accuracy is better than your current best\n\n### Finish your wandb run\n#run.finish()","metadata":{"execution":{"iopub.status.busy":"2023-09-23T15:32:50.478411Z","iopub.execute_input":"2023-09-23T15:32:50.47877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef test(model, test_loader):\n    ### What you call for model to perform inference?\n    model.eval() # TODO train or eval?\n\n    ### List to store predicted phonemes of test data\n    test_predictions = []\n\n    ### Which mode do you need to avoid gradients?\n    with torch.inference_mode(): # TODO\n\n        for i, mfccs in enumerate(tqdm(test_loader)):\n\n            mfccs   = mfccs.to(device)\n\n            logits  = model(mfccs)\n\n            ### Get most likely predicted phoneme with argmax\n            predicted_phonemes = [PHONEMES[i] for i in torch.argmax(logits,dim=1)]\n\n            ### How do you store predicted_phonemes with test_predictions? Hint, look at eval\n            # TODO\n            test_predictions.extend(predicted_phonemes)\n\n    return test_predictions","metadata":{"execution":{"iopub.status.busy":"2023-09-19T19:54:55.514802Z","iopub.execute_input":"2023-09-19T19:54:55.51577Z","iopub.status.idle":"2023-09-19T19:55:04.446062Z","shell.execute_reply.started":"2023-09-19T19:54:55.515718Z","shell.execute_reply":"2023-09-19T19:55:04.444746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = test(model, test_loader)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-19T19:54:17.900773Z","iopub.execute_input":"2023-09-19T19:54:17.901188Z","iopub.status.idle":"2023-09-19T19:54:31.127649Z","shell.execute_reply.started":"2023-09-19T19:54:17.901157Z","shell.execute_reply":"2023-09-19T19:54:31.12556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"./submission.csv\", \"w+\") as f:\n    f.write(\"id,label\\n\")\n    for i in range(len(predictions)):\n        f.write(\"{},{}\\n\".format(i, predictions[i]))","metadata":{"execution":{"iopub.status.busy":"2023-09-15T03:47:29.671884Z","iopub.execute_input":"2023-09-15T03:47:29.672468Z","iopub.status.idle":"2023-09-15T03:47:32.467516Z","shell.execute_reply.started":"2023-09-15T03:47:29.672421Z","shell.execute_reply":"2023-09-15T03:47:32.465421Z"},"trusted":true},"execution_count":null,"outputs":[]}]}