{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://wallup.net/wp-content/uploads/2016/01/19116-Eastern_Imperial_Eagle-nature-animals-birds-eagle.jpg)\n\n# BirdCLEF 2022 : Model Building and Training\n---\n\nIn this notebook we will make the Neural Network model and train it on the data itself.\n\n**This is the second of the 3 notebooks which improvises from the findings of the first notebook and prepare the appropriate model and dataset and train the model on it.**","metadata":{"execution":{"iopub.status.busy":"2022-03-01T15:48:03.948509Z","iopub.execute_input":"2022-03-01T15:48:03.948839Z","iopub.status.idle":"2022-03-01T15:48:04.222983Z","shell.execute_reply.started":"2022-03-01T15:48:03.948792Z","shell.execute_reply":"2022-03-01T15:48:04.222071Z"}}},{"cell_type":"markdown","source":"# Installations","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:52.185606Z","iopub.execute_input":"2022-03-01T17:03:52.185871Z","iopub.status.idle":"2022-03-01T17:03:59.746734Z","shell.execute_reply.started":"2022-03-01T17:03:52.185841Z","shell.execute_reply":"2022-03-01T17:03:59.745747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"# General purpose libraries for loading and manipulating data\nimport os\nimport re\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\n\n\n# Pytorch imports for neural networks and tensor manipulations\nimport torch\nimport torchaudio\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.nn import CrossEntropyLoss\nfrom torchvision.transforms import Resize\nfrom torchaudio.transforms import MelSpectrogram\nfrom torch.utils.data import Dataset, DataLoader, random_split\n\n\n# Libraries for visualization\nimport torchsummary\nfrom termcolor import cprint\nimport matplotlib.pyplot as plt\n\n\n# Libraries to hide warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\n# ipywidgets\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.749407Z","iopub.execute_input":"2022-03-01T17:03:59.749718Z","iopub.status.idle":"2022-03-01T17:03:59.762059Z","shell.execute_reply.started":"2022-03-01T17:03:59.749673Z","shell.execute_reply":"2022-03-01T17:03:59.761272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datapaths","metadata":{}},{"cell_type":"code","source":"train_base_path = \"../input/birdclef-2022/train_metadata.csv\"\ntest_base_path = \"../input/birdclef-2022/test.csv\"\nsample_submission_base_path = \"../input/birdclef-2022/sample_submission.csv\"\nbird_taxonomy_base_path = \"../input/birdclef-2022/eBird_Taxonomy_v2021.csv\"\nlabels_base_path = \"../input/birdclef-2022/scored_birds.json\"\ntrain_dir = \"../input/birdclef-2022/train_audio\"\ntest_dir = \"../input/birdclef-2022/test_soundscapes\"","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.763280Z","iopub.execute_input":"2022-03-01T17:03:59.763967Z","iopub.status.idle":"2022-03-01T17:03:59.773201Z","shell.execute_reply.started":"2022-03-01T17:03:59.763930Z","shell.execute_reply":"2022-03-01T17:03:59.772469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading train metadata","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(train_base_path)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.774256Z","iopub.execute_input":"2022-03-01T17:03:59.775354Z","iopub.status.idle":"2022-03-01T17:03:59.853997Z","shell.execute_reply.started":"2022-03-01T17:03:59.775309Z","shell.execute_reply":"2022-03-01T17:03:59.853254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing\nNow we have to process training data so that it would be helpful for us.","metadata":{}},{"cell_type":"code","source":"imp_features = [\"primary_label\", \"type\", \"rating\", \"filename\"]\ntrain_df = train_df[imp_features]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.856542Z","iopub.execute_input":"2022-03-01T17:03:59.857259Z","iopub.status.idle":"2022-03-01T17:03:59.870520Z","shell.execute_reply.started":"2022-03-01T17:03:59.857218Z","shell.execute_reply":"2022-03-01T17:03:59.869710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this scenario we are only taking the calls which only resemble a proper call not some specific or unique call , cause those will destroy the patterns.","metadata":{}},{"cell_type":"code","source":"def extract_call(data, call = 'call'):\n    try:\n        if re.search(data, call):\n            return \"True\"\n        else:\n            return \"False\"\n    except:\n        return \"False\"","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.871957Z","iopub.execute_input":"2022-03-01T17:03:59.872325Z","iopub.status.idle":"2022-03-01T17:03:59.877599Z","shell.execute_reply.started":"2022-03-01T17:03:59.872289Z","shell.execute_reply":"2022-03-01T17:03:59.876900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Length of data before call extraction : {}\".format(len(train_df)))\ntrain_df[\"type\"] = train_df[\"type\"].apply(extract_call)\ntrain_df = train_df[train_df[\"type\"] == \"True\"]\ntrain_df.drop(\"type\", 1, inplace = True)\nprint(\"Length of data after call extraction : {}\".format(len(train_df)))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:03:59.878889Z","iopub.execute_input":"2022-03-01T17:03:59.879179Z","iopub.status.idle":"2022-03-01T17:04:00.062888Z","shell.execute_reply.started":"2022-03-01T17:03:59.879143Z","shell.execute_reply":"2022-03-01T17:04:00.062182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a class encoding dictionary which will help us find the correct class names in future.","metadata":{}},{"cell_type":"code","source":"class_dict = dict()\n\nfor index, label in enumerate(train_df.primary_label.unique()):\n    class_dict[index] = label\n    train_df[\"primary_label\"].replace(label, index, inplace = True)\nprint(class_dict)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.064122Z","iopub.execute_input":"2022-03-01T17:04:00.064711Z","iopub.status.idle":"2022-03-01T17:04:00.526860Z","shell.execute_reply.started":"2022-03-01T17:04:00.064672Z","shell.execute_reply":"2022-03-01T17:04:00.525983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving the object in a file so that we can use it on further cases.","metadata":{}},{"cell_type":"code","source":"json.dump(class_dict, open(\"class_dict.json\", \"w\"))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.528378Z","iopub.execute_input":"2022-03-01T17:04:00.528658Z","iopub.status.idle":"2022-03-01T17:04:00.534219Z","shell.execute_reply.started":"2022-03-01T17:04:00.528621Z","shell.execute_reply":"2022-03-01T17:04:00.533485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check the processed training metadata.","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.535955Z","iopub.execute_input":"2022-03-01T17:04:00.536485Z","iopub.status.idle":"2022-03-01T17:04:00.549298Z","shell.execute_reply.started":"2022-03-01T17:04:00.536448Z","shell.execute_reply":"2022-03-01T17:04:00.548472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similarly saving this for using in custom dataset and for future.","metadata":{}},{"cell_type":"code","source":"train_df.to_csv(\"training_metadata.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.550862Z","iopub.execute_input":"2022-03-01T17:04:00.551247Z","iopub.status.idle":"2022-03-01T17:04:00.591487Z","shell.execute_reply.started":"2022-03-01T17:04:00.551207Z","shell.execute_reply":"2022-03-01T17:04:00.590688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"markdown","source":"The first task is to fix the random seed i.e. we can replicate all the next scenarios. also setting the audio backend to lod the audio data into tensors.","metadata":{}},{"cell_type":"code","source":"\ntorch.manual_seed(42)\ntorchaudio.set_audio_backend(\"soundfile\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.592891Z","iopub.execute_input":"2022-03-01T17:04:00.593137Z","iopub.status.idle":"2022-03-01T17:04:00.598096Z","shell.execute_reply.started":"2022-03-01T17:04:00.593103Z","shell.execute_reply":"2022-03-01T17:04:00.597326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now , it's time to build our custom dataset which will take the data directory and the processed training metadata and create trainable data.","metadata":{}},{"cell_type":"code","source":"class CLEFDataset(Dataset):\n    \n    def __init__(self,\n                 data_dir,\n                 metadata_path,\n                 size = 640,\n                 transform = None):\n        super(CLEFDataset, self).__init__()\n        self.data_dir = data_dir\n        self.metadata = pd.read_csv(metadata_path)\n        self.size = size\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.metadata)\n    \n    def __getitem__(self, index):\n        path = self.metadata.loc[index, \"filename\"]\n        path = os.path.join(self.data_dir, path)\n        label = self.metadata.loc[index, \"primary_label\"]\n        mono_audio = self.load_audio(path)\n        mono_audio = mono_audio.unsqueeze(dim=0)\n        return mono_audio, label\n    \n    \n    def load_audio(self, path):\n        audio, _ = torchaudio.load(path)\n        if self.transform != None:\n            for aug in self.transform:\n                audio = aug(audio)\n        return audio[0,:]","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.599674Z","iopub.execute_input":"2022-03-01T17:04:00.600268Z","iopub.status.idle":"2022-03-01T17:04:00.610019Z","shell.execute_reply.started":"2022-03-01T17:04:00.600172Z","shell.execute_reply":"2022-03-01T17:04:00.609191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We also need a bit of data equivalence, so that training can be more specific.","metadata":{}},{"cell_type":"code","source":"augm = [\n    MelSpectrogram(n_mels = 128),\n    Resize((128, 128))\n]\naugm","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.614222Z","iopub.execute_input":"2022-03-01T17:04:00.614459Z","iopub.status.idle":"2022-03-01T17:04:00.623773Z","shell.execute_reply.started":"2022-03-01T17:04:00.614401Z","shell.execute_reply":"2022-03-01T17:04:00.622901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now creating the dataset","metadata":{}},{"cell_type":"code","source":"metadata_path = \"./training_metadata.csv\"\ndataset = CLEFDataset(train_dir, metadata_path, transform = augm)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.625159Z","iopub.execute_input":"2022-03-01T17:04:00.625596Z","iopub.status.idle":"2022-03-01T17:04:00.641711Z","shell.execute_reply.started":"2022-03-01T17:04:00.625550Z","shell.execute_reply":"2022-03-01T17:04:00.640977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dataset)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.643022Z","iopub.execute_input":"2022-03-01T17:04:00.643271Z","iopub.status.idle":"2022-03-01T17:04:00.649121Z","shell.execute_reply.started":"2022-03-01T17:04:00.643238Z","shell.execute_reply":"2022-03-01T17:04:00.648289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, label = dataset[10]\nprint(\"Audio Shape : {} , label : {}\".format(data.shape, label))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:20:58.593762Z","iopub.execute_input":"2022-03-01T17:20:58.594360Z","iopub.status.idle":"2022-03-01T17:20:58.683197Z","shell.execute_reply.started":"2022-03-01T17:20:58.594321Z","shell.execute_reply":"2022-03-01T17:20:58.682373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset is created correctly.\nNow we should split the dataset into training and validation sets.\n\n**Train-Validation Ratio = 4:1**","metadata":{}},{"cell_type":"code","source":"x1 = int(len(dataset) * 0.8)\nx2 = len(dataset) - x1\ntrain_ds, val_ds = random_split(dataset, [x1, x2])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.769992Z","iopub.execute_input":"2022-03-01T17:04:00.770250Z","iopub.status.idle":"2022-03-01T17:04:00.776105Z","shell.execute_reply.started":"2022-03-01T17:04:00.770214Z","shell.execute_reply":"2022-03-01T17:04:00.775052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Length of Training Dataset : {}\".format(len(train_ds)))\nprint(\"Length of Validation Dataset : {}\".format(len(val_ds)))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.777799Z","iopub.execute_input":"2022-03-01T17:04:00.778325Z","iopub.status.idle":"2022-03-01T17:04:00.789223Z","shell.execute_reply.started":"2022-03-01T17:04:00.778287Z","shell.execute_reply":"2022-03-01T17:04:00.788402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now it's time to create patch of data which will be a better way to train the model as it won't need too much space to load the whole data but patches of it. \nNote : The datasets are shuffled so that sparsity stays present.","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ntrain_dl = DataLoader(train_ds, batch_size = BATCH_SIZE, shuffle = True)\nval_dl = DataLoader(val_ds, batch_size = BATCH_SIZE, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.790548Z","iopub.execute_input":"2022-03-01T17:04:00.790740Z","iopub.status.idle":"2022-03-01T17:04:00.799087Z","shell.execute_reply.started":"2022-03-01T17:04:00.790715Z","shell.execute_reply":"2022-03-01T17:04:00.798276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check the data chunks","metadata":{}},{"cell_type":"code","source":"for patch, labels in train_dl:\n    print(patch.shape, labels.shape)\n    break\nfor patch, labels in val_dl:\n    print(patch.shape, labels.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:00.802230Z","iopub.execute_input":"2022-03-01T17:04:00.802556Z","iopub.status.idle":"2022-03-01T17:04:13.044745Z","shell.execute_reply.started":"2022-03-01T17:04:00.802465Z","shell.execute_reply":"2022-03-01T17:04:13.043979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Network Model","metadata":{}},{"cell_type":"markdown","source":"In here we have used several CNN and ANN layers just to be sure we do not leave any crucial data.","metadata":{}},{"cell_type":"code","source":"# Convolution shape updating function\ndef conv_shape(shape, kernel_size, stride, padding):\n    H, W = shape[0], shape[1]\n    H = ((H - kernel_size + 2*padding) // stride) + 1\n    W = ((W - kernel_size + 2*padding) // stride) + 1\n    return H, W","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.046102Z","iopub.execute_input":"2022-03-01T17:04:13.046518Z","iopub.status.idle":"2022-03-01T17:04:13.052604Z","shell.execute_reply.started":"2022-03-01T17:04:13.046479Z","shell.execute_reply":"2022-03-01T17:04:13.051677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Conv(nn.Module):\n    \n    def __init__(self, \n                   in_channels,\n                   out_channels,\n                   kernel_size,\n                   stride=(1,1),\n                   padding=(0,0),\n                   momentum=0.15):\n        super(Conv, self).__init__()\n        self.conv_block = nn.Sequential(\n            nn.BatchNorm2d(in_channels, momentum = momentum),\n            nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding),\n            nn.ReLU()\n        )\n        \n    def forward(self, x):\n        return self.conv_block(x)\n\n\nclass CLEFNetwork(nn.Module):\n    \n    def __init__(self,\n                 num_classes,\n                 in_channels = 1,\n                 H = 128,\n                 W = 128,\n                 num_downs = 3):\n        super(CLEFNetwork, self).__init__()\n        \n        self.num_C = num_classes\n        self.num_downs = num_downs\n        self.in_channels = in_channels\n        self.C = 8\n        self.H, self.W = self.calc_HW(H, W)\n        self.in_conv_block = Conv(self.in_channels, self.C, 7, (2, 2))\n        self.conv_block = nn.ModuleList(\n                [\n                    Conv(self.C * 2**i, self.C * 2**(i+1), 3, (2, 2))\n                    for i in range(self.num_downs-1)\n                ]\n        )\n        self.fc_block = nn.Sequential(\n                nn.Linear(self.H * self.W * self.C * 2**(self.num_downs - 1), 1024),\n                nn.Linear(1024, 1024),\n                nn.Linear(1024, self.num_C)\n        )\n        \n    def calc_HW(self, H, W):\n        H, W = conv_shape((H, W), 7, 2, 0)\n        for num_down in range(self.num_downs - 1):\n            H, W = conv_shape((H, W), 3, 2, 0)\n        return H, W\n        \n        \n    def forward(self, x):\n        x = self.in_conv_block(x)\n        for block in self.conv_block:\n            x = block(x)\n        x = x.view(x.shape[0], -1)\n        x = self.fc_block(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.054163Z","iopub.execute_input":"2022-03-01T17:04:13.054450Z","iopub.status.idle":"2022-03-01T17:04:13.069825Z","shell.execute_reply.started":"2022-03-01T17:04:13.054394Z","shell.execute_reply":"2022-03-01T17:04:13.069108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now loading the class label dictionary which contains the tital number of classes.","metadata":{}},{"cell_type":"code","source":"class_labels_path = \"./class_dict.json\"","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.071174Z","iopub.execute_input":"2022-03-01T17:04:13.071610Z","iopub.status.idle":"2022-03-01T17:04:13.081327Z","shell.execute_reply.started":"2022-03-01T17:04:13.071575Z","shell.execute_reply":"2022-03-01T17:04:13.080397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels = json.load(open(class_labels_path, \"r\"))\nnum_classes = len(class_labels.keys())\nprint(\"Number of class : {}\".format(num_classes))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.082881Z","iopub.execute_input":"2022-03-01T17:04:13.083486Z","iopub.status.idle":"2022-03-01T17:04:13.090770Z","shell.execute_reply.started":"2022-03-01T17:04:13.083378Z","shell.execute_reply":"2022-03-01T17:04:13.089816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try a simple forward pass with some random data on the model.","metadata":{}},{"cell_type":"code","source":"model = CLEFNetwork(num_classes)\nrand_data = torch.rand(5, 1, 128, 128)\nmodel(rand_data).shape","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.092210Z","iopub.execute_input":"2022-03-01T17:04:13.092557Z","iopub.status.idle":"2022-03-01T17:04:13.229079Z","shell.execute_reply.started":"2022-03-01T17:04:13.092515Z","shell.execute_reply":"2022-03-01T17:04:13.228354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before starting the training , let's check whether all the layers are passing through the model parameters, otherwise they won't be updated with the gradients on backtracking.","metadata":{}},{"cell_type":"code","source":"for name, param in model.named_parameters():\n  print(f\"{name} : {param.shape}, requires_grad : {param.requires_grad}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.230393Z","iopub.execute_input":"2022-03-01T17:04:13.230664Z","iopub.status.idle":"2022-03-01T17:04:13.238545Z","shell.execute_reply.started":"2022-03-01T17:04:13.230628Z","shell.execute_reply":"2022-03-01T17:04:13.237065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All looks fine. Let's visualize the model","metadata":{}},{"cell_type":"code","source":"\ntorchsummary.summary(model, (1, 128, 128), device = \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.240051Z","iopub.execute_input":"2022-03-01T17:04:13.240611Z","iopub.status.idle":"2022-03-01T17:04:13.264937Z","shell.execute_reply.started":"2022-03-01T17:04:13.240565Z","shell.execute_reply":"2022-03-01T17:04:13.264170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need an accuracy counting function for training purpose.","metadata":{}},{"cell_type":"code","source":"def accuracy_func(pred, true):\n    pred = torch.argmax(pred, dim = 1)\n    acc = sum(true == pred)\n    return acc","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.265999Z","iopub.execute_input":"2022-03-01T17:04:13.266200Z","iopub.status.idle":"2022-03-01T17:04:13.271383Z","shell.execute_reply.started":"2022-03-01T17:04:13.266176Z","shell.execute_reply":"2022-03-01T17:04:13.270671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model training and saving best models","metadata":{}},{"cell_type":"markdown","source":"The first task in these phase is to set the hyperparameters.\n\nAlso we need to check whether any distributive device (**GPU** , **TPU**) is present or not as it may be efficient for model training.","metadata":{}},{"cell_type":"code","source":"EPOCHS = 10\noptim = Adam(model.parameters(), lr = 1e-4)\ncriterion = CrossEntropyLoss()\ndevice = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:04:13.272955Z","iopub.execute_input":"2022-03-01T17:04:13.273369Z","iopub.status.idle":"2022-03-01T17:04:13.322245Z","shell.execute_reply.started":"2022-03-01T17:04:13.273336Z","shell.execute_reply":"2022-03-01T17:04:13.321352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training:\n\nNow it is the most important moment of the whole task.\n\nThe training loop will take the best model on the accuracy and loss metrics.","metadata":{}},{"cell_type":"code","source":"train_init = time.time()\ncprint(\"Started training...\", \"blue\")\nbest_loss = np.inf\nbest_acc = 0.0\nif device == \"cuda:0\":\n    print(\"Model Loaded on GPU...\")\n    model = model.cuda()\nupdate = 0\nTL, VL, TA, VA = [], [], [], []\nfor epoch in range(EPOCHS):\n    print(f\"Epoch {epoch + 1} :\")\n    epoch_init = time.time()\n    train_loss = val_loss = 0.0\n    train_acc = val_acc = 0\n    tot_val_data_point = 0\n    model.train()\n    for train_index, (patch, labels) in enumerate(train_dl):\n        optim.zero_grad()\n        if device == \"cuda:0\":\n            dev_patch = patch.cuda()\n            dev_labels = labels.cuda()\n        else:\n            dev_patch = patch\n            dev_labels = labels\n        output = model(dev_patch)\n        acc = accuracy_func(output, dev_labels)\n        train_acc += acc\n        loss = criterion(output, dev_labels)\n        train_loss += loss.item()\n        TL.append(loss.item())\n        TA.append(acc / dev_patch.shape[0])\n        if train_index % 10 == 9:\n            print(f\"      [Step {train_index + 1}] Loss : {'%.6f'%loss.item()}\")\n        loss.backward()\n        optim.step()\n        \n    model.eval()\n    with torch.no_grad():\n        for val_index, (patch, labels) in enumerate(val_dl):\n            if device == \"cuda:0\":\n                dev_patch = patch.cuda()\n                dev_labels = labels.cuda()\n            else:\n              dev_patch = patch\n              dev_labels = labels\n            output = model(dev_patch)\n            acc = accuracy_func(output, dev_labels)\n            val_acc += acc\n            loss = criterion(output, dev_labels)\n            val_loss += loss.item()\n            VL.append(loss.item())\n            VA.append(acc / dev_patch.shape[0])\n    TRAIN_ACC = train_acc / len(train_ds)\n    VAL_ACC = val_acc / len(val_ds)\n    print(f\"   Train Loss : {'%.6f'%train_loss} | Train accuracy : {'%.6f'%TRAIN_ACC}\")\n    print(f\"   Validation Loss : {'%.6f'%val_loss} | Validation Accuracy : {'%.6f'%VAL_ACC}\")\n    updation_flag = False\n    if val_loss < best_loss:\n      update = 0\n      updation_flag = True\n      best_loss = val_loss\n      cprint(\"Loss Updation : Positive\", \"green\")\n      torch.save({\n          \"model\" : model.state_dict(),\n          \"optim\" : optim.state_dict(),\n          \"epoch\" : epoch + 1\n      }, \"best_loss_model.pt\")\n    if VAL_ACC > best_acc:\n      update = 0\n      updation_flag = True\n      best_acc = VAL_ACC\n      cprint(\"Accuracy Updation : Positive\", \"green\")\n      torch.save({\n          \"model\" : model.state_dict(),\n          \"optim\" : optim.state_dict(),\n          \"epoch\" : epoch + 1\n      }, \"best_accuracy_model.pt\")\n    if updation_flag == False:\n        cprint(\"Model Updation : Negative\\n\", \"red\")\n        update += 1\n    print(f\"   Execution Time : {'%.3f'%(time.time() - epoch_init)} seconds\\n\")\n    if update >= 5:\n      cprint(\"Model Stopped due to continuous model learning degradation\\n\", \"red\")\n      break\ncprint(\"Training finished...\", \"blue\")\ncprint(f\"Exceution Time : {'%.3f'%(time.time() - train_init)} seconds\", \"blue\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:16:28.737181Z","iopub.execute_input":"2022-03-01T17:16:28.737656Z","iopub.status.idle":"2022-03-01T17:17:37.451137Z","shell.execute_reply.started":"2022-03-01T17:16:28.737617Z","shell.execute_reply":"2022-03-01T17:17:37.449566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization: \n\nNow that the training has been completed, we can plot the loss and accuracy curves to see the model performances.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (20, 6))\nplt.plot(TL, label = \"training loss\")\nplt.plot(VL, label = \"validation loss\")\nplt.xlabel(\"Steps\")\nplt.ylabel(\"Loss\")\nplt.title(\"Loss Curves\", size = 20)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:17:41.341851Z","iopub.execute_input":"2022-03-01T17:17:41.342596Z","iopub.status.idle":"2022-03-01T17:17:41.567572Z","shell.execute_reply.started":"2022-03-01T17:17:41.342554Z","shell.execute_reply":"2022-03-01T17:17:41.566822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The accuracies are stored as GPU tensors , so we have make them ordinary float variables for plotting.","metadata":{}},{"cell_type":"code","source":"for index in range(len(TA)):\n    TA[index] = float(TA[index].cpu().detach())\nfor index in range(len(VA)):\n    VA[index] = float(VA[index].cpu().detach())","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:17:45.443591Z","iopub.execute_input":"2022-03-01T17:17:45.444067Z","iopub.status.idle":"2022-03-01T17:17:45.450959Z","shell.execute_reply.started":"2022-03-01T17:17:45.444017Z","shell.execute_reply":"2022-03-01T17:17:45.450035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20, 6))\nplt.plot(TA, label = \"training accuracy score\")\nplt.plot(VA, label = \"validation accuracy score\")\nplt.xlabel(\"Steps\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"Accuracy Curves\", size = 20)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T17:17:48.477029Z","iopub.execute_input":"2022-03-01T17:17:48.477330Z","iopub.status.idle":"2022-03-01T17:17:48.715160Z","shell.execute_reply.started":"2022-03-01T17:17:48.477295Z","shell.execute_reply":"2022-03-01T17:17:48.714294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thanks for visiting :)\n\n# Do UPVOTE if you like the notebook :)\n## Also follow me on [kaggle](https://www.kaggle.com/sagnik1511) , [GitHub](https://github.com/sagnik1511) and on [LinkedIn](https://www.linkedin.com/in/sagnik1511)","metadata":{}}]}