{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Prerequisite \n1. Kaggle dataset in ./data folder","metadata":{"id":"11df6315"}},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nimport pandas as pd\nimport torchaudio\nimport time\nimport copy\nimport torch.nn as nn\nfrom sklearn.preprocessing import LabelEncoder\nimport matplotlib.pyplot as plt","metadata":{"id":"e8de2d14","execution":{"iopub.status.busy":"2022-04-04T02:03:52.703063Z","iopub.execute_input":"2022-04-04T02:03:52.703976Z","iopub.status.idle":"2022-04-04T02:03:52.710294Z","shell.execute_reply.started":"2022-04-04T02:03:52.703935Z","shell.execute_reply":"2022-04-04T02:03:52.708522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-04-04T02:03:52.738642Z","iopub.execute_input":"2022-04-04T02:03:52.738835Z","iopub.status.idle":"2022-04-04T02:03:52.943449Z","shell.execute_reply.started":"2022-04-04T02:03:52.738811Z","shell.execute_reply":"2022-04-04T02:03:52.942684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dataset","metadata":{"id":"083f67bc"}},{"cell_type":"code","source":"class BirdClefDataset(Dataset):\n    def __init__(self, file, audio_dir, transformation, target_sample_rate, duration, device):\n        self.df = pd.read_csv(file)\n        self.df['primary_label_encoded'] = LabelEncoder().fit_transform(self.df['primary_label'])\n        self.audio_dir = audio_dir\n        self.device = device\n        self.transformation = transformation.to(self.device)\n        self.target_sample_rate = target_sample_rate \n        self.num_samples = target_sample_rate * duration\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        audio_path = f'/kaggle/input/birdclef-2022/{self.audio_dir}/{self.df.iloc[index, 12]}'\n        label = self.df.iloc[index, 13]\n        signal, sr = torchaudio.load(audio_path)\n        signal = signal.to(self.device)\n        \n        if sr != self.target_sample_rate:\n            resampler = torchaudio.transforms.Resample(sr, self.target_sample_rate)\n            signal = resampler(signal)\n        \n        if signal.shape[0]>1:\n            signal = torch.mean(signal, dim=0, keepdim=True)\n        \n        if signal.shape[1] > self.num_samples:\n            signal = signal[:, :self.num_samples]\n        \n        if signal.shape[1] < self.num_samples:\n            num_missing_samples = self.num_samples - signal.shape[1]\n            last_dim_padding = (0, num_missing_samples)\n            signal = torch.nn.functional.pad(signal, last_dim_padding)\n        \n        signal = self.transformation(signal)\n        return signal, label\n  ","metadata":{"id":"a42d6eaf","execution":{"iopub.status.busy":"2022-04-04T02:03:52.945842Z","iopub.execute_input":"2022-04-04T02:03:52.946119Z","iopub.status.idle":"2022-04-04T02:03:52.959729Z","shell.execute_reply.started":"2022-04-04T02:03:52.946082Z","shell.execute_reply":"2022-04-04T02:03:52.95898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model definition","metadata":{"id":"e544ab0b"}},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(\n                in_channels=1,\n                out_channels=16,\n                kernel_size=3,\n                stride=1,\n                padding=2 \n            ),\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2)\n        )\n    \n        self.conv2 = nn.Sequential(\n            nn.Conv2d(\n                in_channels=16,\n                out_channels=32,\n                kernel_size=3,\n                stride=1,\n                padding=2 \n            ),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2)\n        )\n        \n        self.conv3 = nn.Sequential(\n            nn.Conv2d(\n                in_channels=32,\n                out_channels=64,\n                kernel_size=3,\n                stride=1,\n                padding=2 \n            ),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2)\n        )\n        \n        self.conv3 = nn.Sequential(\n            nn.Conv2d(\n                in_channels=32,\n                out_channels=64,\n                kernel_size=3,\n                stride=1,\n                padding=2 \n            ),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2)\n        )\n        \n        self.conv4 = nn.Sequential(\n            nn.Conv2d(\n                in_channels=64,\n                out_channels=128,\n                kernel_size=3,\n                stride=1,\n                padding=2 \n            ),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2)\n        )\n        self.flatten = nn.Flatten()\n        self.linear = nn.Linear(18560, 152)\n        # self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, input_data):\n        x = self.conv1(input_data)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.flatten(x)\n        logits = self.linear(x)\n        return logits","metadata":{"id":"b31e1b27","execution":{"iopub.status.busy":"2022-04-04T02:03:52.960913Z","iopub.execute_input":"2022-04-04T02:03:52.961692Z","iopub.status.idle":"2022-04-04T02:03:52.997979Z","shell.execute_reply.started":"2022-04-04T02:03:52.961654Z","shell.execute_reply":"2022-04-04T02:03:52.997262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricAverage(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"id":"97dafd1f","execution":{"iopub.status.busy":"2022-04-04T02:03:53.000492Z","iopub.execute_input":"2022-04-04T02:03:53.000897Z","iopub.status.idle":"2022-04-04T02:03:53.011591Z","shell.execute_reply.started":"2022-04-04T02:03:53.000863Z","shell.execute_reply":"2022-04-04T02:03:53.010749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy(output, target):\n    batch_size = target.shape[0]\n    _, pred = torch.max(output, dim=-1)\n    correct = pred.eq(target).sum() * 1.0\n    acc = correct.item() / batch_size\n    return acc","metadata":{"id":"e1fbcdf0","execution":{"iopub.status.busy":"2022-04-04T02:03:53.01302Z","iopub.execute_input":"2022-04-04T02:03:53.013327Z","iopub.status.idle":"2022-04-04T02:03:53.023466Z","shell.execute_reply.started":"2022-04-04T02:03:53.013292Z","shell.execute_reply":"2022-04-04T02:03:53.022746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train step","metadata":{"id":"b407eb0e"}},{"cell_type":"code","source":"def train(model, data_loader, criterion, optimizer, epoch, device):\n    model.train()\n    iter_time = MetricAverage()\n    losses = MetricAverage()\n    acc = MetricAverage()\n    for idx, (data, target) in enumerate(data_loader):\n        start = time.time()\n        if torch.cuda.is_available():\n            data = data.to(device)\n            target = target.type(torch.LongTensor)\n            target = target.to(device)\n            \n        out = model.forward(data)\n        loss = criterion.forward(out, target)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        batch_acc = accuracy(out, target)\n\n        losses.update(loss, out.shape[0])\n        acc.update(batch_acc, out.shape[0])\n\n        iter_time.update(time.time() - start)\n        if idx % 50 == 0:\n            print(('TRAIN: Epoch: [{0}][{1}/{2}]\\t'\n                   'Time {iter_time.val:.3f} ({iter_time.avg:.3f})\\t'\n                   'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n                   'Prec @1 {top1.val:.4f} ({top1.avg:.4f})\\t')\n                   .format(epoch, idx, len(data_loader), iter_time=iter_time, loss=losses, top1=acc))\n    return losses.avg, acc.avg            ","metadata":{"id":"ab669c34","execution":{"iopub.status.busy":"2022-04-04T02:03:53.024845Z","iopub.execute_input":"2022-04-04T02:03:53.025095Z","iopub.status.idle":"2022-04-04T02:03:53.036795Z","shell.execute_reply.started":"2022-04-04T02:03:53.025061Z","shell.execute_reply":"2022-04-04T02:03:53.035883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(model, val_loader, criterion, epoch, device):\n    model.eval()\n    iter_time = MetricAverage()\n    losses = MetricAverage()\n    acc = MetricAverage()\n\n    num_class = 152\n    cm =torch.zeros(num_class, num_class)\n    for idx, (data, target) in enumerate(val_loader):\n        start = time.time()\n\n        if torch.cuda.is_available():\n            data = data.to(device)\n            target = target.type(torch.LongTensor)\n            target = target.to(device)\n        with torch.no_grad():\n            out = model.forward(data)\n            loss = criterion.forward(out, target)\n        \n        batch_acc = accuracy(out, target)\n\n        _, preds = torch.max(out, 1)\n        for t, p in zip(target.view(-1), preds.view(-1)):\n            cm[t.long(), p.long()] += 1\n\n        losses.update(loss, out.shape[0])\n        acc.update(batch_acc, out.shape[0])\n\n        iter_time.update(time.time() - start)\n        if idx % 50 == 0:\n            print(('VALIDATE:  Epoch: [{0}][{1}/{2}]\\t'\n                'Time {iter_time.val:.3f} ({iter_time.avg:.3f})\\t')\n                .format(epoch, idx, len(val_loader), iter_time=iter_time, loss=losses, top1=acc))\n    cm = cm / cm.sum(1)\n    per_cls_acc = cm.diag().detach().cpu().numpy().tolist()\n    for i, acc_i in enumerate(per_cls_acc):\n        print(\"Accuracy of Class {}: {:.4f}\".format(i, acc_i))\n        print(\"* Prec @1: {top1.avg:.4f}\".format(top1=acc))\n    return losses.avg,acc.avg, cm","metadata":{"id":"a6a9577f","execution":{"iopub.status.busy":"2022-04-04T02:03:53.038721Z","iopub.execute_input":"2022-04-04T02:03:53.039032Z","iopub.status.idle":"2022-04-04T02:03:53.049473Z","shell.execute_reply.started":"2022-04-04T02:03:53.039002Z","shell.execute_reply":"2022-04-04T02:03:53.04862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_learning_rate(optimizer, epoch, steps, learning_rate):\n    epoch += 1\n    if epoch > steps[1]:\n        lr = learning_rate * 0.01\n    elif epoch > steps[0]:\n        lr = learning_rate * 0.1\n    else:\n        lr = learning_rate\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = lr","metadata":{"id":"abcca8d1","execution":{"iopub.status.busy":"2022-04-04T02:03:53.050865Z","iopub.execute_input":"2022-04-04T02:03:53.051247Z","iopub.status.idle":"2022-04-04T02:03:53.059841Z","shell.execute_reply.started":"2022-04-04T02:03:53.051186Z","shell.execute_reply":"2022-04-04T02:03:53.059129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    seed=117\n    num_fold = 5\n    sample_rate= 32000\n    n_fft=1024\n    hop_length=512\n    n_mels=64\n    duration=7\n    batch_size= 32\n    learning_rate= 0.01\n    reg= 0.0005\n    epochs= 10\n    steps= [6,8]\n    momentum= 0.9\n    file= \"/kaggle/input/birdclef-2022/train_metadata.csv\"\n    audio_dir= \"train_audio\"\n    save_best=True","metadata":{"id":"df505ae9","execution":{"iopub.status.busy":"2022-04-04T02:03:53.06385Z","iopub.execute_input":"2022-04-04T02:03:53.064454Z","iopub.status.idle":"2022-04-04T02:03:53.070687Z","shell.execute_reply.started":"2022-04-04T02:03:53.064417Z","shell.execute_reply":"2022-04-04T02:03:53.069846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-04-04T02:03:53.11212Z","iopub.execute_input":"2022-04-04T02:03:53.112609Z","iopub.status.idle":"2022-04-04T02:03:53.850631Z","shell.execute_reply.started":"2022-04-04T02:03:53.112581Z","shell.execute_reply":"2022-04-04T02:03:53.84974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = \"cuda\"\nelse:\n    device = \"cpu\"\n\nmel_spectrogram = torchaudio.transforms.MelSpectrogram(\n        sample_rate=config.sample_rate,\n        n_fft=config.n_fft,\n        hop_length=config.hop_length,\n        n_mels=config.n_mels\n    )\n\ndataset = BirdClefDataset(config.file, config.audio_dir, mel_spectrogram, config.sample_rate, config.duration, device)","metadata":{"id":"1783ae88","execution":{"iopub.status.busy":"2022-04-04T02:03:53.85306Z","iopub.execute_input":"2022-04-04T02:03:53.853614Z","iopub.status.idle":"2022-04-04T02:03:53.937932Z","shell.execute_reply.started":"2022-04-04T02:03:53.853576Z","shell.execute_reply":"2022-04-04T02:03:53.937225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_history(train_loss_history,validation_loss_history) -> None:\n    \"\"\"Plots the loss history\"\"\"\n    plt.figure()\n    epoch_idxs = range(len(train_loss_history))\n\n    plt.plot(epoch_idxs, train_loss_history, \"-b\", label=\"training\")\n    plt.plot(epoch_idxs, validation_loss_history, \"-r\", label=\"validation\")\n    plt.title(\"Loss history\")\n    plt.legend()\n    plt.ylabel(\"Loss\")\n    plt.xlabel(\"Epochs\")\n    plt.show()\n\ndef plot_accuracy(train_accuracy_history,validation_accuracy_history) -> None:\n    \"\"\"Plots the accuracy history\"\"\"\n    plt.figure()\n    epoch_idxs = range(len(train_accuracy_history))\n    plt.plot(epoch_idxs, train_accuracy_history, \"-b\", label=\"training\")\n    plt.plot(epoch_idxs, validation_accuracy_history, \"-r\", label=\"validation\")\n    plt.title(\"Accuracy history\")\n    plt.legend()\n    plt.ylabel(\"Accuracy\")\n    plt.xlabel(\"Epochs\")\n    plt.show()\n\ndef main():\n    learning_rate = 0.0001\n    model = CNN()\n    train_size = int(0.8 * len(dataset))\n    val_size = len(dataset) - train_size\n    train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n    print(\"train_size\",train_size)\n    print(\"val_size\",val_size)\n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=100, shuffle=False)\n\n\n    if torch.cuda.is_available():\n        model = model.cuda()\n\n    criterion = nn.CrossEntropyLoss()\n    #optimizer = torch.optim.SGD(model.parameters(), config.learning_rate, momentum=config.momentum, weight_decay=config.reg)\n    #optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate, weight_decay=config.reg)\n    print(\"learning_rate=\", learning_rate)\n    optimizer = torch.optim.SGD(model.parameters(), learning_rate, momentum=config.momentum, weight_decay=config.reg)\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=config.reg)\n\n    best = 0.0\n    best_cm = None\n    best_model = None\n    train_loss_history = []\n    validation_loss_history = []\n    train_accuracy_history = []\n    validation_accuracy_history = []    \n    \n    for epoch in range(config.epochs):\n        #adjust_learning_rate(optimizer, epoch, config.steps, config.learning_rate)\n        adjust_learning_rate(optimizer, epoch, config.steps, learning_rate)\n        train_loss,train_acc = train(model, train_loader, criterion, optimizer, epoch, device)\n        train_loss_history.append(train_loss.item())\n        train_accuracy_history.append(train_acc)\n            \n        val_loss,val_acc, cm = validate(model, val_loader, criterion, epoch, device)\n        \n        validation_loss_history.append(val_loss.item())\n        validation_accuracy_history.append(val_acc)        \n        \n        print(\n            f\"Epoch:{epoch + 1}\"\n            + f\" Train Loss:{train_loss:.4f}\"\n            + f\" Val Loss: {val_loss:.4f}\"\n            + f\" Train Accuracy: {train_acc:.4f}\"\n            + f\" Validation Accuracy: {val_acc:.4f}\"\n        )\n        \n        if val_acc > best:\n            best = val_acc\n            best_cm = cm\n            best_model = copy.deepcopy(model)\n\n    print('Best Prec @1 Acccuracy: {:.4f}'.format(best))\n    per_cls_acc = best_cm.diag().detach().cpu().numpy().tolist()\n    plot_loss_history(train_loss_history,validation_loss_history)\n    plot_accuracy(train_accuracy_history,validation_accuracy_history)\n    # for i, acc_i in enumerate(per_cls_acc):\n    #     print(\"Accuracy of Class {}: {:.4f}\".format(i, acc_i))\n\n","metadata":{"id":"a3bff9ed","execution":{"iopub.status.busy":"2022-04-04T02:03:53.939362Z","iopub.execute_input":"2022-04-04T02:03:53.939653Z","iopub.status.idle":"2022-04-04T02:03:53.95973Z","shell.execute_reply.started":"2022-04-04T02:03:53.939574Z","shell.execute_reply":"2022-04-04T02:03:53.958939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Execute model","metadata":{"id":"8ef0fea5"}},{"cell_type":"code","source":"main()","metadata":{"id":"ea6537ab","outputId":"bece3c99-b73c-4bd7-8240-e2a6b87771ee","execution":{"iopub.status.busy":"2022-04-04T02:03:53.963313Z","iopub.execute_input":"2022-04-04T02:03:53.963532Z","iopub.status.idle":"2022-04-04T04:41:31.506041Z","shell.execute_reply.started":"2022-04-04T02:03:53.963508Z","shell.execute_reply":"2022-04-04T04:41:31.505228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main()\n#For learning rate = 0.0001","metadata":{"id":"77c5e377","execution":{"iopub.status.busy":"2022-04-04T05:12:22.029436Z","iopub.execute_input":"2022-04-04T05:12:22.029708Z","iopub.status.idle":"2022-04-04T05:12:49.64028Z","shell.execute_reply.started":"2022-04-04T05:12:22.029678Z","shell.execute_reply":"2022-04-04T05:12:49.63356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main()\n#For learning rate = 0.00001 ---> change in def main()\n#main(0.00001)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main()\n#For learning rate = 0.000001 ---> change in def main()\n#main(0.000001)","metadata":{},"execution_count":null,"outputs":[]}]}