{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport ast\nimport random\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm import tqdm\nimport torchaudio\nimport IPython.display as ipd\nfrom collections import Counter\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score,classification_report\nfrom sklearn.utils import class_weight\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass config:\n seed=2022\n num_fold = 6\n sample_rate= 32_000\n n_fft=1024\n hop_length=512\n n_mels=64\n duration=5\n num_classes = 12\n train_batch_size = 32\n valid_batch_size = 64\n model_name = 'resnet50'\n epochs = 100\n device = 'cuda' if torch.cuda.is_available() else 'cpu'\n learning_rate = 0.0001\n \ndef seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\nseed_everything(config.seed)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-25T14:40:48.962612Z","iopub.execute_input":"2022-11-25T14:40:48.962881Z","iopub.status.idle":"2022-11-25T14:40:48.977561Z","shell.execute_reply.started":"2022-11-25T14:40:48.962850Z","shell.execute_reply":"2022-11-25T14:40:48.976521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('../input/birdclef-2022/train_metadata.csv')\nndf = pd.read_csv(\"../input/swahili-trainsplit/Trainset.csv\")\nvaldf = pd.read_csv(\"/kaggle/input/swahili-trainsplit/valset.csv\")\ntest = pd.read_csv(\"/kaggle/input/swahili-trainsplit/testset (2).csv\")\nndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:49.279534Z","iopub.execute_input":"2022-11-25T14:40:49.279804Z","iopub.status.idle":"2022-11-25T14:40:49.301581Z","shell.execute_reply.started":"2022-11-25T14:40:49.279774Z","shell.execute_reply":"2022-11-25T14:40:49.300895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {'mbili': 0, 'tatu': 1, 'ndio': 2, 'nne': 3, 'nane': 4, 'hapana': 5, 'sita': 6, 'tisa': 7, 'moja': 8, 'saba': 9, 'tano': 10, 'kumi': 11}\nndf['label_encoded'] = ndf['Swahili_word'].apply(lambda x : label_map[x])\nvaldf['label_encoded'] = valdf['Swahili_word'].apply(lambda x : label_map[x])\ntest['label_encoded'] = test['Swahili_word'].apply(lambda x : label_map[x])\n\n\n\ny = torch.FloatTensor(ndf['label_encoded'])\n\nclass_weights=class_weight.compute_class_weight(class_weight = 'balanced',classes = np.unique(y), y = y.numpy())\nclass_weights=torch.tensor(class_weights,dtype=torch.float)\n\nclass_weights = class_weights.to(config.device)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:49.621001Z","iopub.execute_input":"2022-11-25T14:40:49.621799Z","iopub.status.idle":"2022-11-25T14:40:49.637454Z","shell.execute_reply.started":"2022-11-25T14:40:49.621760Z","shell.execute_reply":"2022-11-25T14:40:49.636753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"swahili_labels = ndf['Swahili_word'].unique()\n","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:50.004011Z","iopub.execute_input":"2022-11-25T14:40:50.004760Z","iopub.status.idle":"2022-11-25T14:40:50.009994Z","shell.execute_reply.started":"2022-11-25T14:40:50.004723Z","shell.execute_reply":"2022-11-25T14:40:50.009241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Next we created folds.","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=config.num_fold)\nfor k, (_, val_ind) in enumerate(skf.split(X=ndf, y=ndf['label_encoded'])):\n ndf.loc[val_ind, 'fold'] = k","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:50.606094Z","iopub.execute_input":"2022-11-25T14:40:50.606664Z","iopub.status.idle":"2022-11-25T14:40:50.619258Z","shell.execute_reply.started":"2022-11-25T14:40:50.606626Z","shell.execute_reply":"2022-11-25T14:40:50.618565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mel_spectrogram = torchaudio.transforms.MelSpectrogram(sample_rate=config.sample_rate, \n n_fft=config.n_fft, \n hop_length=config.hop_length, \n n_mels=config.n_mels)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:50.842501Z","iopub.execute_input":"2022-11-25T14:40:50.842770Z","iopub.status.idle":"2022-11-25T14:40:50.848462Z","shell.execute_reply.started":"2022-11-25T14:40:50.842739Z","shell.execute_reply":"2022-11-25T14:40:50.847677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SwahiliDataset(Dataset):\n def __init__(self, df, transformation, target_sample_rate, duration):\n self.audio_paths = df['Word_id'].values\n self.labels = df['label_encoded'].values\n self.transformation = transformation\n self.target_sample_rate = target_sample_rate\n self.num_samples = target_sample_rate*duration\n \n def __len__(self):\n return len(self.audio_paths)\n \n def __getitem__(self, index):\n audio_path = f'../input/swahili-data/Swahili_words/{self.audio_paths[index]}'\n signal, sr = torchaudio.load(audio_path) # loaded the audio\n \n # Now we first checked if the sample rate is same as TARGET_SAMPLE_RATE and if it not equal we perform resampling\n if sr != self.target_sample_rate:\n resampler = torchaudio.transforms.Resample(sr, self.target_sample_rate)\n signal = resampler(signal)\n \n # Next we check the number of channels of the signal\n #signal -> (num_channels, num_samples) - Eg.-(2, 14000) -> (1, 14000)\n if signal.shape[0]>1:\n signal = torch.mean(signal, axis=0, keepdim=True)\n \n # Lastly we check the number of samples of the signal\n #signal -> (num_channels, num_samples) - Eg.-(1, 14000) -> (1, self.num_samples)\n # If it is more than the required number of samples, we truncate the signal\n if signal.shape[1] > self.num_samples:\n signal = signal[:, :self.num_samples]\n \n # If it is less than the required number of samples, we pad the signal\n if signal.shape[1]\n# Model 🤖","metadata":{}},{"cell_type":"markdown","source":"### So I will first start with a custom CNN model. After that, we will the see the usage of pretrained and other advanced models.","metadata":{}},{"cell_type":"code","source":"class SwahiliModel(nn.Module):\n def __init__(self):\n super(SwahiliModel, self).__init__()\n self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1)\n self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1)\n self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)\n self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)\n self.fc1 = nn.Linear(128*8*39, 64)\n self.fc2 = nn.Linear(64, config.num_classes)\n #self.softmax = nn.Softmax(dim = None)\n \n def forward(self, x):\n x = F.relu(self.conv1(x))\n x = self.pool1(x)\n x = F.relu(self.conv2(x))\n x = self.pool2(x)\n x = F.relu(self.conv3(x))\n x = self.pool3(x)\n x = x.view(x.size(0), -1)\n x = F.relu(self.fc1(x))\n x = self.fc2(x)\n #x = self.softmax(x)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:51.870701Z","iopub.execute_input":"2022-11-25T14:40:51.871438Z","iopub.status.idle":"2022-11-25T14:40:51.881053Z","shell.execute_reply.started":"2022-11-25T14:40:51.871403Z","shell.execute_reply":"2022-11-25T14:40:51.879944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now we will fine tune a pretrained model. Here I have used Resnet50. You can use any pretrained model and do experiments.","metadata":{}},{"cell_type":"code","source":"class SwahiliResnet(nn.Module):\n def __init__(self):\n super(SwahiliResnet, self).__init__()\n self.base_model = models.__getattribute__(config.model_name)(pretrained=True)\n for param in self.base_model.parameters():\n param.requires_grad = False\n \n in_features = self.base_model.fc.in_features\n \n self.base_model.fc = nn.Sequential(\n nn.Linear(in_features, 1024), \n nn.ReLU(), \n nn.Dropout(p=0.2),\n nn.Linear(1024, 512), \n nn.ReLU(), \n nn.Dropout(p=0.2),\n nn.Linear(512, config.num_classes))\n \n def forward(self, x):\n x = self.base_model(x)\n return x","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:52.324573Z","iopub.execute_input":"2022-11-25T14:40:52.324834Z","iopub.status.idle":"2022-11-25T14:40:52.331919Z","shell.execute_reply.started":"2022-11-25T14:40:52.324804Z","shell.execute_reply":"2022-11-25T14:40:52.331163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Utility Functions 📋","metadata":{}},{"cell_type":"markdown","source":"### Next we define some functions to train the model. These are the basic functions which we use to train any pytorch based models.","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:12:56.001998Z","iopub.execute_input":"2022-03-20T18:12:56.002486Z","iopub.status.idle":"2022-03-20T18:12:56.00889Z","shell.execute_reply.started":"2022-03-20T18:12:56.002451Z","shell.execute_reply":"2022-03-20T18:12:56.00809Z"}}},{"cell_type":"code","source":"def loss_fn(outputs, labels):\n return nn.CrossEntropyLoss(weight = class_weights)(outputs, labels)\n\ndef train(model, data_loader, optimizer, scheduler, device, epoch):\n model.train()\n \n running_loss = 0\n loop = tqdm(data_loader, position=0)\n for i, (mels, labels) in enumerate(loop):\n mels = mels.to(device)\n labels = labels.to(device)\n \n outputs = model(mels)\n _, preds = torch.max(outputs, 1)\n \n loss = loss_fn(outputs, labels)\n \n loss.backward()\n optimizer.step()\n optimizer.zero_grad()\n \n if scheduler is not None:\n scheduler.step()\n \n running_loss += loss.item()\n \n loop.set_description(f\"Epoch [{epoch+1}/{config.epochs}]\")\n loop.set_postfix(loss=loss.item())\n\n return model,running_loss/len(data_loader)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:52.966050Z","iopub.execute_input":"2022-11-25T14:40:52.966657Z","iopub.status.idle":"2022-11-25T14:40:52.974761Z","shell.execute_reply.started":"2022-11-25T14:40:52.966620Z","shell.execute_reply":"2022-11-25T14:40:52.973789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid(model, data_loader,test_data_loader, device, epoch):\n model.eval()\n \n running_loss = 0\n pred = []\n label = []\n \n loop = tqdm(data_loader, position=0)\n for mels, labels in loop:\n mels = mels.to(device)\n labels = labels.to(device)\n \n outputs = model(mels)\n _, preds = torch.max(outputs, 1)\n \n loss = loss_fn(outputs, labels)\n \n running_loss += loss.item()\n \n pred.extend(preds.view(-1).cpu().detach().numpy())\n label.extend(labels.view(-1).cpu().detach().numpy())\n \n loop.set_description(f\"Epoch [{epoch+1}/{config.epochs}]\")\n loop.set_postfix(loss=loss.item())\n \n valid_f1 = f1_score(label, pred, average='macro')\n \n pred = []\n label = []\n \n loop = tqdm(test_data_loader, position=0)\n for mels, labels in loop:\n mels = mels.to(device)\n labels = labels.to(device)\n \n outputs = model(mels)\n _, preds = torch.max(outputs, 1)\n \n \n \n pred.extend(preds.view(-1).cpu().detach().numpy())\n label.extend(labels.view(-1).cpu().detach().numpy())\n \n loop.set_description(f\"Epoch [{epoch+1}/{config.epochs}]\")\n \n \n test_f1 = f1_score(label, pred, average='macro')\n test_cr = classification_report(label, pred)\n print(test_cr.split(\"\\n\"))\n \n return running_loss/len(data_loader), valid_f1,test_f1","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:53.180674Z","iopub.execute_input":"2022-11-25T14:40:53.180924Z","iopub.status.idle":"2022-11-25T14:40:53.191266Z","shell.execute_reply.started":"2022-11-25T14:40:53.180895Z","shell.execute_reply":"2022-11-25T14:40:53.190410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checkpoint = {'model': Classifier(),\n #'state_dict': model.state_dict(),\n #'optimizer' : optimizer.state_dict()}\n\n#torch.save(checkpoint, 'checkpoint.pth')\n\ndef load_checkpoint(filepath):\n checkpoint = torch.load(filepath)\n model.load_state_dict(checkpoint['state_dict'])\n optimizer.load_state_dict(checkpoint['optimizer'])\n #epoch = checkpoint['epoch']\n loss = checkpoint['loss']\n \n\n model.train()\n return model\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:53.396279Z","iopub.execute_input":"2022-11-25T14:40:53.396543Z","iopub.status.idle":"2022-11-25T14:40:53.401952Z","shell.execute_reply.started":"2022-11-25T14:40:53.396514Z","shell.execute_reply":"2022-11-25T14:40:53.401217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SwahiliModel().to(config.device) # check version 3 for this\n#model = BirdCLEFResnet().to(config.device)\nmodel = SwahiliResnet()\noptimizer = Adam(model.parameters(), lr=config.learning_rate)\n\ncheckpoint = {'model': model, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), 'loss' : 0.5}\ntorch.save(checkpoint, f'./model_0.bin')\n\ndef run(fold):\n train_loader, valid_loader,test_loader = get_data(fold)\n \n \n \n #model = torch.load(f'./model_{fold}.bin')\n #model.load_state_dict(torch.load(f'./model_{fold}.bin'))\n model = load_checkpoint(f'./model_{fold}.bin').to(config.device)\n \n \n \n scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, eta_min=1e-5, T_max=10)\n \n best_valid_f1 = 0\n for epoch in range(config.epochs):\n train_loss = train(model, train_loader, optimizer, None, config.device, epoch)\n valid_loss, valid_f1,test_f1 = valid(model, valid_loader, test_loader, config.device, epoch)\n print(f\"Test F1 ----> {test_f1}\")\n if valid_f1 > best_valid_f1:\n print(f\"Validation F1 Improved - {best_valid_f1} ---> {valid_f1}\")\n checkpoint = {'model': model, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), 'loss' : valid_loss}\n torch.save(checkpoint, f'./model_{fold + 1}.bin')\n print(f\"Saved model checkpoint at ./model_{fold + 1}.bin\")\n best_valid_f1 = valid_f1\n \n return best_valid_f1\n\nfor fold in range(config.num_fold):\n print(\"=\" * 30)\n print(\"Training Fold - \", fold)\n print(\"=\" * 30)\n best_valid_f1 = run(fold)\n print(f'Best F1 Score: {best_valid_f1:.5f}')\n \n gc.collect()\n torch.cuda.empty_cache() ","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:53.648632Z","iopub.execute_input":"2022-11-25T14:40:53.649221Z","iopub.status.idle":"2022-11-25T14:40:59.872622Z","shell.execute_reply.started":"2022-11-25T14:40:53.649181Z","shell.execute_reply":"2022-11-25T14:40:59.871309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Training ⚙️","metadata":{}},{"cell_type":"markdown","source":"**TESTING**","metadata":{}},{"cell_type":"code","source":"# import json\n\n# TEST_AUDIO_PATH = '../input/swahili-data/Swahili_words'\n\n# with open('../input/birdclef-2022/scored_birds.json') as fp:\n# SCORED_BIRDS = json.load(fp)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:40:09.710619Z","iopub.execute_input":"2022-11-25T14:40:09.711093Z","iopub.status.idle":"2022-11-25T14:40:09.714923Z","shell.execute_reply.started":"2022-11-25T14:40:09.711035Z","shell.execute_reply":"2022-11-25T14:40:09.714029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n\n# TEST_AUDIO_PATH = '../input/swahili-data/Swahili_words'\n\n# with open('../input/birdclef-2022/scored_birds.json') as fp:\n# SCORED_BIRDS = json.load(fp)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T14:33:03.347413Z","iopub.status.idle":"2022-11-25T14:33:03.347855Z","shell.execute_reply.started":"2022-11-25T14:33:03.347608Z","shell.execute_reply":"2022-11-25T14:33:03.347631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import math\n# class TestDataset(Dataset):\n# def __init__(self, df, transformation, target_sample_rate, duration):\n# self.audio_paths = df['Word_id'].values\n# #self.labels = df['birds'].values\n# self.transformation = transformation\n# self.target_sample_rate = target_sample_rate\n# self.num_samples = target_sample_rate*duration\n# self.end_sample = [5*target_sample_rate for i in range(len(df))]\n# # self.end_sample = df['ending_second'].values * target_sample_rate\n \n# def __len__(self):\n# return len(self.audio_paths)\n \n# def __getitem__(self, index):\n# audio_path = f'../input/swahili-data/Swahili_words/{self.audio_paths[index]}'\n# signal, sr = torchaudio.load(audio_path) # loaded the audio\n \n# # Now we first checked if the sample rate is same as TARGET_SAMPLE_RATE and if it not equal we perform resampling\n# if sr != self.target_sample_rate:\n# resampler = torchaudio.transforms.Resample(sr, self.target_sample_rate)\n# signal = resampler(signal)\n \n# # Next we check the number of channels of the signal\n# #signal -> (num_channels, num_samples) - Eg.-(2, 14000) -> (1, 14000)\n# if signal.shape[0]>1:\n# signal = torch.mean(signal, axis=0, keepdim=True)\n \n# # Seperate the 5 second chunk we want from the signal\n# signal = signal[:, (self.end_sample[index]-self.num_samples):self.end_sample[index]]\n \n# # Lastly we check the number of samples of the signal\n# #signal -> (num_channels, num_samples) - Eg.-(1, 14000) -> (1, self.num_samples)\n# # If it is more than the required number of samples, we truncate the signal\n# if signal.shape[1] > self.num_samples:\n# signal = signal[:, :self.num_samples]\n \n# # If it is less than the required number of samples, we pad the signal\n# if signal.shape[1]