{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"# Pytorch\n# ------------------------------------------------------\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n# Audio \n# ------------------------------------------------------\nimport torchaudio\nimport IPython.display as ipd\n\n# For Models\n# ------------------------------------------------------\nimport timm\n\n# scikit-learn\n# ------------------------------------------------------\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\n\n# BASE\n# ------------------------------------------------------\nimport os\nimport gc\nimport random\nimport numpy as np \nimport pandas as pd\n\n# DATA VISUALIZATION\n# ------------------------------------------------------\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\n\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-07T23:07:43.597262Z","iopub.execute_input":"2023-03-07T23:07:43.597723Z","iopub.status.idle":"2023-03-07T23:07:48.251427Z","shell.execute_reply.started":"2023-03-07T23:07:43.597688Z","shell.execute_reply":"2023-03-07T23:07:48.250270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\"epochs\": 10,\n          \"num_fold\": 5,\n          \"num_classes\": 264,\n          \"train_batch_size\": 16,\n          \"valid_batch_size\": 16,\n          \"sample_rate\": 32_000,\n          \"hop_length\": 512,\n          \"max_time\": 5,\n          \"n_mels\": 224,\n          \"n_fft\": 1024,\n          }","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.253830Z","iopub.execute_input":"2023-03-07T23:07:48.254214Z","iopub.status.idle":"2023-03-07T23:07:48.261000Z","shell.execute_reply.started":"2023-03-07T23:07:48.254176Z","shell.execute_reply":"2023-03-07T23:07:48.260009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\ndf.head().style.background_gradient(cmap=\"Accent\")","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.262613Z","iopub.execute_input":"2023-03-07T23:07:48.263347Z","iopub.status.idle":"2023-03-07T23:07:48.437142Z","shell.execute_reply.started":"2023-03-07T23:07:48.263309Z","shell.execute_reply":"2023-03-07T23:07:48.436225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.439919Z","iopub.execute_input":"2023-03-07T23:07:48.440294Z","iopub.status.idle":"2023-03-07T23:07:48.462075Z","shell.execute_reply.started":"2023-03-07T23:07:48.440256Z","shell.execute_reply":"2023-03-07T23:07:48.461053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().style.background_gradient(cmap=\"Accent\")","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.464944Z","iopub.execute_input":"2023-03-07T23:07:48.465700Z","iopub.status.idle":"2023-03-07T23:07:48.490736Z","shell.execute_reply.started":"2023-03-07T23:07:48.465670Z","shell.execute_reply":"2023-03-07T23:07:48.489659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyse Audio Files","metadata":{}},{"cell_type":"code","source":"ipd.Audio(\"/kaggle/input/birdclef-2023/train_audio/abethr1/XC531557.ogg\")","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.492526Z","iopub.execute_input":"2023-03-07T23:07:48.492920Z","iopub.status.idle":"2023-03-07T23:07:48.515264Z","shell.execute_reply.started":"2023-03-07T23:07:48.492878Z","shell.execute_reply":"2023-03-07T23:07:48.514361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(\"/kaggle/input/birdclef-2023/train_audio/blcapa2/XC120191.ogg\")\n","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.516211Z","iopub.execute_input":"2023-03-07T23:07:48.516578Z","iopub.status.idle":"2023-03-07T23:07:48.532273Z","shell.execute_reply.started":"2023-03-07T23:07:48.516538Z","shell.execute_reply":"2023-03-07T23:07:48.531161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename_1 = \"/kaggle/input/birdclef-2023/train_audio/abethr1/XC531557.ogg\"\nfilename_2 = \"/kaggle/input/birdclef-2023/train_audio/blcapa2/XC120191.ogg\"\n\nfig, ax = plt.subplots(2, 1, figsize=(20, 10))\nfig.suptitle(\"Sound Waves\", fontsize=15)\n\nsignal_1, sr = torchaudio.load(filename_1)\nsns.lineplot(x=np.arange(len(signal_1[0,:].detach().numpy())), y=signal_1[0,:].detach().numpy(), ax=ax[0], color='#4400FF')\nax[0].set_title(\"Audio 1\")\n\nsignal_2, sr = torchaudio.load(filename_2)\nsns.lineplot(x=np.arange(len(signal_2[0,:].detach().numpy())), y=signal_2[0,:].detach().numpy(), ax=ax[1], color='#4400FF')\nax[1].set_title(\"Audio 2\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:48.533923Z","iopub.execute_input":"2023-03-07T23:07:48.534234Z","iopub.status.idle":"2023-03-07T23:07:50.157289Z","shell.execute_reply.started":"2023-03-07T23:07:48.534204Z","shell.execute_reply":"2023-03-07T23:07:50.156332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"encoder = LabelEncoder()\ndf['primary_label_encoded'] = encoder.fit_transform(df['primary_label'])","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.159042Z","iopub.execute_input":"2023-03-07T23:07:50.159789Z","iopub.status.idle":"2023-03-07T23:07:50.170339Z","shell.execute_reply.started":"2023-03-07T23:07:50.159751Z","shell.execute_reply":"2023-03-07T23:07:50.169350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Num_classes\nlen(df['primary_label_encoded'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.175526Z","iopub.execute_input":"2023-03-07T23:07:50.175837Z","iopub.status.idle":"2023-03-07T23:07:50.186345Z","shell.execute_reply.started":"2023-03-07T23:07:50.175811Z","shell.execute_reply":"2023-03-07T23:07:50.185393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Folds","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG[\"num_fold\"])\nfor k, (_, val_ind) in enumerate(skf.split(X=df, y=df['primary_label_encoded'])):\n    df.loc[val_ind, 'kfold'] = k","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.187900Z","iopub.execute_input":"2023-03-07T23:07:50.188247Z","iopub.status.idle":"2023-03-07T23:07:50.205130Z","shell.execute_reply.started":"2023-03-07T23:07:50.188214Z","shell.execute_reply":"2023-03-07T23:07:50.204273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch Dataset","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/code/debarshichanda/pytorch-w-b-birdclef-22-starter\nclass BirdCLEFDataset(Dataset):\n    def __init__(self, df, target_sample_rate=CONFIG[\"sample_rate\"], max_time=5, image_transforms=None):\n        self.file_paths = df['filename'].values\n        self.labels = df['primary_label_encoded'].values\n        self.target_sample_rate = target_sample_rate\n        num_samples = target_sample_rate * max_time\n        self.num_samples = num_samples\n        self.image_transforms = image_transforms\n        \n    def __len__(self):\n        return len(self.file_paths)\n    \n    def __getitem__(self, index):\n        filepath = f'/kaggle/input/birdclef-2023/train_audio/{self.file_paths[index]}'\n        audio, sample_rate = torchaudio.load(filepath)\n        audio = self.to_mono(audio)\n        \n        if sample_rate != self.target_sample_rate:\n            resample = Resample(sample_rate, self.target_sample_rate)\n            audio = resample(audio)\n        \n        if audio.shape[0] > self.num_samples:\n            audio = self.crop_audio(audio)\n            \n        if audio.shape[0] < self.num_samples:\n            audio = self.pad_audio(audio)\n            \n        mel_spectogram = torchaudio.transforms.MelSpectrogram(sample_rate=self.target_sample_rate, \n                                        n_mels=CONFIG['n_mels'], \n                                        n_fft=CONFIG['n_fft'])\n        mel = mel_spectogram(audio)\n        label = torch.tensor(self.labels[index])\n        \n        # Convert to Image\n        image = torch.stack([mel, mel, mel])\n        \n        # Normalize Image\n        max_val = torch.abs(image).max()\n        image = image / max_val\n        \n        return image, label\n            \n    def pad_audio(self, audio):\n        pad_length = self.num_samples - audio.shape[0]\n        last_dim_padding = (0, pad_length)\n        audio = F.pad(audio, last_dim_padding)\n        return audio\n        \n    def crop_audio(self, audio):\n        return audio[:self.num_samples]\n        \n    def to_mono(self, audio):\n        return torch.mean(audio, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.207310Z","iopub.execute_input":"2023-03-07T23:07:50.207968Z","iopub.status.idle":"2023-03-07T23:07:50.220003Z","shell.execute_reply.started":"2023-03-07T23:07:50.207931Z","shell.execute_reply":"2023-03-07T23:07:50.218983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'\n    \n    \nclass BirdCLEFModel(nn.Module):\n    def __init__(self, model_name=\"tf_efficientnet_b4_ns\", embedding_size=768, pretrained=True):\n        super(BirdCLEFModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.embedding = nn.Linear(in_features, embedding_size)\n        self.fc = nn.Linear(embedding_size, CONFIG['num_classes'])\n\n    def forward(self, images):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        embedding = self.embedding(pooled_features)\n        output = self.fc(embedding)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.223122Z","iopub.execute_input":"2023-03-07T23:07:50.223615Z","iopub.status.idle":"2023-03-07T23:07:50.235133Z","shell.execute_reply.started":"2023-03-07T23:07:50.223584Z","shell.execute_reply":"2023-03-07T23:07:50.234061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(outputs, labels):\n    return nn.CrossEntropyLoss()(outputs, labels)\n\ndef train(model, data_loader, optimizer, scheduler, device, epoch):\n    model.train()\n    \n    running_loss = 0\n    loop = tqdm(data_loader, position=0)\n    for i, (mels, labels) in enumerate(loop):\n        mels = mels.to(device)\n        labels = labels.to(device)\n        \n        outputs = model(mels)\n        _, preds = torch.max(outputs, 1)\n        \n        loss = loss_fn(outputs, labels)\n        \n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        \n        if scheduler is not None:\n            scheduler.step()\n            \n        running_loss += loss.item()\n        \n        loop.set_description(f\"Epoch [{epoch+1}/{CONFIG['epochs']}]\")\n        loop.set_postfix(loss=loss.item())\n\n    return running_loss/len(data_loader)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.236562Z","iopub.execute_input":"2023-03-07T23:07:50.237592Z","iopub.status.idle":"2023-03-07T23:07:50.247305Z","shell.execute_reply.started":"2023-03-07T23:07:50.237556Z","shell.execute_reply":"2023-03-07T23:07:50.246377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid(model, data_loader, device, epoch):\n    model.eval()\n    \n    running_loss = 0\n    pred = []\n    label = []\n    \n    loop = tqdm(data_loader, position=0)\n    for mels, labels in loop:\n        mels = mels.to(device)\n        labels = labels.to(device)\n        \n        outputs = model(mels)\n        _, preds = torch.max(outputs, 1)\n        \n        loss = loss_fn(outputs, labels)\n            \n        running_loss += loss.item()\n        \n        pred.extend(preds.view(-1).cpu().detach().numpy())\n        label.extend(labels.view(-1).cpu().detach().numpy())\n        \n        loop.set_description(f\"Epoch [{epoch+1}/{CONFIG['epochs']}]\")\n        loop.set_postfix(loss=loss.item())\n        \n    valid_f1 = f1_score(label, pred, average='macro')\n    \n    return running_loss/len(data_loader), valid_f1","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.248722Z","iopub.execute_input":"2023-03-07T23:07:50.249618Z","iopub.status.idle":"2023-03-07T23:07:50.259267Z","shell.execute_reply.started":"2023-03-07T23:07:50.249582Z","shell.execute_reply":"2023-03-07T23:07:50.258639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    train_dataset = BirdCLEFDataset(df_train, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n    valid_dataset = BirdCLEFDataset(df_valid, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.260677Z","iopub.execute_input":"2023-03-07T23:07:50.261528Z","iopub.status.idle":"2023-03-07T23:07:50.270004Z","shell.execute_reply.started":"2023-03-07T23:07:50.261491Z","shell.execute_reply":"2023-03-07T23:07:50.269011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = BirdCLEFModel().to(device)\noptimizer = Adam(model.parameters(), lr=1e-4)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, eta_min=1e-5, T_max=10)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T23:07:50.271446Z","iopub.execute_input":"2023-03-07T23:07:50.272125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_valid_f1 = 0\ntrain_loader, valid_loader = prepare_loaders(df, 0) # fold 0\nfor epoch in range(CONFIG['epochs']):\n    train_loss = train(model, train_loader, optimizer, scheduler, device, epoch)\n    valid_loss, valid_f1 = valid(model, valid_loader, device, epoch)\n    if valid_f1 > best_valid_f1:\n        print(f\"Validation F1 Improved - {best_valid_f1} ---> {valid_f1}\")\n        torch.save(model.state_dict(), f'./model_0.bin')\n        print(f\"Saved model checkpoint at ./model_0.bin\")\n        best_valid_f1 = valid_f1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_valid_f1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}