{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc # garbage collection\nimport cv2\nimport math\nimport copy\nimport time\nimport random\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\n\n# Audio \nimport torchaudio\nfrom torchaudio.transforms import MelSpectrogram, Resample\nfrom IPython.display import Audio\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score\n\n# For Image Models\nimport timm\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\"seed\": 11,\n          \"epochs\": 10,\n          \"model_name\": \"tf_efficientnet_b0_ns\",\n          \"embedding_size\": 768,\n          \"num_classes\": 264,\n          \"train_batch_size\": 64,\n          \"valid_batch_size\": 128,\n          \"learning_rate\": 1e-4,\n          \"scheduler\": 'CosineAnnealingLR',\n          \"min_lr\": 1e-6,\n          \"T_max\": 500,\n          \"weight_decay\": 1e-6,\n          \"n_fold\": 5,\n          \"n_accumulate\": 1, # gradient accumulation\n          \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n          \"competition\": \"BirdCLEF23\",\n          \"_wandb_kernel\": \"deb\",\n          # Audio Specific\n          \"sample_rate\": 32_000,\n          \"max_time\": 5,\n          \"n_mels\": 224,\n          \"n_fft\"}\n\ndef set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/birdclef-2023'\nTRAIN_DIR = '../input/birdclef-2023/train_audio'\nTEST_DIR = '../input/birdclef-2023/test_soundscapes'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(filename):\n    return f\"{TRAIN_DIR}/{filename}\"\ndf = pd.read_csv(f\"{ROOT_DIR}/train_metadata.csv\")\ndf['file_path'] = df['filename'].apply(get_train_file_path)\ndf.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_file = df.loc[0].file_path\nmetadata = torchaudio.info(sample_file)\nprint(metadata)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T20:25:12.446156Z","iopub.execute_input":"2023-04-09T20:25:12.447103Z","iopub.status.idle":"2023-04-09T20:25:12.527560Z","shell.execute_reply.started":"2023-04-09T20:25:12.447058Z","shell.execute_reply":"2023-04-09T20:25:12.524187Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's try loading the file in\nwaveform, sample_rate = torchaudio.load(sample_file)\ndef plot_waveform(waveform, sample_rate):\n    waveform = waveform.numpy()\n\n    num_channels, num_frames = waveform.shape\n    time_axis = torch.arange(0, num_frames) / sample_rate\n\n    figure, axes = plt.subplots(num_channels, 1)\n    if num_channels == 1:\n        axes = [axes]\n    for c in range(num_channels):\n        axes[c].plot(time_axis, waveform[c], linewidth=1)\n        axes[c].grid(True)\n        if num_channels > 1:\n            axes[c].set_ylabel(f\"Channel {c+1}\")\n    figure.suptitle(\"waveform\")\n    plt.show(block=False)\n    \nplot_waveform(waveform, sample_rate)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Audio(waveform.numpy()[0], rate=sample_rate)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\nall_labels = sorted(df['primary_label'].values)\nencoder.fit(all_labels)\n\ndf['primary_label'] = encoder.transform(df['primary_label'])\n\nwith open(\"le.pkl\", \"wb\") as fp:\n    joblib.dump(encoder, fp)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.primary_label)):\n      df.loc[val_ , \"kfold\"] = fold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdCLEFDataset(Dataset):\n    def __init__(self, df, target_sample_rate, max_time, image_transforms=None):\n        self.file_paths = df['file_path'].values\n        self.labels = df['primary_label'].values\n        self.target_sample_rate = target_sample_rate\n        num_samples = target_sample_rate * max_time\n        self.num_samples = num_samples\n        self.image_transforms = image_transforms\n        \n    def __len__(self):\n        return len(self.file_paths)\n    \n    def __getitem__(self, index):\n        filepath = self.file_paths[index]\n        audio, sample_rate = torchaudio.load(filepath)\n        audio = self.to_mono(audio)\n        \n        if sample_rate != self.target_sample_rate:\n            resample = Resample(sample_rate, self.target_sample_rate)\n            audio = resample(audio)\n        \n        if audio.shape[0] > self.num_samples:\n            audio = self.crop_audio(audio)\n            \n        if audio.shape[0] < self.num_samples:\n            audio = self.pad_audio(audio)\n            \n        mel_spectogram = MelSpectrogram(sample_rate=self.target_sample_rate, \n                                        n_mels=CONFIG['n_mels'], \n                                        n_fft=CONFIG['n_fft'])\n        mel = mel_spectogram(audio)\n        \n        label_onehot = torch.zeros(CONFIG['num_classes'])\n        label_onehot[self.labels[index]] = 1\n        label = torch.tensor(self.labels[index])\n        \n        # Convert to Image\n        image = torch.stack([mel, mel, mel])\n        \n        # Normalize Image\n        max_val = torch.abs(image).max()\n        image = image / max_val\n        \n        return {\n            \"image\": image, \n            \"label\": label,\n            \"label_onehot\": label_onehot\n        }\n            \n    def pad_audio(self, audio):\n        pad_length = self.num_samples - audio.shape[0]\n        last_dim_padding = (0, pad_length)\n        audio = F.pad(audio, last_dim_padding)\n        return audio\n        \n    def crop_audio(self, audio):\n        return audio[:self.num_samples]\n        \n    def to_mono(self, audio):\n        return torch.mean(audio, axis=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, model_name, embedding_size, pretrained=True):\n        super(BirdCLEFModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.embedding = nn.Linear(in_features, embedding_size)\n        self.fc = nn.Linear(embedding_size, CONFIG['num_classes'])\n\n    def forward(self, images):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        embedding = self.embedding(pooled_features)\n        output = self.fc(embedding)\n        return output\n    \nmodel = BirdCLEFModel(CONFIG['model_name'], CONFIG['embedding_size'])\nmodel.to(CONFIG['device']);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(outputs, labels):\n    return nn.CrossEntropyLoss()(outputs, labels)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images = data['image'].to(device)\n        labels = data['label'].to(device)\n        \n        batch_size = images.size(0)\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss = loss / CONFIG['n_accumulate']\n            \n        loss.backward()\n    \n        if (step + 1) % CONFIG['n_accumulate'] == 0:\n            optimizer.step()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect() #garbage collection, presumably to prevent the GPU from blowing up\n    \n    return epoch_loss","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def padded_cmap(solution, submission, padding_factor=5): # I don't quite get this part why do you need to pad?\n    new_rows = []\n    for i in range(padding_factor):\n        new_rows.append([1 for i in range(len(solution.columns))])\n    new_rows = pd.DataFrame(new_rows)\n    new_rows.columns = solution.columns\n    padded_solution = pd.concat([solution, new_rows]).reset_index(drop=True).copy()\n    padded_submission = pd.concat([submission, new_rows]).reset_index(drop=True).copy()\n    score = average_precision_score(\n        padded_solution.values,\n        padded_submission.values,\n        average='macro',\n    )\n    return score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.inference_mode() #InferenceMode is a new context manager analogous to no_grad to be used when you are certain your operations will have no interactions with autograd (e.g., model training). Code run under this mode gets better performance by disabling view tracking and version counter bumps\ndef valid_one_epoch(model, dataloader, device, epoch):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    LABELS = []\n    PREDS = []\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:        \n        images = data['image'].to(device)\n        labels = data['label_onehot'].to(device)\n        \n        batch_size = images.size(0)\n\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        PREDS.append(F.softmax(outputs, dim=1).cpu().detach().numpy())\n        LABELS.append(labels.cpu().detach().numpy())\n        \n        bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])   \n    \n    LABELS = np.vstack(LABELS)\n    PREDS = np.vstack(PREDS)\n    val_cmap = padded_cmap(pd.DataFrame(LABELS), pd.DataFrame(PREDS))\n    gc.collect()\n    \n    return epoch_loss, val_cmap","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    # To automatically log gradients\n#     wandb.watch(model, log_freq=100)\n    \n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_cmap = 0\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=CONFIG['device'], epoch=epoch)\n        \n        val_epoch_loss, val_epoch_cmap = valid_one_epoch(model, valid_loader, \n                                                       device=CONFIG['device'], \n                                                       epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n        history['Valid Loss'].append(val_epoch_loss)\n        history['Valid CMAP'].append(val_epoch_cmap)\n        \n        # Log the metrics\n#         wandb.log({\"Train Loss\": train_epoch_loss})\n#         wandb.log({\"Valid Loss\": val_epoch_loss})\n#         wandb.log({\"Valid CMAP\": val_epoch_cmap})\n        \n        # deep copy the model\n        if val_epoch_cmap >= best_epoch_cmap:\n            print(f\"{b_}Validation CMAP Improved ({best_epoch_cmap} ---> {val_epoch_cmap})\")\n            best_epoch_cmap = val_epoch_cmap\n#             run.summary[\"Best CMAP Score\"] = best_epoch_cmap\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = \"CMAP{:.4f}_epoch{:.0f}.bin\".format(best_epoch_cmap, epoch)\n            torch.save(model.state_dict(), PATH)\n            # Save a model file from the current directory\n            print(f\"Model Saved{sr_}\")\n            \n        print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best CMAP: {:.4f}\".format(best_epoch_cmap))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CONFIG['scheduler'] == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n                                                   eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n                                                             eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == None:\n        return None\n        \n    return scheduler","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    train_dataset = BirdCLEFDataset(df_train, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n    valid_dataset = BirdCLEFDataset(df_valid, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n                       weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n                              device=CONFIG['device'],\n                              num_epochs=CONFIG['epochs'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}