{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BirdCLEF23 Starter ACM AI","metadata":{}},{"cell_type":"markdown","source":"# TODOS\n* add more description on how we're preprocessing the data cuz it kinda just works rn and I have no idea why\n* comment code for readability\n* add some validation metrics (ROC, F1 etc.)\n* plot the validation vs training accuracy","metadata":{}},{"cell_type":"markdown","source":"![](https://storage.googleapis.com/kaggle-competitions/kaggle/25954/logos/header.png?t=2021-03-19-18-32-57)","metadata":{}},{"cell_type":"markdown","source":"# Install Required Libraries","metadata":{}},{"cell_type":"code","source":"!pip install timm\n# !pip install --upgrade wandb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-10T01:01:45.118614Z","iopub.execute_input":"2023-04-10T01:01:45.11926Z","iopub.status.idle":"2023-04-10T01:01:54.713049Z","shell.execute_reply.started":"2023-04-10T01:01:45.119221Z","shell.execute_reply":"2023-04-10T01:01:54.711778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Required Libraries 📚","metadata":{}},{"cell_type":"code","source":"import os\nimport gc # garbage collection\nimport cv2\nimport math\nimport copy\nimport time\nimport random\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\n\n# Audio \nimport torchaudio\nfrom torchaudio.transforms import MelSpectrogram, Resample\nfrom IPython.display import Audio\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score\n\n# For Image Models\nimport timm\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:54.721057Z","iopub.execute_input":"2023-04-10T01:01:54.721419Z","iopub.status.idle":"2023-04-10T01:01:56.907127Z","shell.execute_reply.started":"2023-04-10T01:01:54.721376Z","shell.execute_reply":"2023-04-10T01:01:56.906011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://i.imgur.com/gb6B4ig.png\" width=\"400\" alt=\"Weights & Biases\" />\n\n<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\"> Weights & Biases (W&B) is a set of machine learning tools that helps you build better models faster. <strong>Kaggle competitions require fast-paced model development and evaluation</strong>. There are a lot of components: exploring the training data, training different models, combining trained models in different combinations (ensembling), and so on.</span>\n\n> <span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">⏳ Lots of components = Lots of places to go wrong = Lots of time spent debugging</span>\n\n<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">To learn more about Weights and Biases check out this <strong><a href=\"https://www.kaggle.com/ayuraj/experiment-tracking-with-weights-and-biases\">kernel</a></strong>.</span>","metadata":{}},{"cell_type":"code","source":"# import wandb\n\n# try:\n#     from kaggle_secrets import UserSecretsClient\n#     user_secrets = UserSecretsClient()\n#     api_key = user_secrets.get_secret(\"wandb_api\")\n#     wandb.login(key=api_key)\n#     anony = None\n# except:\n#     anony = \"must\"\n#     print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:56.908931Z","iopub.execute_input":"2023-04-10T01:01:56.909331Z","iopub.status.idle":"2023-04-10T01:01:56.914687Z","shell.execute_reply.started":"2023-04-10T01:01:56.909275Z","shell.execute_reply":"2023-04-10T01:01:56.913161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Configuration ⚙️","metadata":{}},{"cell_type":"code","source":"CONFIG = {\"seed\": 2023,\n          \"epochs\": 3,\n          \"model_name\": \"tf_efficientnet_b0_ns\",\n          \"embedding_size\": 768,\n          \"num_classes\": 264,\n          \"train_batch_size\": 64,\n          \"valid_batch_size\": 128,\n          \"learning_rate\": 1e-4,\n          \"scheduler\": 'CosineAnnealingLR',\n          \"min_lr\": 1e-6,\n          \"T_max\": 500,\n          \"weight_decay\": 1e-6,\n          \"n_fold\": 5,\n          \"n_accumulate\": 1, # gradient accumulation\n          \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n          \"competition\": \"BirdCLEF23\",\n          \"_wandb_kernel\": \"deb\",\n          # Audio Specific\n          \"sample_rate\": 32_000,\n          \"max_time\": 5,\n          \"n_mels\": 224,\n          \"n_fft\": 1024,\n          }","metadata":{"execution":{"iopub.status.busy":"2023-04-10T02:23:52.514801Z","iopub.execute_input":"2023-04-10T02:23:52.515198Z","iopub.status.idle":"2023-04-10T02:23:52.522465Z","shell.execute_reply.started":"2023-04-10T02:23:52.515162Z","shell.execute_reply":"2023-04-10T02:23:52.521389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Seed for Reproducibility","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:56.961448Z","iopub.execute_input":"2023-04-10T01:01:56.961851Z","iopub.status.idle":"2023-04-10T01:01:56.972657Z","shell.execute_reply.started":"2023-04-10T01:01:56.961804Z","shell.execute_reply":"2023-04-10T01:01:56.971684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/birdclef-2023'\nTRAIN_DIR = '../input/birdclef-2023/train_audio'\nTEST_DIR = '../input/birdclef-2023/test_soundscapes'","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:56.974301Z","iopub.execute_input":"2023-04-10T01:01:56.975147Z","iopub.status.idle":"2023-04-10T01:01:56.981115Z","shell.execute_reply.started":"2023-04-10T01:01:56.97511Z","shell.execute_reply":"2023-04-10T01:01:56.980076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(filename):\n    return f\"{TRAIN_DIR}/{filename}\"","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:56.982771Z","iopub.execute_input":"2023-04-10T01:01:56.983736Z","iopub.status.idle":"2023-04-10T01:01:56.988988Z","shell.execute_reply.started":"2023-04-10T01:01:56.983696Z","shell.execute_reply":"2023-04-10T01:01:56.987852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train_metadata.csv\")\ndf['file_path'] = df['filename'].apply(get_train_file_path)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:56.990568Z","iopub.execute_input":"2023-04-10T01:01:56.99164Z","iopub.status.idle":"2023-04-10T01:01:57.083667Z","shell.execute_reply.started":"2023-04-10T01:01:56.991604Z","shell.execute_reply":"2023-04-10T01:01:57.082468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find out the class distribution\ncounts = defaultdict(int)\nfor index, row in df.iterrows():\n    counts[row.primary_label] += 1\n    \nprint(f\"min: {min(counts, key=counts.get)} {min(counts.values())}\")\nprint(f\"max: {max(counts, key=counts.get)} {max(counts.values())}\")\nprint(f\"mean: {sum(counts.values()) / len(counts)}\")\n\nplt.hist(counts.values(), 50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:57.085442Z","iopub.execute_input":"2023-04-10T01:01:57.085918Z","iopub.status.idle":"2023-04-10T01:01:58.170861Z","shell.execute_reply.started":"2023-04-10T01:01:57.08588Z","shell.execute_reply":"2023-04-10T01:01:58.168833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 16941 datapoints, with most classes only containing a few datapoints, which kinda sucks.","metadata":{}},{"cell_type":"code","source":"# Let's grab the first audio file and see what's up\nsample_file = df.loc[0].file_path\nmetadata = torchaudio.info(sample_file)\nprint(metadata)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:58.172561Z","iopub.execute_input":"2023-04-10T01:01:58.172939Z","iopub.status.idle":"2023-04-10T01:01:58.194032Z","shell.execute_reply.started":"2023-04-10T01:01:58.172903Z","shell.execute_reply":"2023-04-10T01:01:58.192899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's try loading the file in\nwaveform, sample_rate = torchaudio.load(sample_file)\ndef plot_waveform(waveform, sample_rate):\n    waveform = waveform.numpy()\n\n    num_channels, num_frames = waveform.shape\n    time_axis = torch.arange(0, num_frames) / sample_rate\n\n    figure, axes = plt.subplots(num_channels, 1)\n    if num_channels == 1:\n        axes = [axes]\n    for c in range(num_channels):\n        axes[c].plot(time_axis, waveform[c], linewidth=1)\n        axes[c].grid(True)\n        if num_channels > 1:\n            axes[c].set_ylabel(f\"Channel {c+1}\")\n    figure.suptitle(\"waveform\")\n    plt.show(block=False)\n    \nplot_waveform(waveform, sample_rate)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:01:58.195821Z","iopub.execute_input":"2023-04-10T01:01:58.196197Z","iopub.status.idle":"2023-04-10T01:02:13.782718Z","shell.execute_reply.started":"2023-04-10T01:01:58.196161Z","shell.execute_reply":"2023-04-10T01:02:13.781383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Audio(waveform.numpy()[0], rate=sample_rate)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.785293Z","iopub.execute_input":"2023-04-10T01:02:13.786603Z","iopub.status.idle":"2023-04-10T01:02:13.866845Z","shell.execute_reply.started":"2023-04-10T01:02:13.786554Z","shell.execute_reply":"2023-04-10T01:02:13.865103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize Audio (Wandb)","metadata":{}},{"cell_type":"code","source":"# run = wandb.init(project=CONFIG['competition'],\n#                  job_type='Visualization',\n#                  name='Audio Visualization',\n#                  anonymous='must')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.876486Z","iopub.execute_input":"2023-04-10T01:02:13.877024Z","iopub.status.idle":"2023-04-10T01:02:13.882163Z","shell.execute_reply.started":"2023-04-10T01:02:13.876966Z","shell.execute_reply":"2023-04-10T01:02:13.881269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview_table = wandb.Table(columns=['Audio', 'Label', 'Rating'])\n\n# temp_df = df.sample(5000).reset_index(drop=True)\n\n# for i in tqdm(range(len(temp_df))):\n#     row = temp_df.loc[i]\n#     audio = wandb.Audio(row.file_path, sample_rate=CONFIG['sample_rate'])\n#     preview_table.add_data(audio,\n#                            row.primary_label,\n#                            row.rating)\n\n# wandb.log({'Visualization': preview_table})\n# run.finish()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.883709Z","iopub.execute_input":"2023-04-10T01:02:13.884394Z","iopub.status.idle":"2023-04-10T01:02:13.891648Z","shell.execute_reply.started":"2023-04-10T01:02:13.88436Z","shell.execute_reply":"2023-04-10T01:02:13.890797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\"><a href=\"https://wandb.ai/dchanda/BirdCLEF23\">View the Complete Table Here ⮕</a></span>","metadata":{}},{"cell_type":"code","source":"# # This is just to display the W&B run page in this interactive session.\n# from IPython import display\n\n# # we create an IFrame and set the width and height\n# iF = display.IFrame(run.url, width=1080, height=720)\n# iF","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-10T01:02:13.893081Z","iopub.execute_input":"2023-04-10T01:02:13.893732Z","iopub.status.idle":"2023-04-10T01:02:13.900536Z","shell.execute_reply.started":"2023-04-10T01:02:13.8937Z","shell.execute_reply":"2023-04-10T01:02:13.899616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\nall_labels = sorted(df['primary_label'].values)\nencoder.fit(all_labels)\n\ndf['primary_label'] = encoder.transform(df['primary_label'])\n\nwith open(\"le.pkl\", \"wb\") as fp:\n    joblib.dump(encoder, fp)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.902067Z","iopub.execute_input":"2023-04-10T01:02:13.902888Z","iopub.status.idle":"2023-04-10T01:02:13.92651Z","shell.execute_reply.started":"2023-04-10T01:02:13.902853Z","shell.execute_reply":"2023-04-10T01:02:13.925655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Folds","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.primary_label)):\n      df.loc[val_ , \"kfold\"] = fold","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.92796Z","iopub.execute_input":"2023-04-10T01:02:13.928608Z","iopub.status.idle":"2023-04-10T01:02:13.949593Z","shell.execute_reply.started":"2023-04-10T01:02:13.928575Z","shell.execute_reply":"2023-04-10T01:02:13.94877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Class","metadata":{}},{"cell_type":"code","source":"class BirdCLEFDataset(Dataset):\n    def __init__(self, df, target_sample_rate, max_time, image_transforms=None):\n        self.file_paths = df['file_path'].values\n        self.labels = df['primary_label'].values\n        self.target_sample_rate = target_sample_rate\n        num_samples = target_sample_rate * max_time\n        self.num_samples = num_samples\n        self.image_transforms = image_transforms\n        \n    def __len__(self):\n        return len(self.file_paths)\n    \n    def __getitem__(self, index):\n        filepath = self.file_paths[index]\n        audio, sample_rate = torchaudio.load(filepath)\n        audio = self.to_mono(audio)\n        \n        if sample_rate != self.target_sample_rate:\n            resample = Resample(sample_rate, self.target_sample_rate)\n            audio = resample(audio)\n        \n        if audio.shape[0] > self.num_samples:\n            audio = self.crop_audio(audio)\n            \n        if audio.shape[0] < self.num_samples:\n            audio = self.pad_audio(audio)\n            \n        mel_spectogram = MelSpectrogram(sample_rate=self.target_sample_rate, \n                                        n_mels=CONFIG['n_mels'], \n                                        n_fft=CONFIG['n_fft'])\n        mel = mel_spectogram(audio)\n        \n        label_onehot = torch.zeros(CONFIG['num_classes'])\n        label_onehot[self.labels[index]] = 1\n        label = torch.tensor(self.labels[index])\n        \n        # Convert to Image\n        image = torch.stack([mel, mel, mel]) # TODO: like, why are we converting to an image?\n        \n        # Normalize Image\n        max_val = torch.abs(image).max()\n        image = image / max_val\n        \n        return {\n            \"image\": image, \n            \"label\": label,\n            \"label_onehot\": label_onehot\n        }\n            \n    def pad_audio(self, audio):\n        pad_length = self.num_samples - audio.shape[0]\n        last_dim_padding = (0, pad_length)\n        audio = F.pad(audio, last_dim_padding)\n        return audio\n        \n    def crop_audio(self, audio):\n        return audio[:self.num_samples]\n        \n    def to_mono(self, audio):\n        return torch.mean(audio, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.951015Z","iopub.execute_input":"2023-04-10T01:02:13.951644Z","iopub.status.idle":"2023-04-10T01:02:13.966091Z","shell.execute_reply.started":"2023-04-10T01:02:13.951611Z","shell.execute_reply":"2023-04-10T01:02:13.96516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GeM Pooling\n\n<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\">Code taken from <a href=\"https://amaarora.github.io/2020/08/30/gempool.html\">GeM Pooling Explained</a></span>\n\n![](https://i.imgur.com/thTgYWG.jpg)","metadata":{}},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.967786Z","iopub.execute_input":"2023-04-10T01:02:13.968602Z","iopub.status.idle":"2023-04-10T01:02:13.979609Z","shell.execute_reply.started":"2023-04-10T01:02:13.968567Z","shell.execute_reply":"2023-04-10T01:02:13.978458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Model","metadata":{}},{"cell_type":"code","source":"class BirdCLEFModel(nn.Module):\n    def __init__(self, model_name, embedding_size, pretrained=True):\n        super(BirdCLEFModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.embedding = nn.Linear(in_features, embedding_size)\n        self.fc = nn.Linear(embedding_size, CONFIG['num_classes'])\n\n    def forward(self, images):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        embedding = self.embedding(pooled_features)\n        output = self.fc(embedding)\n        return output\n    \nmodel = BirdCLEFModel(CONFIG['model_name'], CONFIG['embedding_size'])\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:13.981512Z","iopub.execute_input":"2023-04-10T01:02:13.98237Z","iopub.status.idle":"2023-04-10T01:02:15.201453Z","shell.execute_reply.started":"2023-04-10T01:02:13.982331Z","shell.execute_reply":"2023-04-10T01:02:15.200381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(outputs, labels):\n    return nn.CrossEntropyLoss()(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.202752Z","iopub.execute_input":"2023-04-10T01:02:15.20324Z","iopub.status.idle":"2023-04-10T01:02:15.209099Z","shell.execute_reply.started":"2023-04-10T01:02:15.203199Z","shell.execute_reply":"2023-04-10T01:02:15.207934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    train_acc = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images = data['image'].to(device)\n        labels = data['label'].to(device)\n        \n        batch_size = images.size(0)\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss = loss / CONFIG['n_accumulate']\n        \n#         train_acc += np.sum(np.argmax(outputs.cpu().detach().numpy()) == np.argmax(labels.cpu().detach().numpy()))\n            \n        loss.backward()\n    \n        if (step + 1) % CONFIG['n_accumulate'] == 0:\n            optimizer.step()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    train_acc /= len(dataloader)\n    \n    gc.collect() #garbage collection, presumably to prevent the GPU from blowing up\n    \n    return epoch_loss, train_acc","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.211018Z","iopub.execute_input":"2023-04-10T01:02:15.211429Z","iopub.status.idle":"2023-04-10T01:02:15.224507Z","shell.execute_reply.started":"2023-04-10T01:02:15.211368Z","shell.execute_reply":"2023-04-10T01:02:15.223466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Function","metadata":{}},{"cell_type":"code","source":"def padded_cmap(solution, submission, padding_factor=5): # I don't quite get this part why do you need to pad?\n    new_rows = []\n    for i in range(padding_factor):\n        new_rows.append([1 for i in range(len(solution.columns))])\n    new_rows = pd.DataFrame(new_rows)\n    new_rows.columns = solution.columns\n    padded_solution = pd.concat([solution, new_rows]).reset_index(drop=True).copy()\n    padded_submission = pd.concat([submission, new_rows]).reset_index(drop=True).copy()\n    score = average_precision_score(\n        padded_solution.values,\n        padded_submission.values,\n        average='macro',\n    )\n    return score","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.226089Z","iopub.execute_input":"2023-04-10T01:02:15.227034Z","iopub.status.idle":"2023-04-10T01:02:15.234632Z","shell.execute_reply.started":"2023-04-10T01:02:15.226997Z","shell.execute_reply":"2023-04-10T01:02:15.233897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.inference_mode() #InferenceMode is a new context manager analogous to no_grad to be used when you are certain your operations will have no interactions with autograd (e.g., model training). Code run under this mode gets better performance by disabling view tracking and version counter bumps\ndef valid_one_epoch(model, dataloader, device, epoch):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    LABELS = []\n    PREDS = []\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:        \n        images = data['image'].to(device)\n        labels = data['label_onehot'].to(device)\n        \n        batch_size = images.size(0)\n\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        PREDS.append(F.softmax(outputs, dim=1).cpu().detach().numpy())\n        LABELS.append(labels.cpu().detach().numpy())\n        \n        bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])   \n    \n    LABELS = np.vstack(LABELS)\n    PREDS = np.vstack(PREDS)\n    val_cmap = padded_cmap(pd.DataFrame(LABELS), pd.DataFrame(PREDS))\n    gc.collect()\n    \n    return epoch_loss, val_cmap","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.235967Z","iopub.execute_input":"2023-04-10T01:02:15.237088Z","iopub.status.idle":"2023-04-10T01:02:15.247088Z","shell.execute_reply.started":"2023-04-10T01:02:15.237051Z","shell.execute_reply":"2023-04-10T01:02:15.246365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run the Training Loop","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    # To automatically log gradients\n#     wandb.watch(model, log_freq=100)\n    \n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_cmap = 0\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        train_epoch_loss, train_acc = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=CONFIG['device'], epoch=epoch)\n        \n        val_epoch_loss, val_epoch_cmap = valid_one_epoch(model, valid_loader, \n                                                       device=CONFIG['device'], \n                                                       epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n        history['Valid Loss'].append(val_epoch_loss)\n        history['Valid CMAP'].append(val_epoch_cmap)\n        history['Train Accuracy'].append(train_acc)\n        \n        # Log the metrics\n#         wandb.log({\"Train Loss\": train_epoch_loss})\n#         wandb.log({\"Valid Loss\": val_epoch_loss})\n#         wandb.log({\"Valid CMAP\": val_epoch_cmap})\n        \n        # deep copy the model\n        if val_epoch_cmap >= best_epoch_cmap:\n            print(f\"{b_}Validation CMAP Improved ({best_epoch_cmap} ---> {val_epoch_cmap})\")\n            best_epoch_cmap = val_epoch_cmap\n#             run.summary[\"Best CMAP Score\"] = best_epoch_cmap\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = \"CMAP{:.4f}_epoch{:.0f}.bin\".format(best_epoch_cmap, epoch)\n            torch.save(model.state_dict(), PATH)\n            # Save a model file from the current directory\n            print(f\"Model Saved{sr_}\")\n            \n        print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best CMAP: {:.4f}\".format(best_epoch_cmap))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.248727Z","iopub.execute_input":"2023-04-10T01:02:15.249464Z","iopub.status.idle":"2023-04-10T01:02:15.262198Z","shell.execute_reply.started":"2023-04-10T01:02:15.249427Z","shell.execute_reply":"2023-04-10T01:02:15.261156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CONFIG['scheduler'] == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n                                                   eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n                                                             eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == None:\n        return None\n        \n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.263888Z","iopub.execute_input":"2023-04-10T01:02:15.264168Z","iopub.status.idle":"2023-04-10T01:02:15.27423Z","shell.execute_reply.started":"2023-04-10T01:02:15.264136Z","shell.execute_reply":"2023-04-10T01:02:15.273205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    train_dataset = BirdCLEFDataset(df_train, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n    valid_dataset = BirdCLEFDataset(df_valid, target_sample_rate=CONFIG['sample_rate'], max_time=CONFIG['max_time'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.277911Z","iopub.execute_input":"2023-04-10T01:02:15.278247Z","iopub.status.idle":"2023-04-10T01:02:15.287189Z","shell.execute_reply.started":"2023-04-10T01:02:15.278216Z","shell.execute_reply":"2023-04-10T01:02:15.28637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\">Prepare Dataloaders</span>","metadata":{}},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.288633Z","iopub.execute_input":"2023-04-10T01:02:15.290031Z","iopub.status.idle":"2023-04-10T01:02:15.30383Z","shell.execute_reply.started":"2023-04-10T01:02:15.290003Z","shell.execute_reply":"2023-04-10T01:02:15.302892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\">Define Optimizer and Scheduler</span>","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n                       weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.305386Z","iopub.execute_input":"2023-04-10T01:02:15.305768Z","iopub.status.idle":"2023-04-10T01:02:15.312561Z","shell.execute_reply.started":"2023-04-10T01:02:15.30573Z","shell.execute_reply":"2023-04-10T01:02:15.311182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\">Start Training</span>","metadata":{}},{"cell_type":"code","source":"# run = wandb.init(project=CONFIG['competition'], \n#                  config=CONFIG,\n#                  job_type='Train',\n#                  tags=['gem-pooling', CONFIG['model_name']],\n#                  anonymous='must')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T01:02:15.314237Z","iopub.execute_input":"2023-04-10T01:02:15.315393Z","iopub.status.idle":"2023-04-10T01:02:15.322807Z","shell.execute_reply.started":"2023-04-10T01:02:15.315347Z","shell.execute_reply":"2023-04-10T01:02:15.321362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n                              device=CONFIG['device'],\n                              num_epochs=CONFIG['epochs'])# wtf is this stupid ass shit error","metadata":{"execution":{"iopub.status.busy":"2023-04-10T02:24:29.758348Z","iopub.execute_input":"2023-04-10T02:24:29.758745Z","iopub.status.idle":"2023-04-10T03:03:45.749099Z","shell.execute_reply.started":"2023-04-10T02:24:29.758711Z","shell.execute_reply":"2023-04-10T03:03:45.747895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run.finish()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T02:22:18.976988Z","iopub.status.idle":"2023-04-10T02:22:18.977887Z","shell.execute_reply.started":"2023-04-10T02:22:18.977593Z","shell.execute_reply":"2023-04-10T02:22:18.977623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpreting Results\n\n<span style=\"color: #000508; font-family: Segoe UI; font-size: 1.5em; font-weight: 300;\"><a href=\"https://wandb.ai/dchanda/BirdCLEF23\">View the Complete Dashboard Here ⮕</a></span>","metadata":{}},{"cell_type":"code","source":"# # This is just to display the W&B run page in this interactive session.\n# from IPython import display\n\n# # we create an IFrame and set the width and height\n# iF = display.IFrame(run.url, width=1080, height=720)\n# iF","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-10T02:22:18.979511Z","iopub.status.idle":"2023-04-10T02:22:18.980362Z","shell.execute_reply.started":"2023-04-10T02:22:18.98004Z","shell.execute_reply":"2023-04-10T02:22:18.980066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO: add some validation metrics (ROC, F1 etc.)\n# TODO: plot the validation vs training accuracy\nplt.plot(history['Train Loss'])\nplt.plot(history['Valid Loss'])\nplt.legend(['train', 'val'], loc='upper left')\nplt.xlabel('epoch')\nplt.ylabel('loss')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T03:14:54.024067Z","iopub.execute_input":"2023-04-10T03:14:54.024485Z","iopub.status.idle":"2023-04-10T03:14:54.26084Z","shell.execute_reply.started":"2023-04-10T03:14:54.024449Z","shell.execute_reply":"2023-04-10T03:14:54.259729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![Upvote!](https://img.shields.io/badge/Upvote-If%20you%20like%20my%20work-07b3c8?style=for-the-badge&logo=kaggle)","metadata":{}},{"cell_type":"code","source":"plt.plot(history['Valid CMAP'])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T03:15:45.72427Z","iopub.execute_input":"2023-04-10T03:15:45.725044Z","iopub.status.idle":"2023-04-10T03:15:45.948442Z","shell.execute_reply.started":"2023-04-10T03:15:45.725005Z","shell.execute_reply":"2023-04-10T03:15:45.947302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}