{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Bibliography\n\nArai, H. (2020). 6th place solution and some thoughts. Retrieved\nfrom https://www.kaggle.com/competitions/birdsong-recognition/discussion/183204\n\nHenkel, C., Pfeiffer, P., & Singer, P. (2021). Recognizing bird species in diverse\nsoundscapes under weak supervision. \n\nHidehisa, A. (2020). Introduction to Sound Event Detection. https://kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection\n\nKong, Q., Cao, Y., Iqbal, T., Wang, Y., Wang, W., & Plumbley, M. D. (2020). Panns: Large-scale pretrained audio neural networks for audio pattern recognition (No. arXiv:1912.10211). Retrieved from http://arxiv.org/abs/1912.10211 (arXiv:1912.10211 [cs, eess] type: article) doi: 10.48550/arXiv.1912.10211\n","metadata":{}},{"cell_type":"code","source":"import sys\nimport json\nimport os\nimport typing\nimport importlib.util\nfrom hashlib import sha1\nfrom types import SimpleNamespace\nfrom librosa import display\nimport librosa\nimport numpy as np\nimport pandas as pd\nimport pytorch_lightning as pl\nimport torch\nimport torchaudio as ta\nfrom torch import Tensor, nn\nfrom torch.utils import data\n\nimport sklearn.metrics as metrics\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-13T04:21:30.73853Z","iopub.execute_input":"2022-06-13T04:21:30.73878Z","iopub.status.idle":"2022-06-13T04:21:30.744371Z","shell.execute_reply.started":"2022-06-13T04:21:30.738751Z","shell.execute_reply":"2022-06-13T04:21:30.743441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resources = SimpleNamespace() # Contains filenames\ndataset   = SimpleNamespace() # Contains datasets, data loaders, label mappings\ncfg       = SimpleNamespace() # Contains hyper-parameters","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:30.766846Z","iopub.execute_input":"2022-06-13T04:21:30.767136Z","iopub.status.idle":"2022-06-13T04:21:30.771173Z","shell.execute_reply.started":"2022-06-13T04:21:30.767107Z","shell.execute_reply":"2022-06-13T04:21:30.770303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Paths","metadata":{}},{"cell_type":"code","source":"# Base path for kaggle data\nresources.base = \"/kaggle\"\n# Directory with model and weights\nresources.model = f\"{resources.base}/input/pannscnn14att\"\n\nresources.last_checkpoint = f\"{resources.base}/input/pannscnn14att/PANNsCNN14Att_BirdClef2022.ckpt\"\n\n# Pre-trained weights from Kong et al. (2020) https://zenodo.org/record/3987831. Trained on AudioSet\n# I use the \"Cnn14_DecisionLevelAtt_mAP\" variant\n# My checkpoints are fine tuned on-top of this \nresources.panns_cnn14_att_weights = f\"{resources.base}/input/pannscnn14att/Cnn14_DecisionLevelAtt_mAP0.425.pth\"\n\n# pytorch lightning checkpoint for training\nresources.last_checkpoint = f\"{resources.base}/input/pannscnn14att/PANNsCNN14Att_BirdClef2022.ckpt\"\n\n# CSV file describing the dataset\nresources.data_index   = f\"{resources.base}/input/birdclef-2022/train_metadata.csv\"\n# JSON file containing birds used for scoring\nresources.scored_birds = f\"{resources.base}/input/birdclef-2022/scored_birds.json\"\n\n# Base path for training data\nresources.train_audio  = f\"{resources.base}/input/birdclef-2022/train_audio\"\n# Base path for test data\nresources.test_soundscapes = f\"{resources.base}/input/birdclef-2022/test_soundscapes\"\n\n# Where to save for submission\nresources.save_submission  = f\"{resources.base}/working/submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:30.792671Z","iopub.execute_input":"2022-06-13T04:21:30.792955Z","iopub.status.idle":"2022-06-13T04:21:30.798317Z","shell.execute_reply.started":"2022-06-13T04:21:30.792927Z","shell.execute_reply":"2022-06-13T04:21:30.797633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import model architecture.\n# Kong et al. (2020)'s Cnn14_DecisionLevelAtt with minor modification\n# Available https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/models.py\nsys.path.append(resources.model)\nfrom PANNsCNN14Att import PANNsCNN14Att","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:30.82057Z","iopub.execute_input":"2022-06-13T04:21:30.820863Z","iopub.status.idle":"2022-06-13T04:21:30.825083Z","shell.execute_reply.started":"2022-06-13T04:21:30.820835Z","shell.execute_reply":"2022-06-13T04:21:30.823966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Setup","metadata":{}},{"cell_type":"code","source":"dataset.full_data_index = pd.read_csv(resources.data_index)\ndataset.labels = list(dataset.full_data_index[\"primary_label\"].unique())\n\n# Mapping from bird name -> numeric label\ndataset.class_mappings = dict(((label, numeric_label) for numeric_label, label in enumerate(dataset.labels)))\n\n# Load scored bird information\nwith open(resources.scored_birds) as f:\n    dataset.scored_birds = json.load(f)\n    # numeric label -> scored bird name\n    dataset.scored_birds_mapping = dict(\n        (dataset.class_mappings[bird], bird) for bird in dataset.scored_birds)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:30.851016Z","iopub.execute_input":"2022-06-13T04:21:30.851446Z","iopub.status.idle":"2022-06-13T04:21:30.920477Z","shell.execute_reply.started":"2022-06-13T04:21:30.851417Z","shell.execute_reply":"2022-06-13T04:21:30.919736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define datasets\n\ndef class_balanced_sample(data_index: pd.DataFrame, n: int, random_state=None) -> pd.DataFrame: \n    \"\"\"\n    Class balanced sample that avoids duplicate data as much as possible. \n    Majority classes are under sampled (sampling without replacement). Minority\n    classes are over sampled (sampling with replacement).\n\n    :param data_index: Data frame defining a dataset\n    :param n: Number of instances that should exist per class\n    :param random_state: Random state to ensure deterministic sampling\n    :return: A new data index balanced by 'primary_label'\n    \"\"\"    \n    gt_n = data_index.value_counts(\"primary_label\", sort=False) > n\n    majority_labels = gt_n.index[gt_n]\n    minority_labels = gt_n.index[~gt_n]\n\n    # Under sample (sample without replacement)\n    majority = data_index[data_index[\"primary_label\"].isin(majority_labels)]\n    majority = majority.groupby(\"primary_label\").sample(n=n, random_state=random_state)\n\n    # Over sample (sample with replacement)\n    minority = data_index[data_index[\"primary_label\"].isin(minority_labels)]\n    minority = minority.groupby(\"primary_label\").sample(n=n, replace=True, random_state=random_state)\n\n    return pd.concat((majority, minority)).reset_index(drop=True)\n    \n\ndef create_test_train_split(data_index: pd.DataFrame) -> typing.Tuple[pd.DataFrame, pd.DataFrame]:\n    \"\"\"Generate a class balanced test train split in a deterministic way. \n    Ensure all classes are represented in both splits.\n\n    :param data_index: Data frame defining a dataset\n    :return: A test and train data index\n    \"\"\"\n\n    # Handle a unicorn by using it in both the training and testing set\n    unicorn = data_index.value_counts(\"primary_label\", sort=False) == 1\n    unicorn = unicorn.index[unicorn]\n\n    unicorn_data = data_index[data_index[\"primary_label\"].isin(unicorn)]\n    data_index = data_index.drop(unicorn_data.index)\n\n    # Ensure that at least one of each kind is in each set\n    minimum_train = data_index.groupby(\"primary_label\").sample(n=1, random_state=1)\n    data_index = data_index.drop(minimum_train.index)\n    minimum_test = data_index.groupby(\"primary_label\").sample(n=1, random_state=2)\n    data_index = data_index.drop(minimum_test.index)\n    \n    # Perform regular split\n    split_larger = data_index.groupby(\"primary_label\").sample(frac=0.8, random_state=3)\n    split_smaller  = data_index.drop(split_larger.index)\n\n    train_index = pd.concat((unicorn_data, minimum_train, split_larger), ignore_index=True).reset_index(drop=True)\n    test_index = pd.concat((unicorn_data, minimum_test, split_smaller), ignore_index=True).reset_index(drop=True)\n\n    # Resample data\n    train_index = class_balanced_sample(train_index, 200, random_state=4)\n    test_index = class_balanced_sample(test_index, 40, random_state=5)\n\n    # Assert that we are using the same split as before\n    train_digest = sha1(bytes(train_index.to_csv(), encoding=\"utf-8\")).hexdigest()\n    test_digest  = sha1(bytes(test_index.to_csv(), encoding=\"utf-8\")).hexdigest()\n    print(f\"{train_digest} {test_digest}\")\n    assert \"8701e71c6cf468e7fbf761a195213b24446d045c\" == train_digest, \"Something changed with the training dataset\"\n    assert \"b69eacc9f5204eca6663a4c4ebf11aad8d201184\" == test_digest, \"Something changed with the test dataset\"\n\n    return train_index, test_index\n\ndef only_scored(data_index: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Filter so the data index only contains scored birds\n\n    :param data_index: Data frame defining a dataset\n    :return: A data frame defining the filtered dataset\n    \"\"\"\n    return data_index[data_index[\"primary_label\"].isin(dataset.scored_birds)].reset_index(drop=True)\n\ndataset.train_index, dataset.test_index = create_test_train_split(dataset.full_data_index)\ndataset.scored_test = only_scored(dataset.test_index)\ndataset.scored_train_index = only_scored(dataset.train_index)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:30.922161Z","iopub.execute_input":"2022-06-13T04:21:30.922537Z","iopub.status.idle":"2022-06-13T04:21:32.142219Z","shell.execute_reply.started":"2022-06-13T04:21:30.922499Z","shell.execute_reply":"2022-06-13T04:21:32.14146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Configuration","metadata":{}},{"cell_type":"code","source":"# Configure hyper-parameters\n\n# https://www.kaggle.com/competitions/birdsong-recognition/discussion/183204\n# People reckon that 30s gives us a very good chance that our sample contains the data\n# we want\ncfg.duration = 30\n\ncfg.batch_size = 16 # Batch size has to be small to fit things into memory\ncfg.threshold = 0.5\ncfg.lr = 0.001\ncfg.epochs = 1\n\ncfg.sample_rate = 32000\ncfg.window_size = 1024\ncfg.hop_size = 320\ncfg.mel_bins = 64\ncfg.fmin = 50\ncfg.fmax = 14000\ncfg.num_classes = 152\n\ncfg.chunk_duration = 5.0\n\n# The pre-trained model has 527 classes\ncfg.pre_trained_num_classes = 527\n\n# Submission Config\ncfg.submission_threshold = 0.01","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:32.143363Z","iopub.execute_input":"2022-06-13T04:21:32.14393Z","iopub.status.idle":"2022-06-13T04:21:32.151875Z","shell.execute_reply.started":"2022-06-13T04:21:32.14389Z","shell.execute_reply":"2022-06-13T04:21:32.150748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Loaders","metadata":{}},{"cell_type":"code","source":"class BIRDClefWaveformDataSet(data.Dataset):\n\n    def __init__(self, data_index: pd.DataFrame, waveform_transform=None):\n        super().__init__()\n        self.data_index = data_index\n        self.waveform_transform = waveform_transform\n\n    def get_random_example(self) -> typing.Tuple[Tensor, Tensor, pd.DataFrame]:\n        idx = np.random.randint(len(self.data_index))\n        metadata = self.data_index.iloc[idx]\n        waveform, labels = self.load_example(idx, False)\n        return waveform, labels, metadata[\"filename\"]\n\n    def __len__(self):\n        return len(self.data_index)\n\n    def load_example(self, idx: int, with_transform: bool) -> Tensor:\n        \"\"\"Load an example from the data index at index idx\n\n        :param idx: An index into the data index\n        :param with_transform: Whether to apply transform or not\n        :return: A tensor of the waveform\n        \"\"\"\n        row = self.data_index.iloc[idx]\n        filename = os.path.join(resources.train_audio, row[\"filename\"])\n\n        # Get the waveform samples and the sample rate from a file\n        waveform, sr = ta.load(filename)\n\n        # Ensure the audio is mono channel\n        waveform = torch.mean(waveform, dim=0)\n\n        if self.waveform_transform != None and with_transform:\n            waveform, _ = self.waveform_transform(waveform, sr)\n\n        # Convert text labels into numeric labels\n        primary_label = dataset.class_mappings[row[\"primary_label\"]]\n        secondary_labels = list(map(dataset.class_mappings.get, eval(row[\"secondary_labels\"])))\n\n        # Populate target distribution\n        target = torch.zeros(cfg.num_classes)\n        target[primary_label]   = 1.0\n        target[secondary_labels] = 0.99\n\n        return waveform, target\n\n    def __getitem__(self, idx: int):\n        return self.load_example(idx, True)\n\nclass RandomWaveformCrop(nn.Module):\n    \"\"\"\n    Randomly crop a waveform to be a certain duration. If it is too small\n    we will pad it to fit.\n    \"\"\"\n\n    def __init__(self, duration: float):\n        super().__init__()\n        self.duration = duration\n\n    def forward(self, waveform: Tensor, sr: int):\n        length = waveform.shape[0]\n        new_length = sr * self.duration\n\n        if new_length < length:\n            # Crop the waveform because it is too bit\n            offset = np.random.randint(length-new_length)\n            waveform = waveform[offset:offset+new_length]\n        elif length < new_length:\n            # Pad the waveform because it is too small\n            new_waveform = torch.zeros((new_length))\n            offset = np.random.randint(new_length - length)\n            new_waveform[offset:offset+length] = waveform\n            waveform = new_waveform\n\n        return waveform, sr","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:32.153894Z","iopub.execute_input":"2022-06-13T04:21:32.154162Z","iopub.status.idle":"2022-06-13T04:21:32.170785Z","shell.execute_reply.started":"2022-06-13T04:21:32.154128Z","shell.execute_reply":"2022-06-13T04:21:32.169811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup data loaders\ndataset.train_dataset       = BIRDClefWaveformDataSet(dataset.train_index, RandomWaveformCrop(cfg.duration))\ndataset.test_dataset        = BIRDClefWaveformDataSet(dataset.test_index, RandomWaveformCrop(cfg.duration))\ndataset.scored_test_dataset = BIRDClefWaveformDataSet(dataset.scored_test, RandomWaveformCrop(cfg.duration))\ndataset.scored_train_dataset = BIRDClefWaveformDataSet(dataset.scored_train_index, RandomWaveformCrop(cfg.duration))\n\n\ndataset.train_loader = data.DataLoader(dataset.train_dataset, batch_size=cfg.batch_size, shuffle=True, num_workers=2)\ndataset.test_loader = data.DataLoader(dataset.test_dataset, batch_size=cfg.batch_size, shuffle=False, num_workers=2)\ndataset.scored_test_loader = data.DataLoader(dataset.scored_test_dataset, batch_size=cfg.batch_size, shuffle=False, num_workers=2)\ndataset.scored_train_loader = data.DataLoader(dataset.scored_train_dataset, batch_size=cfg.batch_size, shuffle=True, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:32.172236Z","iopub.execute_input":"2022-06-13T04:21:32.172687Z","iopub.status.idle":"2022-06-13T04:21:32.182392Z","shell.execute_reply.started":"2022-06-13T04:21:32.172652Z","shell.execute_reply":"2022-06-13T04:21:32.181593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class MyLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.bce = nn.BCELoss(reduction=\"sum\")\n\n    def forward(self, output: dict, target: Tensor):\n        return self.bce(output[\"clipwise_output\"], target)\n\ndef filter_bad_numbers(x: Tensor):\n    \"\"\"Remove infinities, NaNs, and force numbers to between 0 and 1. This is an\n    ugly hack. \n    \n    I'm not sure why it is necessary but I'm not the only one to need it:\n    https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection\n\n    :param x: A tensor to clean\n    :return: A clean tensor\n    \"\"\"\n    x = torch.where(torch.isnan(x), torch.zeros_like(x), x)\n    x = torch.where(torch.isinf(x), torch.zeros_like(x), x)\n    x = x.clamp(0, 1)\n    return x\n\nclass MyBirdSED(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.PANNs = PANNsCNN14Att(\n            cfg.sample_rate, \n            cfg.window_size, \n            cfg.hop_size, \n            cfg.mel_bins, \n            cfg.fmin, \n            cfg.fmax, \n            cfg.pre_trained_num_classes\n        )\n\n        self.criterion = MyLoss()\n        self.eval_y_true = []\n        self.eval_y_pred = []\n        self.train_y_true = []\n        self.train_y_pred = []\n\n    @staticmethod\n    def load_pre_trained():\n        model = MyBirdSED()\n        weights = torch.load(resources.panns_cnn14_att_weights)\n        model.PANNs.load_state_dict(weights[\"model\"], strict=False)\n        return model\n\n    def forward(self, waveforms, mixup_lambda=None):\n        output = self.PANNs.forward(waveforms)\n        output['clipwise_output'] = filter_bad_numbers(output['clipwise_output'][:,:cfg.num_classes])\n        output['framewise_output'] = filter_bad_numbers(output['framewise_output'][:,:,:cfg.num_classes])\n        return output\n\n    def training_step(self, batch, batch_idx):\n        x, target = batch\n\n        output = self.forward(x)\n        loss = self.criterion(output, target)\n\n\n        # Store information for statistics\n        self.train_y_true.append(target.detach())\n        self.train_y_pred.append(output[\"clipwise_output\"].detach())\n\n        # Log\n        self.log(\"train_loss\", loss, on_epoch=True, batch_size=cfg.batch_size)\n        return loss\n\n\n    def training_epoch_start(self):\n        self.train_y_true = []\n        self.train_y_pred = []\n\n    def training_epoch_end(self, step_outputs):\n        y_true = torch.concat(self.train_y_true, dim=0).cpu().numpy() > cfg.threshold\n        y_pred = torch.concat(self.train_y_pred, dim=0).cpu().numpy() > cfg.threshold\n        \n        self.log(\"train_f1\", metrics.f1_score(y_true, y_pred, average=\"macro\", zero_division=1))\n        self.log(\"train_accuracy\", metrics.accuracy_score(y_true, y_pred))\n\n    def validation_step(self, batch: typing.Tuple[Tensor, Tensor], batch_idx):\n        x, target = batch\n\n        output = self.forward(x)\n        loss = self.criterion(output, target)\n\n        # Store information for statistics\n        self.eval_y_true.append(target)\n        self.eval_y_pred.append(output[\"clipwise_output\"])\n\n        # Log\n        self.log(\"test_loss\", loss, on_epoch=True, batch_size=cfg.batch_size)\n        return loss\n\n    def validation_epoch_start(self):\n        self.eval_y_true = []\n        self.eval_y_pred = []\n\n\n    def validation_epoch_end(self, validation_step_outputs):\n        y_true = torch.concat(self.eval_y_true, dim=0).cpu().numpy() > cfg.threshold\n        y_pred = torch.concat(self.eval_y_pred, dim=0).cpu().numpy() > cfg.threshold\n        \n        self.log(\"eval_f1\", metrics.f1_score(y_true, y_pred, average=\"macro\", zero_division=1))\n        self.log(\"eval_accuracy\", metrics.accuracy_score(y_true, y_pred))\n\n\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), cfg.lr)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:32.18383Z","iopub.execute_input":"2022-06-13T04:21:32.184326Z","iopub.status.idle":"2022-06-13T04:21:32.205835Z","shell.execute_reply.started":"2022-06-13T04:21:32.184289Z","shell.execute_reply":"2022-06-13T04:21:32.205055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Save time by using a model prepared earlier\n# model = MyBirdSED.load_from_checkpoint(resources.last_checkpoint)\n\n# Fine tune the model\nmodel = MyBirdSED.load_pre_trained()","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:32.207147Z","iopub.execute_input":"2022-06-13T04:21:32.207505Z","iopub.status.idle":"2022-06-13T04:21:35.801674Z","shell.execute_reply.started":"2022-06-13T04:21:32.207467Z","shell.execute_reply":"2022-06-13T04:21:35.80095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load checkpoint\ntrainer = pl.Trainer(\n    gpus=1,\n    max_epochs=cfg.epochs,\n)\n\ntrainer.fit(model, dataset.train_loader, dataset.test_loader)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:46.298408Z","iopub.execute_input":"2022-06-13T04:21:46.298688Z","iopub.status.idle":"2022-06-13T04:23:08.500082Z","shell.execute_reply.started":"2022-06-13T04:21:46.298658Z","shell.execute_reply":"2022-06-13T04:23:08.499116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot CM\n\ndef highlight_cell(x: int, y: int, ax=None, **kwargs):\n    \"\"\"\n    Highlight a cell at x, y\n\n    :param x: Coordinate to highlight\n    :param y: Coordinate to highlight\n    :param ax: pyplot axis, defaults to None\n    \"\"\"\n    rect = plt.Rectangle((x-.5, y-.5), 1,1, fill=False, **kwargs)\n    ax = ax or plt.gca()\n    ax.add_patch(rect)\n\ndef plot_cm(cm: Tensor):\n    \"\"\"\n    Plot a confusion matrix and highlight scored birds\n\n    :param cm: A tensor representing the confusion matrix\n    \"\"\"\n    x = range(0, cfg.num_classes)\n    \n    plt.figure(figsize=(40, 40))\n    plt.imshow(cm)\n    plt.xticks(x, dataset.labels, rotation=90)\n    plt.yticks(x, dataset.labels)\n\n    # Highlight scored birds\n    for x in dataset.scored_birds_mapping.keys():\n        highlight_cell(x, x, color=\"limegreen\", linewidth=3)\n\n    ax = plt.gca() # get current axis\n    ax.set_xticks(np.arange(-.5, cfg.num_classes, 5), minor=True)\n    ax.set_yticks(np.arange(-.5, cfg.num_classes, 5), minor=True)\n    ax.grid(which='minor', color='grey', linestyle='-', linewidth=1)\n\ny_true = torch.concat(model.eval_y_true).argmax(1).cpu().numpy()\ny_pred = torch.concat(model.eval_y_pred).argmax(1).cpu().numpy()\n\nplot_cm(metrics.confusion_matrix(y_true, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.834355Z","iopub.status.idle":"2022-06-13T04:21:35.834829Z","shell.execute_reply.started":"2022-06-13T04:21:35.834596Z","shell.execute_reply":"2022-06-13T04:21:35.83462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def frame_to_seconds(frame: int) -> float:\n    \"\"\"\n    Because PANNs14Att notion of time is through frame indices, we need a way\n    to convert to seconds.\n\n    :param frame: A frame index\n    :return: Offset of the frame index in seconds\n    \"\"\"\n    return (frame*cfg.hop_size)/cfg.sample_rate\n\n\ndef seconds_to_frames(duration: float) -> float:\n    \"\"\"\n    Convert to frame indices from seconds.\n\n    :param seconds: duration time measured in seconds\n    :return: The number of frames in the duration\n    \"\"\"\n    return int((duration*cfg.sample_rate)/cfg.hop_size)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.836229Z","iopub.status.idle":"2022-06-13T04:21:35.83677Z","shell.execute_reply.started":"2022-06-13T04:21:35.836531Z","shell.execute_reply":"2022-06-13T04:21:35.836558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pretty_prediction(y_hat: Tensor, true_bin_vector: Tensor = None):\n    ranked_batch = y_hat.argsort(dim=1, descending=True)\n    \n    for batch_id, ranked in enumerate(ranked_batch):\n        print(f\"{batch_id:03d}\")\n        for rank, x in enumerate(ranked[:5]):\n            print(f\"  {rank:d} {dataset.labels[x]:10} {y_hat[batch_id][x]*100:.2f}%\")\n\n        if true_bin_vector != None:\n            for numeric_label in true_bin_vector[batch_id].nonzero():\n                # print(numeric_label)\n                label = dataset.labels[numeric_label]\n                rank = ranked.eq(numeric_label).nonzero(as_tuple=True)[0]\n                print(f\"  Actual Class: {int(rank)} {label:8} {float(y_hat[batch_id][numeric_label]*100):.2f}%\")\n\ndef plot_framewise_output(waveform: Tensor, framewise_output: Tensor, chunkwise_output: Tensor, top_k_labels: Tensor):\n\n    fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(20, 10), sharex=True)\n\n    # Axes 0 is a spectrogram\n    D = librosa.stft(waveform.cpu().numpy())\n    S_db = librosa.amplitude_to_db(np.abs(D), ref=np.max)\n    librosa.display.specshow(S_db, sr=cfg.sample_rate, x_axis='time', ax=ax[0])\n\n    # Axes 1 is the framewise prediction\n    framewise_output = framewise_output.squeeze()[:,top_k_labels].cpu()\n    k = top_k_labels.shape[0]\n    duration = frame_to_seconds(len(framewise_output))\n\n    ax[1].set_title(\"Framewise Prediction\")\n    ax[1].imshow(framewise_output.T.numpy(), \n            interpolation='nearest',\n            vmin=0, vmax=1,\n            extent=[0, duration, 0, k],\n            aspect='auto')\n    ax[1].set_yticks(np.arange(k, 0, -1)-0.5, list(map(dataset.labels.__getitem__, top_k_labels)))\n\n    # Axes 2 is the chunkwise prediction\n    chunkwise_output = chunkwise_output.squeeze(dim=0)[:,top_k_labels].cpu()\n\n    ax[2].set_title(\"Chunkwise (5s) Prediction\")\n    ax[2].imshow(chunkwise_output.T.numpy(), \n            interpolation='nearest',\n            vmin=0, vmax=1,\n            extent=[0, (duration//5)*5, 0, k],\n            aspect='auto')\n    ax[2].set_yticks(np.arange(k, 0, -1)-0.5, list(map(dataset.labels.__getitem__, top_k_labels)))\n\ndef chunkwise_predictions(framewise_output: Tensor):\n    \"\"\"PANNsCNN14Att outputs point wise prediction at the fidelity of a frame.\n    We reshape and aggregate (AdaptiveAvgPool1d) to get a coarser prediction.\n\n    :param framewise_output: output of model of shape (batch_size, frames, classes)\n    :return: coarser chunk wise predictions of shape (batch_size, chunks, classes)\n    \"\"\"\n    batch_size = framewise_output.shape[0]\n    chunk_size = seconds_to_frames(cfg.chunk_duration)\n    n_frames = framewise_output.shape[1]\n\n    output_size = n_frames//chunk_size\n    max_pool = nn.AdaptiveAvgPool1d(output_size)\n\n    max_pool_out = max_pool(framewise_output.transpose(1, 2))\n    max_pool_out = max_pool_out.transpose(1, 2)\n\n    return max_pool_out.reshape((batch_size, -1, cfg.num_classes))\n\nmodel.eval()\nmodel = model.cuda()\n\nwith torch.no_grad():\n    labels = None\n\n    x, labels, metadata = dataset.scored_test_dataset.get_random_example()\n    labels = labels.unsqueeze(0).cuda()\n\n    x = x.cuda().unsqueeze(0)\n    output = model(x)\n    pretty_prediction(output[\"clipwise_output\"], labels)\n    top_k = output[\"clipwise_output\"].argsort(dim=1, descending=True)[0,:20]\n    plot_framewise_output(x[0], output[\"framewise_output\"], chunkwise_predictions(output[\"framewise_output\"]), top_k)\n    print(metadata)\n\nmodel.train()\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.837882Z","iopub.status.idle":"2022-06-13T04:21:35.838432Z","shell.execute_reply.started":"2022-06-13T04:21:35.838205Z","shell.execute_reply":"2022-06-13T04:21:35.83823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"class WaveformIterator():\n\n    def __init__(self, filename:str, window_duration: float):\n        self.waveform, self.sr = ta.load(filename)\n        self.waveform = torch.mean(self.waveform, dim=0)\n        self.offset = 0\n        self.waveform_size = self.waveform.shape[0]\n        self.window_size = self.sr * window_duration\n        self.duration = self.waveform_size/self.sr\n\n    def __iter__(self):\n        self.offset = 0\n        return self\n\n    def __next__(self):\n\n        if self.offset >= self.waveform_size:\n            raise StopIteration()\n        \n        self.offset += self.window_size\n\n        new_waveform = torch.zeros(self.window_size)\n\n        # Copy over some of the original waveform. The rest is zeros\n        chunk_start = self.offset-self.window_size\n        chunk_end   = min(self.offset, self.waveform_size)\n        new_waveform[0:chunk_end-chunk_start] = self.waveform[chunk_start:chunk_end]\n\n        return new_waveform, chunk_start//self.sr\n\nclass SubmissionDataset(data.IterableDataset):\n\n    def __init__(self, directory, window_duration):\n        self.directory = directory\n        self.test_soundscapes = os.listdir(directory)\n        self.waveform_iter = None\n        self.file_iter = None\n        self.window_duration = window_duration\n\n    def __iter__(self):\n        self.file_iter = iter(self.test_soundscapes)\n        self.waveform_iter = iter(range(0))\n        return self\n\n    def __next__(self):\n        try:\n            return next(self.waveform_iter), self.file_prefix, self.chunks_in_window\n        except StopIteration:\n            file = next(self.file_iter)\n            path  = os.path.join(self.directory, file)\n\n            self.waveform_iter = iter(WaveformIterator(path, self.window_duration))\n            self.file_prefix = file[:-len(\".ogg\")]\n\n            self.chunks_in_window = self.waveform_iter.duration // cfg.chunk_duration\n            return next(self)\n             ","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.839506Z","iopub.status.idle":"2022-06-13T04:21:35.840061Z","shell.execute_reply.started":"2022-06-13T04:21:35.839814Z","shell.execute_reply":"2022-06-13T04:21:35.839839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef run_model(x: Tensor, row_prefix: str, chunk_id_offset: int, chunks_in_window):\n    model.eval()\n    output = model.forward(x)\n\n    prediction_results = []\n    for label, bird in dataset.scored_birds_mapping.items():\n        \n        pred = output[\"clipwise_output\"][0][label]\n\n        row_id = f\"{row_prefix}_{bird}_{chunk_id_offset+5}\"\n        # print(f\"{row_id:20} {pred*100:0.2f}\")\n        prediction_results.append((row_id, bool(pred > cfg.submission_threshold)))\n\n    model.train()\n    return prediction_results\n        \nmodel = model.cuda()\nsubmission = []\nfor (x, offset), row_prefix, chunks_in_window in SubmissionDataset(resources.test_soundscapes, 5):\n    x = x.cuda().unsqueeze(0)\n    submission.extend(run_model(x, row_prefix, offset, chunks_in_window))","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.841137Z","iopub.status.idle":"2022-06-13T04:21:35.841675Z","shell.execute_reply.started":"2022-06-13T04:21:35.841437Z","shell.execute_reply":"2022-06-13T04:21:35.841463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(submission, columns=['row_id','target'])\ndf_submission.to_csv(resources.save_submission, index=False)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:21:35.842723Z","iopub.status.idle":"2022-06-13T04:21:35.843264Z","shell.execute_reply.started":"2022-06-13T04:21:35.843025Z","shell.execute_reply":"2022-06-13T04:21:35.843065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}