{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Change this flag to train model, else load it from private database\nTRAIN = False\n\nimport cv2\nimport audioread\nimport logging\nimport os\nimport random\nimport time\nimport warnings\nimport pickle\nimport librosa\nimport librosa.display as display\nimport numpy as np\nimport pandas as pd\nimport soundfile as sf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as data\n\nfrom contextlib import contextmanager\nfrom IPython.display import Audio\nfrom pathlib import Path\nfrom typing import Optional, List\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score, average_precision_score\n\n\n!pip install ../input/bird-panns/torchlibrosa-master/torchlibrosa-master/\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_ROOT = \"../input\"\nTRAIN_CSV_DIR = INPUT_ROOT + '/birdsong-recognition/train.csv'\nTRAIN_DIRS = [\n  INPUT_ROOT + \"/birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\nTEST_DIR = INPUT_ROOT + '/birdsong-recognition/test_audio'\nTEST_CSV_DIR = INPUT_ROOT + '/birdsong-recognition/test.csv'\n\nFIRST_LETTER_TO_FOLDER_MAP = {\n    'a': 0, 'b': 0, 'c': 1, 'd': 1, 'e': 1, \n    'f': 1, 'g': 2, 'h': 2, 'i': 2, 'j': 2, \n    'k': 2, 'l': 2, 'm': 2, 'n': 3, 'o': 3,\n    'p': 3, 'q': 3, 'r': 3, 's': 4, 't': 4,\n    'u': 4, 'v': 4, 'w': 4, 'x': 4, 'y': 4\n}\n\nn_epochs = 23\n\n\nLOAD_BEST_WEIGTHS = False\n\nbest_weights_path = '../input/backtoorigins/fold0/checkpoints/last.pth'\nbest_species_path = '../input/backtoorigins/species.pickle'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Csv Analyzer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# df = BirdcallCsv().df\n# df.head()\n# df[df['duration'] >= 5][df['duration'] < 60].groupby('ebird_code').agg('sum').describe()#['duration'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import ast \nfrom collections import Counter\n    \nclass BirdcallCsv:\n    def __init__(self, min_n_species=9, secondary_target_contribution=1, species_pickle_path=None):\n        \n        self.min_n_occurrences = min_n_species\n        self.secondary_target_contribution = secondary_target_contribution\n        self.species_pickle_path = species_pickle_path\n        \n        self.df = pd.read_csv(TRAIN_CSV_DIR)\n        self.df = self.df[self.df['duration'] >= 5] # Your dirty sampling # 100\n        print(self.df.shape)\n        self._create_audio_paths()\n        self._add_additional_folders()\n        self._count_species_occurrences()\n        self._create_maps()\n        self._create_targets()\n        \n        self.df = self.df[['ebird_code', 'species', 'secondary_labels', 'audio_path', 'target', 'primary']]\n        \n    def train_val_split_df(self, val_ratio=0.1):\n        x, x_val, y, y_val = train_test_split(self.df, \n                                self.df[\"ebird_code\"], \n                                stratify=self.df[\"ebird_code\"],\n                                test_size=val_ratio)\n        return x, x_val\n    \n    def _add_additional_folders(self):\n        folders_dict = {\n            'squirrel':'../input/pannrelatedmodules/squirrels'\n        }\n        for class_name, path in folders_dict.items():\n            self._add_folder_to_csv(class_name, path)\n            \n        \n    def _add_folder_to_csv(self, class_name, folder_path):\n        rows = []\n        for file_name in os.listdir(folder_path):\n            file_path = os.path.join(folder_path, file_name)\n            row = {\n                'ebird_code':'',\n                'audio_path':file_path,\n                'species':class_name,\n                'secondary_labels':'[]'\n            }\n            rows.append(row)\n        self.df = c = pd.concat([pd.DataFrame(rows),self.df],ignore_index=True)\n        \n    def _create_audio_paths(self):\n        def _create_audio_path(row):\n            first_letter = row['ebird_code'][0]\n            folder_number = FIRST_LETTER_TO_FOLDER_MAP[first_letter]\n            folder_path = TRAIN_DIRS[folder_number]\n\n            return os.path.join(\n                folder_path,\n                row['ebird_code'], \n                row['filename'].replace('mp3', 'wav')\n            )\n    \n        self.df['audio_path'] = self.df.apply(_create_audio_path, axis=1)\n    \n    def _create_maps(self):\n        def _create_species_to_int_map():\n            bool_mask = self.occurrences_df[0] >= self.min_n_occurrences\n            species_filtered = self.occurrences_df[bool_mask]\n            \n            print(\"Before filtering:\", self.df.shape)\n            self.df = self.df[self.df['species'].isin(species_filtered.index)]\n            print('after filtering:', self.df.shape)\n            self.species_to_int_map = dict(\n                 zip(\n                     species_filtered.index, \n                     range(species_filtered.shape[0])\n                 )\n             )\n\n        def _create_int_to_species_map():\n            self.int_to_species_map = {\n                v: k for k, v in self.species_to_int_map.items()\n            }\n\n        def _create_species_to_code_map():\n            species_to_code_map = self.df.groupby('species').first()\n            self.species_to_code_map = {\n                k:r['ebird_code'] \n                for k, r in species_to_code_map.iterrows()\n            }\n        \n        def _create_int_to_code_map():\n            \n            self.int_to_code_map = {\n                i:self.species_to_code_map.get(s, '')\n                for i, s in self.int_to_species_map.items() \n            }\n        \n        if self.species_pickle_path is not None:\n            self.int_to_species_map = pickle.load(open(self.species_pickle_path,'rb'))\n            self.species_to_int_map = {v:k for k,v in self.int_to_species_map.items()}\n        else:\n            _create_species_to_int_map()\n            _create_int_to_species_map()\n        _create_species_to_code_map()\n        _create_int_to_code_map()\n        \n        pickle_out = open(\"classes.pickle\",\"wb\")\n        pickle.dump(self.int_to_code_map, pickle_out)\n        pickle_out.close()\n        \n        pickle_out = open(\"species.pickle\",\"wb\")\n        pickle.dump(self.int_to_species_map, pickle_out)\n        pickle_out.close()\n        print(\"species saved at species.pickle\")\n        \n    def _create_targets(self):\n        \n        def _create_primary_target(row):\n            n_classes = len(self.species_to_int_map)\n            one_hot_encoding = np.zeros(n_classes)\n            one_hot_encoding[self.species_to_int_map[row['species']]] = 1\n\n            return one_hot_encoding\n        \n        def _create_global_target(row):\n            n_classes = len(self.species_to_int_map)\n\n            species_in_row = self._get_species_in_row(row)\n            many_hot_encoding = np.zeros(n_classes)\n\n            for species in species_in_row:\n                if species in self.species_to_int_map:\n                    ind = self.species_to_int_map[species]\n                    many_hot_encoding[ind] = self.secondary_target_contribution\n\n            many_hot_encoding[self.species_to_int_map[row['species']]] = 1\n\n            return many_hot_encoding\n        \n        self.df['primary'] = self.df.apply(\n            lambda r: _create_primary_target(r),\n            axis=1\n        )\n            \n        self.df['target'] = self.df.apply(\n            lambda r: _create_global_target(r),\n            axis=1\n        )\n    \n    def _get_species_in_row(self, row):\n        \"Returns a list of species names of primary + secondary labels\"\n        \n        sec_species = [\n            l.split('_')[1]\n            for l in ast.literal_eval(row['secondary_labels'])\n        ]\n\n        species = set(sec_species)\n        species.add(row['species'])\n        return species\n        \n    def _count_species_occurrences(self):\n        species_per_example = self.df.apply(self._get_species_in_row, axis = 1)\n\n        species_count = Counter()\n        for species in species_per_example:\n            for spec in species:\n                species_count[spec] += 1\n        self.occurrences_df = pd.DataFrame.from_dict(species_count, orient='index')\n    \n    def label_to_code(self, label):\n        species = self.int_to_species_map[label]\n        if species in self.species_to_code_map:\n            return self.species_to_code_map[species]\n        else:\n            return ''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torch\nfrom sklearn.model_selection import train_test_split\nimport librosa\nimport soundfile as sf\nfrom copy import deepcopy\n\ndef wave_to_frames(wave, frame_size):\n    \"\"\"Transforms a wave into a matrix of dimension (n, frame_size),\n    where n is the number of frames contained in the wave. This means \n    the wave ending of the wave may be truncated by up to frame_size - 1 samples  \n    \"\"\"\n    n = wave.shape[0]\n    if n <= frame_size:\n        new_wave = np.zeros(frame_size)\n        ind = (frame_size - n)//2\n        new_wave[ind:ind+n] = wave\n        return new_wave[np.newaxis,:]\n    \n    n_frames = wave.shape[0]//frame_size\n    frame_matrix = np.reshape(wave[:n_frames*frame_size],\n                            (n_frames, frame_size))\n    return frame_matrix\n \n    \nclass BirdcallDataset(torch.utils.data.Dataset):\n    def __init__(\n        self,\n        df,\n        sr = 32000,\n        batch_sizes_in_seconds=[5],#[5,10,15], \n    ):\n        self.df = df\n        self.sr = sr\n        \n        self.batch_sizes = batch_sizes_in_seconds\n        self.cur_batch_size = self.batch_sizes[0]\n        self.cur_frame_size = int(self.cur_batch_size*self.sr)\n        \n        \n        self.num_classes = self.df.iloc[0]['target'].shape[0]\n        print('Num classes: {}'.format(self.num_classes))\n        \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n\n        y, sr = librosa.load(row[\"audio_path\"], sr=self.sr,\n                               mono=True,\n                               res_type=\"kaiser_fast\")\n\n        len_y = len(y)\n        \n        if len_y < self.cur_frame_size:\n            # Zero Pad at a random location\n            new_y = np.zeros(self.cur_frame_size, dtype=y.dtype)\n            start = np.random.randint(self.cur_frame_size - len_y)\n            new_y[start:start + len_y] = y\n            y = new_y\n        elif len_y > self.cur_frame_size:\n            # truncate at a random location\n            start = np.random.randint(len_y - self.cur_frame_size)\n            new_y = y[start:start + self.cur_frame_size]\n            y = new_y\n\n        return {\n            \"waveform\": y.astype(np.float32), \n            \"targets\": row['target'],\n            \"primary_target\": row['primary']\n        }\n    \n    def __len__(self):\n        return self.df.shape[0]\n\n    def on_new_batch(self):\n        self.cur_batch_size = random.choice(self.batch_sizes)\n        self.cur_frame_size = int(self.cur_batch_size*self.sr)\n\n\nclass TrainingIterDataset(torch.utils.data.IterableDataset):\n    def __init__(self, df, frame_size_in_secs=5, max_mins_per_sample=.5, sr=32000):\n        \n        self.sr = sr\n        \n        self.frame_size_in_secs = frame_size_in_secs\n        self.frame_size = int(frame_size_in_secs*sr)\n        \n        self.max_mins_per_sample = max_mins_per_sample\n        self.max_samples = int(max_mins_per_sample*60*sr)\n        self.max_frames = int(self.max_samples/self.frame_size)\n        self.n_original = df.shape[0]\n        \n        # single-process data loading, return the full iterator\n        self.iter_start = 0\n        self.iter_end = self.n_original\n        \n        self.rows = [\n            {\n                'audio_path':row['audio_path'],\n                'targets':deepcopy(row['target']),  \n                'primary_target':deepcopy(row['primary'])\n            }\n            for i,row in df.iterrows()\n        ]  \n        \n        self._compute_len()\n        \n    def _create_worker_props(self):\n        worker_info = torch.utils.data.get_worker_info()\n                \n        if worker_info is not None:  # in a worker process\n            # split workload\n            per_worker = int(self.n_original / worker_info.num_workers)\n            worker_id = worker_info.id\n            self.iter_start = worker_id * per_worker\n            self.iter_end = min(self.iter_start + per_worker, self.n_original)\n        \n        self._compute_len()\n            \n    def __iter__(self): \n        self._create_worker_props()\n        \n        for row in self.rows[self.iter_start:self.iter_end]:\n            \n            y, sr = sf.read(row['audio_path'])\n        \n            frames = self._create_frames(y)\n\n            for i, frame in enumerate(frames):\n                yield {\n                    \"waveform\": frame, \n                    \"targets\": row['targets'],\n                    \"primary_target\": row['primary_target']\n                }\n        \n    def __len__(self):\n        return self.n_samples\n        \n    \n    def _compute_len(self):\n        rows = self.rows[self.iter_start:self.iter_end]\n        self.n_samples = sum([self._compute_len_sample(r) for r in rows])\n\n    \n    def _compute_len_sample(self, row):\n        with sf.SoundFile(row['audio_path']) as f:\n            samples = len(f)\n        \n        frames = int(samples/self.frame_size)\n        return min(frames, self.max_frames)\n        \n    \n    def _create_frames(self, signal): \n        # Select a random chunk of the signal\n        if signal.shape[0] > self.max_samples:\n            start = np.random.randint(signal.shape[0] - self.max_samples)\n            end = start + self.max_samples\n            signal = signal[start:end]\n        return wave_to_frames(signal, self.frame_size).astype(np.float32)\n        \n    \nclass TrainingIterDatasetSample(torch.utils.data.IterableDataset):\n    def __init__(self, row, frame_size_in_secs=5, max_mins=.5, sr=32000):\n        self.audio_path = row['audio_path']\n        self.targets = row['target']\n        self.primary_target = row['primary']\n        self.sr = sr\n        \n        self.frame_size_in_secs = frame_size_in_secs\n        self.frame_size = int(frame_size_in_secs*self.sr)\n        \n        self.max_mins = max_mins\n        self.max_samples = int(max_mins*60*self.sr)\n        self.max_frames = int(self.max_samples/self.frame_size)\n        \n        self.n_frames = self.create_len()\n        \n        self.num_classes = self.targets.shape[0]\n            \n    def __iter__(self):\n        y, sr = sf.read(self.audio_path)#, \n                             #sr=self.sr)\n        \n        frames = self.create_frames(y)\n        #print(\"Debug:\",frames.shape, self.n_frames)\n        \n        for i, frame in enumerate(frames):\n            #print(i)\n            yield {\n                \"waveform\": frame, \n                \"targets\": self.targets,\n                \"primary_target\": self.primary_target\n            }\n\n    def __len__(self):\n        return self.n_frames\n    \n    def create_len(self, row):\n        with sf.SoundFile(self.audio_path) as f:\n            samples = len(f)\n        \n        frames = int(samples/self.frame_size)\n        return min(frames, self.max_frames)\n        \n    def create_frames(self, signal): \n        # Select a random chunk of the signal\n        if signal.shape[0] > self.max_samples:\n            start = np.random.randint(signal.shape[0] - self.max_samples)\n            end = start + self.max_samples\n            signal = signal[start:end]\n        return wave_to_frames(signal, self.frame_size).astype(np.float32)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport time\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\ndef do_mixup(x, mixup_lambda):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes \n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out\n    \n\ndef append_to_dict(dict, key, value):\n    if key in dict.keys():\n        dict[key].append(value)\n    else:\n        dict[key] = [value]\n\n\ndef forward(model, generator, return_input=False, \n    return_target=False):\n    \"\"\"Forward data to a model.\n    \n    Args: \n      model: object\n      generator: object\n      return_input: bool\n      return_target: bool\n    Returns:\n      audio_name: (audios_num,)\n      clipwise_output: (audios_num, classes_num)\n      (ifexist) segmentwise_output: (audios_num, segments_num, classes_num)\n      (ifexist) framewise_output: (audios_num, frames_num, classes_num)\n      (optional) return_input: (audios_num, segment_samples)\n      (optional) return_target: (audios_num, classes_num)\n    \"\"\"\n    output_dict = {}\n    device = next(model.parameters()).device\n\n    # Forward data to a model in mini-batches\n    for n, batch_data_dict in enumerate(generator):\n        print(n)\n        batch_waveform = move_data_to_device(batch_data_dict['waveform'], device)\n        \n        with torch.no_grad():\n            model.eval()\n            batch_output = model(batch_waveform)\n\n        append_to_dict(output_dict, 'audio_name', batch_data_dict['audio_name'])\n\n        append_to_dict(output_dict, 'clipwise_output', \n            batch_output['clipwise_output'].data.cpu().numpy())\n            \n        if return_input:\n            append_to_dict(output_dict, 'waveform', batch_data_dict['waveform'])\n            \n        if return_target:\n            if 'target' in batch_data_dict.keys():\n                append_to_dict(output_dict, 'target', batch_data_dict['target'])\n\n    for key in output_dict.keys():\n        output_dict[key] = np.concatenate(output_dict[key], axis=0)\n\n    return output_dict\n\n\ndef interpolate(x, ratio):\n    \"\"\"Interpolate data in time domain. This is used to compensate the \n    resolution reduction in downsampling of a CNN.\n    \n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output, frames_num):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value \n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1 :, :].repeat(1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\ndef init_layer(layer):\n    \"\"\"Initialize a Linear or Convolutional layer. \"\"\"\n    nn.init.xavier_uniform_(layer.weight)\n \n    if hasattr(layer, 'bias'):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n    \ndef init_bn(bn):\n    \"\"\"Initialize a Batchnorm layer. \"\"\"\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.)\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        \n        super(ConvBlock, self).__init__()\n        \n        self.conv1 = nn.Conv2d(in_channels=in_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.conv2 = nn.Conv2d(in_channels=out_channels, \n                              out_channels=out_channels,\n                              kernel_size=(3, 3), stride=(1, 1),\n                              padding=(1, 1), bias=False)\n                              \n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n        \n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n        \n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n        \n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n        \n        return x\n\n\nclass Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num):\n        \n        super(Cnn14, self).__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n\n        self.bn0 = nn.BatchNorm2d(64)\n\n        self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)\n        self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)\n        self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)\n        self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)\n        self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)\n        self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)\n\n        self.fc1 = nn.Linear(2048, 2048, bias=True)\n        self.fc_audioset = nn.Linear(2048, classes_num, bias=True)\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        init_layer(self.fc_audioset)\n        \n#     def add_noise(self, batch):\n#         stddev = 0.01\n#         if self.training:\n#             noise = torch.randn(batch.shape, device=device)*stddev\n#             return batch + noise\n#         return batch\n \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"\n        Input: (batch_size, data_length)\"\"\"\n        \n        #input = self.add_noise(input)\n\n        x = self.spectrogram_extractor(input)   # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x)    # (batch_size, 1, time_steps, mel_bins)\n        \n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        \n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n\n        x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')\n        x = F.dropout(x, p=0.2, training=self.training)\n        x = torch.mean(x, dim=3)\n        \n        (x1, _) = torch.max(x, dim=2)\n        x2 = torch.mean(x, dim=2)\n        x = x1 + x2\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = F.relu_(self.fc1(x))\n        embedding = F.dropout(x, p=0.5, training=self.training)\n        clipwise_output = torch.sigmoid(self.fc_audioset(x))\n        \n        output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}\n\n        return output_dict\n\n\nclass Transfer_Cnn14(nn.Module):\n    def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, \n        fmax, classes_num, freeze_base=True):\n        \"\"\"Classifier for a new task using pretrained Cnn14 as a sub module.\n        \"\"\"\n        super(Transfer_Cnn14, self).__init__()\n        audioset_classes_num = 527\n        \n        self.base = Cnn14(sample_rate, window_size, hop_size, mel_bins, fmin, \n            fmax, audioset_classes_num)\n\n        # Transfer to another task layer\n        self.fc_transfer = nn.Linear(2048, classes_num, bias=True)\n\n        if freeze_base:\n            # Freeze AudioSet pretrained layers\n            for param in self.base.parameters():\n                param.requires_grad = False\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.fc_transfer)\n\n    def load_from_pretrain(self, pretrained_checkpoint_path):\n        checkpoint = torch.load(pretrained_checkpoint_path, map_location=torch.device('cpu'))\n        \n        self.base.load_state_dict(checkpoint['model'])\n\n    def load_weights(self, weights_checkpoint_path):\n        checkpoint = torch.load(weights_checkpoint_path, map_location=torch.device('cpu'))\n        self.load_state_dict(checkpoint[\"model_state_dict\"]) \n        \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"Input: (batch_size, data_length)\n        \"\"\"\n        output_dict = self.base(input, mixup_lambda)\n        embedding = output_dict['embedding']\n\n        output_dict['clipwise_output'] = self.fc_transfer(embedding)\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Criterion"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCEWithLogitsLoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)\n    \n\ndef loss_func(output_dict, target_dict):\n    loss = - torch.mean(target_dict['target'] * output_dict['clipwise_output'])\n    return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks and Logging"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback\n\ndef set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = True  # type: ignore\n    torch.backends.cudnn.benchmark = True  # type: ignore\n    \n    \ndef get_logger(out_file=None):\n    logger = logging.getLogger()\n    formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n    logger.handlers = []\n    logger.setLevel(logging.INFO)\n\n    handler = logging.StreamHandler()\n    handler.setFormatter(formatter)\n    handler.setLevel(logging.INFO)\n    logger.addHandler(handler)\n\n    if out_file is not None:\n        fh = logging.FileHandler(out_file)\n        fh.setFormatter(formatter)\n        fh.setLevel(logging.INFO)\n        logger.addHandler(fh)\n    logger.info(\"logger set up\")\n    return logger\n    \n    \n@contextmanager\ndef timer(name: str, logger: Optional[logging.Logger] = None):\n    t0 = time.time()\n    msg = f\"[{name}] start\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    yield\n\n    msg = f\"[{name}] done in {time.time() - t0:.2f} s\"\n    if logger is None:\n        print(msg)\n    else:\n        logger.info(msg)\n    \n    \nset_seed(1213)\n\nclass F1Callback(Callback):\n    def __init__(self,\n                 input_key: str = 'primary_target',#\"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"f1\"):\n        super().__init__(CallbackOrder.Metric)\n\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        \n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n        \n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        y_pred = clipwise_output.argmax(axis=1)\n        y_true = targ.argmax(axis=1)\n\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0).argmax(axis=1)\n        y_true = np.concatenate(self.target, axis=0).argmax(axis=1)\n        score = f1_score(y_true, y_pred, average=\"macro\")\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n\n\nclass mAPCallback(Callback):\n    def __init__(self,\n                 input_key: str = 'primary_target',#\"targets\",\n                 output_key: str = \"logits\",\n                 model_output_key: str = \"clipwise_output\",\n                 prefix: str = \"mAP\"):\n        super().__init__(CallbackOrder.Metric)\n        self.input_key = input_key\n        self.output_key = output_key\n        self.model_output_key = model_output_key\n        self.prefix = prefix\n\n    def on_loader_start(self, state: State):\n        self.prediction: List[np.ndarray] = []\n        self.target: List[np.ndarray] = []\n\n    def on_batch_end(self, state: State):\n        \n        targ = state.input[self.input_key].detach().cpu().numpy()\n        out = state.output[self.output_key]\n\n        clipwise_output = out[self.model_output_key].detach().cpu().numpy()\n\n        self.prediction.append(clipwise_output)\n        self.target.append(targ)\n\n        score = average_precision_score(targ, clipwise_output, average=None)\n        score = np.nan_to_num(score).mean()\n        state.batch_metrics[self.prefix] = score\n\n    def on_loader_end(self, state: State):\n        y_pred = np.concatenate(self.prediction, axis=0)\n        y_true = np.concatenate(self.target, axis=0)\n        score = average_precision_score(y_true, y_pred, average=None)\n        score = np.nan_to_num(score).mean()\n        state.loader_metrics[self.prefix] = score\n        if state.is_valid_loader:\n            state.epoch_metrics[state.valid_loader + \"_epoch_\" +\n                                self.prefix] = score\n        else:\n            state.epoch_metrics[\"train_epoch_\" + self.prefix] = score\n            \n            ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"if LOAD_BEST_WEIGTHS:\n    with open('../input/backtoorigins/species.pickle', 'rb') as f:   \n        species_to_int_map = pickle.load(f)\n        num_classes = len(species_to_int_map)\n    with open('../input/backtoorigins/train_csv.pickle', 'rb') as f:   \n        df_train = pickle.load(f)  \n    df_val = pd.read_csv('../input/backtoorigins/val_csv.pickle')\n    df_val['target'] = df_val.apply(lambda x: np.array(ast.literal_eval(x['target'].replace('.', '').replace(' ',','))), axis=1)\n    df_val['primary'] = df_val.apply(lambda x: np.array(ast.literal_eval(x['primary'].replace('.', '').replace(' ',','))), axis=1)\n    #with open('../input/backtoorigins/val_csv.pickle', 'rb') as f:   \n    #    df_val = pickle.load(f)  \nelse:        \n    birdcall_csv = BirdcallCsv()\n    num_classes = len(birdcall_csv.species_to_int_map)\n    df_train, df_val = birdcall_csv.train_val_split_df(0.1)\ndf_train.to_pickle('train_csv.pickle')\ndf_val.to_pickle('val_csv.pickle')\nprint(\"Saved training and validation csvs\")\n\n# dataset_train = df_train.apply(lambda row: TrainingIterDataset(row), axis=1).tolist()\n# dataset_val = df_val.apply(lambda row: TrainingIterDataset(row), axis=1).tolist()\n#dataset_train = TrainingIterDataset(df_train)\n#dataset_val = TrainingIterDataset(df_val)\n\n#dataset_train, dataset_val = torch.utils.data.ChainDataset(dataset_train), torch.utils.data.ChainDataset(dataset_val)\n\n\ndataset_train, dataset_val = BirdcallDataset(df_train), BirdcallDataset(df_val)\n\n\nprint(\"Training samples: {}\\nValidation samples: {}\".format(len(dataset_train), len(dataset_val)))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"if TRAIN:\n\n    # loaders\n    loaders = {\n        \"train\": torch.utils.data.DataLoader(dataset_train, \n                                     batch_size=32, \n                                     #shuffle=True, \n                                     num_workers=4, \n                                     pin_memory=True, \n                                     drop_last=True),\n        \"valid\": torch.utils.data.DataLoader(dataset_val, \n                                     batch_size=32, \n                                     #shuffle=False,\n                                     num_workers=4,\n                                     pin_memory=True,\n                                     drop_last=False)\n    }\n\n    # model\n    model_config = {\n        \"sample_rate\": 32000,\n        \"window_size\": 1024,\n        \"hop_size\": 320,\n        \"mel_bins\": 64,\n        \"fmin\": 1000,\n        \"fmax\": 14000,\n        \"classes_num\": num_classes,\n        \"freeze_base\": False\n    }\n\n    model = Transfer_Cnn14(**model_config)\n\n    \n    if LOAD_BEST_WEIGTHS:\n        model.load_weights(best_weights_path)\n    else:\n        model.load_from_pretrain(\"../input/panns-pretrained-cnn14model/Cnn14_mAP_0.431.pth\")\n        \n    model.to(device)\n    print('Using GPU: ', torch.cuda.device_count() > 0)\n\n    # Optimizer\n    optimizer = optim.Adam(model.parameters(), lr=0.0001)\n\n    # Scheduler\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=n_epochs)\n\n    # Loss\n    criterion = PANNsLoss().to(device)\n\n    # callbacks\n    callbacks = [\n        F1Callback(input_key=\"primary_target\", output_key=\"logits\", prefix=\"f1\"),\n        mAPCallback(input_key=\"primary_target\", output_key=\"logits\", prefix=\"mAP\"),\n        CheckpointCallback(save_n_best=0)\n    ]\n    warnings.simplefilter(\"ignore\")\n\n    runner = SupervisedRunner(\n        device=device,\n        input_key=\"waveform\",\n        input_target_key=\"targets\")\n\n    runner.train(\n        model=model,\n        criterion=criterion,\n        loaders=loaders,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        num_epochs=n_epochs,\n        verbose=True,\n        logdir=f\"fold0\",\n        callbacks=callbacks,\n        main_metric=\"epoch_f1\",\n        minimize_metric=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bird Detector"},{"metadata":{"trusted":true},"cell_type":"code","source":"def wave_to_frames(wave, frame_size):\n    \"\"\"Transforms a wave into a matrix of dimension (n, frame_size),\n    where n is the number of frames contained in the wave. This means \n    the wave ending of the wave may be truncated by up to frame_size - 1 samples  \n    \"\"\"\n    n = wave.shape[0]\n    if n <= frame_size:\n        new_wave = np.zeros(frame_size)\n        ind = (frame_size - n)//2\n        new_wave[ind:ind+n] = wave\n        return new_wave[np.newaxis,:]\n    \n    n_frames = wave.shape[0]//frame_size\n    frame_matrix = np.reshape(wave[:n_frames*frame_size],\n                            (n_frames, frame_size))\n    return frame_matrix\n \ndef get_model(config: dict, weights_path: str):\n    model = Transfer_Cnn14(**config)\n    model.load_weights(weights_path)\n    model.to(device)\n    model.eval()\n    return model\n\nclass BirdDetector:\n    def __init__(self, model_config,\n                 weights_path, \n                 classes_dict,\n                 threshold=0.5, frame_size_in_secs=5):\n        self.model_config = model_config\n        self.threshold = threshold\n        self.sr = model_config[\"sample_rate\"]\n        self.frame_size_in_secs = frame_size_in_secs\n        self.frame_size = int(self.frame_size_in_secs*self.sr)\n        self.model = get_model(model_config, weights_path)\n        \n        self.batch_size = 32\n        \n        self.classes_dict = classes_dict\n        \n    def birds_on_file(self, path_to_audio_file):\n        \n        signal, _ = librosa.load(path_to_audio_file,\n                               sr=self.sr,\n                               mono=True,\n                               res_type=\"kaiser_fast\")\n        \n        return self.birds_on_signal(signal)\n       \n    def chunkify(self, arr):\n        return [arr[i:i+self.batch_size] for i in range(0,len(arr),self.batch_size)]\n\n    def birds_on_signal(self, signal):\n        \n        frames = wave_to_frames(signal, self.frame_size)\n        labels = []\n        \n        chunks = self.chunkify(frames)\n        all_idxs = []\n        for batch in chunks:\n            batch = torch.from_numpy(batch).float().to(device)\n        \n            prediction = F.sigmoid(self.model(batch)['clipwise_output'])\n            proba = prediction.detach().cpu().numpy()\n            events = proba >= self.threshold\n            idxs = np.argwhere(events)\n            all_idxs += list(idxs[:,1])\n        \n        return self._translate_labels(set(all_idxs))\n    \n    def birds_on_segments(self, full_signal, end_seconds_list):\n        results = []\n        original_duration = 5\n        extra_duration = (self.frame_size_in_secs - original_duration)/2\n        \n        for end_seconds in end_seconds_list:\n            end = min(int((end_seconds + extra_duration)*self.sr), full_signal.shape[0])\n            start = max(int(end - (original_duration + extra_duration)*self.sr),0)\n            signal = full_signal[start:end]\n            labels = self.birds_on_signal(signal)\n            results.append(labels)\n        return results\n    \n    def _translate_labels(self, labels):\n        labels_str_list = []\n        for l in list(labels):\n            ebird_code = self.classes_dict[l]\n            if ebird_code: # Ignores secondaries\n                labels_str_list.append(ebird_code)\n        if labels_str_list:\n            return \" \".join(labels_str_list)\n        else:\n            return \"nocall\"\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_competition_output(detector):\n    \n    # Real test or mock test\n    if os.path.exists(TEST_DIR):\n        test_dir = TEST_DIR\n        test_df = pd.read_csv(TEST_CSV_DIR)\n    else:\n        test_dir = INPUT_ROOT + \"/birdcall-check/test_audio\"\n        test_df = pd.read_csv(INPUT_ROOT + \"/birdcall-check/test.csv\")\n    \n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    output = []\n    for audio_id in unique_audio_id:\n        segments = test_df[test_df[\"audio_id\"] == audio_id]\n        \n        signal, _ = librosa.load(os.path.join(test_dir, audio_id + \".mp3\"),\n                       sr=detector.sr,\n                       mono=True,\n                       res_type=\"kaiser_fast\")\n            \n        if len(segments) == 1:\n            results = detector.birds_on_signal(signal)\n        else:\n            results = detector.birds_on_segments(signal, segments['seconds'])\n        \n        segments['birds'] = results\n        \n        for i, segment in segments.iterrows():\n            output.append({'row_id':segment['row_id'], 'birds':segment['birds']})\n            \n    \n    df = pd.DataFrame(output)\n    \n    df.to_csv('submission.csv', index=False)\n    print(\"saved submission.csv\")\n\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not TRAIN:\n    #weights_path = \"../input/backtoorigins/fold0/checkpoints/last.pth\"\n    #classes_dict_path = \"../input/backtoorigins/classes.pickle\"\n    #classes_dict_path = \"../input/bestweightsyet/classes.pickle\"\n    weights_path = \"../input/605-10-secs/last.pth\"\n    classes_dict_path = \"../input/605-10-secs/classes.pickle\"\n\nelse:\n    weights_path = 'fold0/checkpoints/last.pth'\n    classes_dict_path = 'classes.pickle'\n    \nwith open(classes_dict_path,'rb') as f:\n    classes_dict = pickle.load(f)\n\n    \nmodel_config = {\n    \"sample_rate\": 32000,\n    \"window_size\": 1024,\n    \"hop_size\": 320,\n    \"mel_bins\": 64,\n    \"fmin\": 1000,\n    \"fmax\": 14000,\n    \"classes_num\": len(classes_dict)\n}\n\ndetector = BirdDetector(model_config,\n                        weights_path,\n                        classes_dict,\n                        threshold=0.55,\n                        frame_size_in_secs=10)\nprint('Loaded detector...')\ncreate_competition_output(detector)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Validate with test"},{"metadata":{"trusted":true},"cell_type":"code","source":"\npaths = [\n    '../input/birdsong-recognition/example_test_audio/BLKFR-10-CPL_20190611_093000.pt540.mp3',\n    '../input/birdsong-recognition/example_test_audio/ORANGE-7-CAP_20190606_093000.pt623.mp3'\n]\n\ndef load_test():\n    \n    signals = [\n        librosa.load(p, sr=32000)[0] for p in paths\n    ]\n\n    print('Loaded validation signals.')\n    \n    tests = {\n        'BLKFR-10-CPL': signals[0],\n        'ORANGE-7-CAP': signals[1],\n    }\n    \n    return tests\n\ntests = load_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_validation_output():\n    input_df_path = '../input/birdsong-recognition/example_test_audio_summary.csv'\n    \n    df = pd.read_csv(input_df_path)\n    \n    unique_audio_id = df.filename.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    output = []\n    for audio_id in unique_audio_id:\n        segments = df[df[\"filename\"] == audio_id]\n        \n        signal = tests[audio_id]\n      \n        results = detector.birds_on_segments(signal, segments['seconds'])\n        \n        segments['predicted'] = results\n        \n        for i, segment in segments.iterrows():\n            output.append({\n                'filename':segment['filename'],\n                'seconds':segment['seconds'],\n                'birds':segment['birds'],\n                'predicted':segment['predicted']\n            })\n            \n    \n    df = pd.DataFrame(output)\n    \n    df.to_csv('validation.csv', index=False)\n    print(\"saved validation.csv\")\n\n    \n    return df\n\ncreate_validation_output()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}