{"cells":[{"metadata":{"_uuid":"f85c591c-8db0-4214-86a5-406576e4215b","_cell_guid":"e12c5e93-bcb3-4d76-be32-e2f2ca1ad2ff","trusted":true},"cell_type":"markdown","source":"All cridets [@hidehisaarai1213](https://www.kaggle.com/hidehisaarai1213)\n\nThis notebook based on this [Introduction to Sound Event Detection](https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection)"},{"metadata":{"_uuid":"e78a17c5-fe28-4fae-8172-ffcb13a62510","_cell_guid":"04a4d9a8-8d8f-4df6-9467-1c881589296a","trusted":true},"cell_type":"markdown","source":"### Install packages"},{"metadata":{"_uuid":"3d1eee54-56ed-4acd-96fe-7b795680ba7a","_cell_guid":"a240c1c2-8b50-478c-bd1d-94f44fcc5071","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"local = False\ntest_only = False\n\n\nif not local:\n    !pip -q install timm\n    !pip -q install torchlibrosa\n    !pip -q install audiomentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fd65ce1-7d56-4097-923b-b2d77527c33d","_cell_guid":"eebc1f0a-a609-4898-bdb0-b21566db5217","trusted":true},"cell_type":"markdown","source":"### import packages"},{"metadata":{"_uuid":"7c8479ca-90cb-4a3e-81bd-f1bde42e56f6","_cell_guid":"a6f4264b-8f21-4e76-bdf9-baae6ddc8831","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os, sys, glob, random, time\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport librosa, librosa.display\nimport soundfile as sf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nfrom functools import partial\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom transformers import get_linear_schedule_with_warmup\nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20fa678b-897d-4620-9875-cb07be10b2b3","_cell_guid":"36263878-3127-4cb8-8ded-ca8be1096eaa","trusted":true},"cell_type":"markdown","source":"### About Sound Event Detection(SED)\n\nSound event detection (SED) is the task of detecting the type as well as\nthe onset and offset times of sound events in audio streams.\n\nIn this notebook i will show how to train Sound Event Detection (SED) model with only weak annotation.\n\nIn SED task, we need to detect sound events from continuous (long) audio clip, and provide prediction of what sound event exists from when to when.\n\nfor more details\n\n-> [Polyphonic Sound Event Detection\nwith Weak Labeling Paper](http://www.cs.cmu.edu/~yunwang/papers/cmu-thesis.pdf)\n\n-> [Introduction to Sound Event Detection Notebook](https://www.kaggle.com/hidehisaarai1213/introduction-to-sound-event-detection)"},{"metadata":{"_uuid":"e2bb84f6-14ea-4bfc-a30c-c314a478485a","_cell_guid":"34529fa7-9513-4eb0-a372-83f2a00fdeaa","trusted":true},"cell_type":"markdown","source":"### PANN Utils\n\n-> [PANNs repository](https://github.com/qiuqiangkong/audioset_tagging_cnn/)\n\n-> [PANNs paper](https://arxiv.org/abs/1912.10211)"},{"metadata":{"_uuid":"ba05c333-c265-41f5-80db-233187aa2dab","_cell_guid":"cfa8c1b6-6299-4133-86d7-078d23c399a3","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5439c201-89d3-436a-8b5d-83dc6a93b56b","_cell_guid":"f3518928-54ba-4f8e-bff5-8374064a1dcc","trusted":true},"cell_type":"markdown","source":"### Create Folds"},{"metadata":{"_uuid":"9ac6b616-981c-439d-8cbb-4a3c1fcec411","_cell_guid":"1fa2ace0-914b-4299-8a86-5391bed10a1c","trusted":true},"cell_type":"code","source":"def create_folds():\n    ''' \n    Split kaggle dataset into [5] folds. Re-write the csv with fold information.\n    implicit input: kaggle dataset rfcx-species-audio-detection/train_tp.csv\n    output: ./train_folds.csv\n    '''\n    train = pd.read_csv(args.train_tp_csv).sort_values(\"recording_id\")\n    \n    train_gby = train.groupby(\"recording_id\")[[\"species_id\"]].first().reset_index()\n    train_gby = train_gby.sample(frac=1, random_state=args.seed).reset_index(drop=True)\n    train_gby.loc[:, 'kfold'] = -1\n    \n    X = train_gby[\"recording_id\"].values\n    y = train_gby[\"species_id\"].values\n    \n    kfold = StratifiedKFold(n_splits=args.FOLDS)\n    for fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n        #print(\"i_idx\", t_idx, \"v_idx\", v_idx)\n        train_gby.loc[v_idx, \"kfold\"] = fold # mark validation set : 0..4\n    \n    train = train.merge(train_gby[['recording_id', 'kfold']], on=\"recording_id\", how=\"left\")\n    print('train.kfold.value_counts:\\n{}'.format(train.kfold.value_counts()))\n    if os.path.dirname(args.train_csv):\n        os.makedirs(os.path.dirname(args.train_csv), exist_ok = True)\n    train.to_csv(args.train_csv, index=False)\n\ndef create_foldsfp():\n    ''' \n    Split kaggle dataset into [5] folds. Re-write the csv with fold information.\n    implicit input: kaggle dataset rfcx-species-audio-detection/train_tp.csv\n    output: ./train_folds.csv\n    '''\n    trainfp = pd.read_csv(args.train_fp_csv).sort_values(\"recording_id\")\n    \n    trainfp_gby = trainfp.groupby(\"recording_id\")[[\"species_id\"]].first().reset_index()\n    trainfp_gby = trainfp_gby.sample(frac=1, random_state=args.seed).reset_index(drop=True)\n    trainfp_gby.loc[:, 'kfold'] = -1\n    \n    X = trainfp_gby[\"recording_id\"].values\n    y = trainfp_gby[\"species_id\"].values\n    \n    kfold = StratifiedKFold(n_splits=args.FOLDS)\n    # t_idx: array of indexes of the trainset, v_idx: array of testset\n    for fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n        #print(\"i_idx\", t_idx, \"v_idx\", v_idx)\n        trainfp_gby.loc[v_idx, \"kfold\"] = fold # mark validation set : 0..4\n    \n    trainfp = trainfp.merge(trainfp_gby[['recording_id', 'kfold']], on=\"recording_id\", how=\"left\")\n    print('trainfp.kfold.value_counts:\\n{}'.format(trainfp.kfold.value_counts()))\n    if os.path.dirname(args.trainfp_csv):\n        os.makedirs(os.path.dirname(args.trainfp_csv), exist_ok = True)\n    trainfp.to_csv(args.trainfp_csv, index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f018ef90-394f-4114-b83b-2d2955d5d17f","_cell_guid":"5bd77595-b38f-402c-9d54-cacd76f56c72","trusted":true},"cell_type":"markdown","source":"### SED Model\n\n1. Model takes raw waveform and converted into log-melspectogram using `torchlibrosa`'s module\n2. spectogram converted into 3-channels input for ImageNet pretrain model to extract features from CNN's\n3. Although it's downsized through several convolution and pooling layers, \n   the size of it's third dimension and it still contains time information. \n   Each element of this dimension is segment. In SED model, we provide prediction for each of this.\n\n4. This figure gives us an intuitive explanation what is weak annotation and \n   what is strong annotation in terms of sound event detection. For this competition,\n   we only have weak annotation (clip level annotation). Therefore, we need to train our SED model in weakly-supervised manner.\n\n5. In weakly-supervised setting, we only have clip-level annotation, therefore we also need to aggregate that in time axis. Hense, we at first put classifier that outputs class existence probability for each time step just after the feature extractor and then aggregate the output of the classifier result in time axis. In this way we can get both clip-level prediction and segment-level prediction (if the time resolution is high, it can be treated as event-level prediction). Then we train it normally by using BCE loss with clip-level prediction and clip-level annotation."},{"metadata":{"_uuid":"3f714b56-d6c2-4f13-9058-651e2d5a7ac0","_cell_guid":"0cb8f442-0e8c-42c0-9ef7-6b9365ec67ff","trusted":true},"cell_type":"code","source":"class Identity(nn.Module):\n    def __init__(self):\n        super(Identity, self).__init__()\n        \n    def forward(self, x):\n        print('identity module x.shape=', x.shape)\n        return x\n\nclass AttBlock(nn.Module):\n    def __init__(self,in_ft, out_ft):\n        super(AttBlock, self).__init__()\n        self.in_ft, self.out_ft = in_ft, out_ft\n        self.conv1 = nn.Conv2d(\n                                in_channels=in_ft,\n                                out_channels=out_ft,\n                                kernel_size=(1,4),\n                                stride=(1,4),\n                                padding=0,\n                                bias=False)        \n        self.conv2 = nn.Conv2d(\n                                in_channels=in_ft,\n                                out_channels=out_ft,\n                                kernel_size=(3,4),\n                                stride=(1,4),\n                                padding=(1,0),\n                                bias=False)        \n        init_layer(self.conv1)\n        init_layer(self.conv2)\n    \n    # With attention block\n    # def forward(self, x): #[16, 512, 10, 4]\n    #     xshape = x.shape\n    #     x = x.transpose(2,3) #[16, 512, 4, 10]\n    #     x = x.reshape(xshape[0], -1, xshape[2]) #[16, 2048, 10]\n    #     x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) #[16, 2048, 10]\n    #     x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1) #[16, 2048, 10]\n    #     x = x1 + x2 #[16, 2048, 10]\n    #     x = x.reshape(xshape[0], xshape[1], xshape[3], xshape[2]) #[16, 512, 4, 10]\n    #     x = x.transpose(2,3) #[16, 512, 10, 4]\n    \n    #     x1 = self.conv1(x) #[16, 24, 10, 1]\n    #     # x1 = self.bn1(x1)   #[16, 24, 10, 1]\n    #     x1 = x1.squeeze(dim=3) #[16, 24, 10]\n    #     x2 = self.conv2(x) #[16, 24, 10, 1]\n    #     # x2 = self.bn2(x2)   #[16, 24, 10, 1]\n    #     x2 = x2.squeeze(dim=3) #[16, 24, 10]\n        \n    #     norm_att = torch.softmax(torch.clamp(x1, -10, 10), dim=-1) #[16, 24, 10]\n    #     cla = torch.sigmoid(x2) #[16, 24, 10]\n    #     x = torch.sum(norm_att * cla, dim=2) #[16, 24]\n        \n    #     return x\n\n    # Without attention block\n    def forward(self, x): #[16, 512, 10, 4]\n        # xshape = x.shape\n        # x = x.transpose(2,3) #[16, 512, 4, 10]\n        # x = x.reshape(xshape[0], -1, xshape[2]) #[16, 2048, 10]\n        # x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1) #[16, 2048, 10]\n        # x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1) #[16, 2048, 10]\n        # x = x1 + x2 #[16, 2048, 10]\n        # x = x.reshape(xshape[0], xshape[1], xshape[3], xshape[2]) #[16, 512, 4, 10]\n        # x = x.transpose(2,3) #[16, 512, 10, 4]\n    \n        x = self.conv1(x) #[16, 24, 10, 1]\n        # x = self.bn1(x)   #[16, 24, 10, 1]\n        x = x.squeeze(dim=3) #[16, 24, 10]\n        \n        x = torch.max(x, dim=2)[0] #[16, 24]\n        # x = torch.sum(x, dim=2) #[16, 24] insteqd of max => extremely high loss !\n        x = torch.sigmoid(x)\n        \n        return x\n    \nclass AudioSEDModel(nn.Module):\n    def __init__(self, encoder, sample_rate, window_size, hop_size, \n                 mel_bins, fmin, fmax, classes_num):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # self.bn0 = nn.BatchNorm2d(mel_bins)\n        self.bn0 = nn.BatchNorm2d(1)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n                \n        # Model Encoder\n        self.encoder = torchvision.models.resnet18(pretrained=True)\n        self.enc_out_feat = self.encoder.fc.in_features\n        \n        # feature extraction ot finetuning, applied to pretrained encoder only\n        for param in self.encoder.parameters(): \n                param.requires_grad = args.finetune\n            \n        # class re-assignment : #[16, 512] -> #[16, 24]\n\n        # Replace pre-trained encoder's last layer\n        self.encoder.avgpool = AttBlock(self.enc_out_feat, classes_num)\n        self.encoder.fc = nn.Sequential()\n        \n        self.init_weight()\n\n    def init_weight(self):\n        init_bn(self.bn0)\n        \n    def forward(self, input):\n        \"\"\"Input : (batch_size, data_length) : [16, 160000] \"\"\"\n\n        x = self.spectrogram_extractor(input) # [16, 1, 313, 2049]\n        # batch_size x 1 x time_steps x freq_bins\n        x = self.logmel_extractor(x) # [16, 1, 313, 128]\n        # batch_size x 1 x time_steps x mel_bins\n        bs, _, frames_num, _ = x.shape # 16, 313\n\n        if self.training:\n            x = self.spec_augmenter(x)\n\n        # x = x.transpose(1, 3) # [16, 128, 313, 1]\n        x = self.bn0(x)\n        # x = x.transpose(1, 3) # [16, 1, 313, 128]\n        \n        # features extraction\n        \n        x = x.expand(x.shape[0], 3, x.shape[2], x.shape[3]) # [16, 3, 313, 128]\n        # Intermediate utput shape (batch size, channels, time, frequency)\n        # Final output shape (batch size, classes_num)\n        x = self.encoder(x) #[16, 24]\n        #x = F.dropout(x, p=0.5, training=self.training)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16fc4d0c-b9dd-44e8-a225-5514011426a7","_cell_guid":"41993a03-07e5-4536-a97f-173af486c575","trusted":true},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"_uuid":"4bef9726-af5b-4181-872f-a80d3312839b","_cell_guid":"02950f1f-7d55-43c4-9d19-08e2374ecd49","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def crop_or_pad(y, sr, period, record, mode=\"train\"):\n    len_y = len(y)\n    effective_length = sr * period\n    rint = np.random.randint(len(record['t_min']))\n    sample_start = record['t_min'][rint] * sr\n    sample_end = record['t_max'][rint] * sr\n    if len_y > effective_length:\n        # Positioning sound slice\n        center = np.round((sample_start + sample_end) / 2)\n        beginning = center - effective_length / 2\n        if beginning < 0:\n            beginning = 0\n        beginning = np.random.randint(beginning, center)\n        ending = beginning + effective_length\n        if ending > len_y:\n            ending = len_y\n        beginning = ending - effective_length\n        y = y[beginning:ending].astype(np.float32)\n    else:\n        y = y.astype(np.float32)\n        beginning = 0\n        ending = effective_length\n\n\n    beginning_time = beginning / sr\n    ending_time = ending / sr\n    label = np.zeros(24, dtype='f')\n\n    for i in range(len(record['t_min'])):\n        if (record['t_min'][i] <= ending_time) and (record['t_max'][i] >= beginning_time):\n            label[record['species_id'][i]] = 1\n    \n    return y, label","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6140f262-3f93-4877-b261-b6a02b0b2fa0","_cell_guid":"5dc605b6-090d-4f7c-9f61-1c38a99653c6","trusted":true},"cell_type":"code","source":"class SedDataset:\n    def __init__(self, df, period=10, stride=5, audio_transform=None, data_path=\"train\", mode=\"train\"):\n\n        self.period = period\n        self.stride = stride\n        self.audio_transform = audio_transform\n        self.data_path = data_path\n        self.mode = mode\n\n        self.df = df.groupby(\"recording_id\").agg(lambda x: list(x)).reset_index()\n        \n        self.species = {k:[] for k in range(24)}\n        if (mode == 'train'):\n            for i in range(self.df.shape[0]):\n                record = self.df.iloc[i]\n                for k in record[\"species_id\"]:\n                    self.species[k].append(i)\n\n        \n    def getSpecies(self, target): # (bs, nrOfClass)\n        species = target.view(-1)\n        _, idx = torch.max(target, dim=1)\n        data = []\n        for b in range(target.shape[0]):\n            rec_list = self.species[idx[b].item()]\n            rint = np.random.randint(len(rec_list))\n            rec = rec_list[rint]\n            data.append(self.__getitem__(rec))\n        return data\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        record = self.df.iloc[idx]\n\n        files = glob.glob(f\"{self.data_path}/{record['recording_id']}.*\")\n        if len(files) == 0:\n            raise Exception(f\"{self.data_path}/{record['recording_id']}.*\")\n        # y, sr = sf.read(f\"{self.data_path}/{record['recording_id']}.flac\")\n        y, sr = sf.read(files[0])\n        \n        if self.mode != \"test\":\n            y, label = crop_or_pad(y, sr, period=self.period, record=record, mode=self.mode)\n\n            if self.audio_transform:\n                y = self.audio_transform(samples=y, sample_rate=sr)\n        else:\n            y_ = []\n            i = 0\n            effective_length = self.period * sr\n            stride = self.stride * sr\n            # y = np.stack([y[i:i+effective_length].astype(np.float32) for i in range(0, 60*sr+stride-effective_length, stride)])\n            for i in range(0, 60*sr-effective_length+1, stride):\n                y_.append(y[i:i+effective_length].astype(np.float32))\n            y = np.stack(y_)\n            label = np.zeros(24, dtype='f')\n        \n        return {\n            \"image\" : y,\n            \"target\" : label,\n            \"id\" : record['recording_id']\n        }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ba3d684-163d-4cc8-81f3-597e99df5dcc","_cell_guid":"8e8075b8-dbc5-4473-b691-4ee2255ca0c9","trusted":true},"cell_type":"markdown","source":"### Augmentations"},{"metadata":{"_uuid":"8d71ba59-5bbc-4db5-9c0b-95e4849a0c95","_cell_guid":"873cfaee-657d-447f-9f9d-70b5d7ac515e","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import audiomentations as AA\n\ntrain_audio_transform = AA.Compose([\n    AA.AddGaussianNoise(p=0.5),\n    AA.AddGaussianSNR(p=0.5),\n    #AA.AddBackgroundNoise(\"../input/train_audio/\", p=1)\n    #AA.AddImpulseResponse(p=0.1),\n    #AA.AddShortNoises(\"../input/train_audio/\", p=1)\n    #AA.FrequencyMask(min_frequency_band=0.0,  max_frequency_band=0.2, p=0.1),\n    #AA.TimeMask(min_band_part=0.0, max_band_part=0.2, p=0.1),\n    #AA.PitchShift(min_semitones=-0.5, max_semitones=0.5, p=0.1),\n    #AA.Shift(p=0.1),\n    #AA.Normalize(p=0.1),\n    #AA.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=1, p=0.05),\n    #AA.PolarityInversion(p=0.05),\n    #AA.Gain(p=0.2)\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7e1b716-84fb-40eb-840f-52c9de4af5b5","_cell_guid":"75f35c0b-3c50-4777-acbf-4d6f42ca430e","trusted":true},"cell_type":"markdown","source":"### Utils"},{"metadata":{"_uuid":"6296a10a-a88e-414d-a2b8-ab062e9eaac6","_cell_guid":"335d0177-031b-47c0-853d-d62f3f601993","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def _lwlrap_sklearn(truth, scores):\n    \"\"\"Reference implementation from https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8\"\"\"\n    sample_weight = np.sum(truth > 0, axis=1) # truth[144,24], 144 samples = 9(batches)x16(batchsz)\n    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)\n    overall_lwlrap = metrics.label_ranking_average_precision_score(\n        truth[nonzero_weight_sample_indices, :] > 0,                 # (144, 24)\n        scores[nonzero_weight_sample_indices, :],                    # (144, 24)\n        sample_weight=sample_weight[nonzero_weight_sample_indices])  # (144,)\n    return overall_lwlrap\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass MetricMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = []\n        self.y_pred = []\n    \n    def update(self, y_true, y_pred):\n        # print(f'torch.max(y_true)={torch.max(y_true)} shape{y_true.shape} tot-len {len(self.y_true)}')\n        self.y_true.extend(y_true.cpu().detach().numpy().tolist())\n        self.y_pred.extend(y_pred.cpu().detach().numpy().tolist())\n\n    @property\n    def avg(self):\n        #score_class, weight = lwlrap(np.array(self.y_true), np.array(self.y_pred))\n        self.score = _lwlrap_sklearn(np.array(self.y_true), np.array(self.y_pred)) #(score_class * weight).sum()\n        return {\n            \"lwlrap\" : self.score\n        }\n\ndef seed_everithing(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\ndef reseedOnResume():\n    if args.pretrain_weights and os.path.exists(args.pretrain_weights):        \n        checkpoint = torch.load(args.pretrain_weights, map_location=args.device)\n        if type(checkpoint) is dict and 'seed' in checkpoint:\n                new_seed = checkpoint['seed'] + 1\n                print(f\"-- Re-seed -- : old {args.seed} new {new_seed}\")\n                args.seed = new_seed","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3dbbbc53-5be9-4960-927b-e7d858491845","_cell_guid":"d485cf05-1f5e-488e-8a37-f702f50c628b","trusted":true},"cell_type":"markdown","source":"### Losses"},{"metadata":{"_uuid":"a33e771c-fdf1-4582-a415-b0499f1fe8cd","_cell_guid":"6fbe0058-972c-4fcb-913a-419f2910213d","trusted":true},"cell_type":"code","source":"from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss\n\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input = torch.clamp(input, 0.0, 0.99999)\n        target = target.float()\n        return self.bce(input, target)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"594409e8-5d2d-49fc-ae47-c247c4c10867","_cell_guid":"0fc09030-ad4d-4954-9f07-d0d15dee6280","trusted":true},"cell_type":"markdown","source":"### Functions"},{"metadata":{"_uuid":"46516b07-b3e9-423f-bfcf-aa5f2aa69da1","_cell_guid":"ed532057-f140-4985-b37d-d33b63b4a8f9","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# def train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch, fpDs):\n#     losses = AverageMeter()\n#     scores = MetricMeter()\n\n#     model.train()\n#     for i, sample in enumerate(loader):\n#         # -- TP --\n#         optimizer.zero_grad()\n#         input = sample['image'].to(args.device)\n#         target = sample['target'].to(args.device)\n#         output = model(input)\n#         loss = criterion(output, target)\n#         # loss.backward()      \n#         # optimizer.step()\n#         # -- FP --\n#         optimizer.zero_grad()\n#         fp_sample = fpDs.getSpecies(target)\n#         input = [fp_sample[i]['image'] for i in range(len(fp_sample))]\n#         input = torch.tensor(input).to(args.device)\n#         target = [fp_sample[i]['target'] for i in range(len(fp_sample))]\n#         target = torch.tensor(target).to(args.device)\n#         output = model(input)\n#         index_fp = target == 1\n#         index_nonfp = target == 0\n#         target[index_fp] = 0\n#         target[index_nonfp] = 0.2      \n#         loss2 = criterion(output, target) / 10.\n#         # loss.backward()\n#         # optimizer.step()\n#         # -- av loss of TP + FP\n#         loss = loss + loss2\n#         loss.backward()\n#         optimizer.step()\n        \n#         if scheduler and args.step_scheduler:\n#             scheduler.step()\n\n#         bs = input.size(0)\n#         scores.update(target, output)\n#         losses.update(loss.item(), bs)\n\n#     # print(f\"Train E:{epoch} - Loss{losses.avg:0.4f}\")\n#     return scores.avg, losses.avg\n\ndef train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch, fpDs):\n    losses = AverageMeter()\n    scores = MetricMeter()\n\n    model.train()\n    for i, sample in enumerate(loader):\n        optimizer.zero_grad()\n        criterion = nn.BCELoss(reduction='none')\n        # -- TP --\n        input = sample['image'].to(args.device)\n        target = sample['target'].to(args.device)\n        output = model(input)\n        loss = criterion(output, target)\n        loss = torch.mean(loss)\n        loss.backward()      \n        # -- FP --\n        fp_sample = fpDs.getSpecies(target)\n        fp_input = torch.tensor([fp_sample[i]['image'] for i in range(len(fp_sample))])\n        fp_target = torch.tensor([fp_sample[i]['target'] for i in range(len(fp_sample))])\n        fp_input = fp_input.to(args.device)\n        index_fp = (fp_target == 1).to(args.device)\n        index_nonfp = (fp_target == 0).to(args.device)\n        fp_output = model(fp_input)\n        fp_target = (fp_target * 0).to(args.device)\n        fp_loss = criterion(fp_output[index_fp], fp_target[index_fp])\n        fp_loss = torch.mean(fp_loss)*0.1\n        fp_loss.backward()        \n        # -- apply accumulated grad     \n        optimizer.step()\n        \n        if scheduler and args.step_scheduler:\n            scheduler.step()\n\n        bs = input.size(0)\n        scores.update(target, output)\n        losses.update(loss.item(), bs)\n\n    # print(f\"Train E:{epoch} - Loss{losses.avg:0.4f}\")\n    return scores.avg, losses.avg\n        \ndef valid_epoch(args, model, loader, criterion, epoch):\n    losses = AverageMeter()\n    scores = MetricMeter()\n    model.eval()\n    with torch.no_grad():\n        for i, sample in enumerate(loader):\n            input = sample['image'].to(args.device)\n            target = sample['target'].to(args.device)\n            output = model(input)\n            loss = criterion(output, target)\n\n            bs = input.size(0)\n            scores.update(target, output)\n            losses.update(loss.item(), bs)\n    # print(f\"Valid E:{epoch} - Loss:{losses.avg:0.4f}\")\n    return scores.avg, losses.avg\n\ndef test_epoch(args, model, loader):\n    model.eval()\n    pred_list = []\n    id_list = []\n    with torch.no_grad():\n        for i, sample in enumerate(loader):\n            input = sample[\"image\"].to(args.device)\n            bs, seq, w = input.shape\n            input = input.reshape(bs*seq, w)\n            id = sample[\"id\"]\n            output = model(input)\n            output = output.reshape(bs, seq, -1)\n            output = torch.sum(output, dim=1) / seq\n            #output, _ = torch.max(output, dim=1)\n            output = output.cpu().detach().numpy().tolist()\n            pred_list.extend(output)\n            id_list.extend(id)\n    \n    return pred_list, id_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a7107a9-cb18-4e85-9127-3c2228883c88","_cell_guid":"7dbe2bbe-fb9e-4ae2-ae2f-6febe87ea600","trusted":true},"cell_type":"markdown","source":"### Main Function"},{"metadata":{"_uuid":"3d4af984-07f1-4d0d-9798-2de10ed6866d","_cell_guid":"ad1b3760-b237-48fc-a12f-2e6cd2c5c150","trusted":true},"cell_type":"code","source":"def test_one_model(modelNr, modelPath=None):\n    if not modelPath:\n        modelPath=os.path.join(args.save_path, f'fold-{modelNr}.bin')\n    if not os.path.exists(modelPath):\n        print('test_one_model model does not exists:', modelPath)\n        return\n\n    sub_df = pd.read_csv(args.sample_submission_csv)\n    if args.DEBUG:\n        sub_df = sub_df.sample(20)\n\n    test_dataset = SedDataset(\n        df = sub_df,\n        period=args.period,\n        stride=5,\n        audio_transform=None,\n        data_path=args.test_data_path,\n        mode=\"test\"\n    )\n\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=args.batch_size,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n\n    model = AudioSEDModel(**args.model_param)\n    model = model.to(args.device)\n    \n    checkpoint = torch.load(modelPath, map_location=args.device)\n    if type(checkpoint) is dict:\n        model.load_state_dict(checkpoint['state'], strict=False)\n    else:\n        model.load_state_dict(checkpoint, strict=False) \n    # model.load_state_dict(torch.load(modelPath, map_location=args.device))\n    model = model.to(args.device)\n\n    target_cols = sub_df.columns[1:].values.tolist()\n    test_pred, ids = test_epoch(args, model, test_loader)\n    print('(n recordings, n species) =', # (nb of recordings, 24-species)\n          np.array(test_pred).shape, ', max', np.max(np.array(test_pred)))\n\n    test_pred_df = pd.DataFrame({\n        \"recording_id\" : sub_df.recording_id.values\n    })\n    test_pred_df[target_cols] = test_pred\n    test_pred_df.to_csv(os.path.join(args.save_path, f\"fold-{modelNr}-submission.csv\"), index=False)\n    print(os.path.join(args.save_path, f\"fold-{modelNr}-submission.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"942e84d1-0f8d-42f8-b5e4-60437bfac76c","_cell_guid":"bc1460fe-0654-4713-83e2-98034c93cb74","trusted":true},"cell_type":"code","source":"import re\ndef main_test():\n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)    \n\n    files = glob.glob(f\"{args.save_path}/fold-*.bin\")\n    files.extend(glob.glob(f\"{args.input_dir}/fold-*.bin\"))\n    print(f\"main_test : {len(files)} models found\")\n    for f in files:\n        print(f'model file : {f}')\n        modelNr = re.compile('.*fold-([0-9]+).bin').match(f).group(1)\n        print(f'main_test : model {f}')\n        test_one_model(modelNr, f)\n\n    # Ensemble multiple fold-n-submission.csv into submission.csv\n    print('Ensemble : ')\n    fsubs = glob.glob(os.path.join(args.save_path, \"fold-*-submission.csv\"))\n    sub = None\n    sub_df = None\n    cnt = 0\n    for fsub in fsubs:\n        print('sub = ',fsub)\n        sub_df = pd.read_csv(fsub)\n        if cnt == 0:\n            sub = sub_df.iloc[:, 1:]\n        else:\n            sub += sub_df.iloc[:, 1:]\n        cnt += 1\n    if cnt > 0:\n        sub_df.iloc[:, 1:] = sub / cnt\n        sub_df.to_csv(\"submission.csv\", index=False)\n        print('Output in submission.csv')\n    else:\n        print('Ensembling not done')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58851dc0-bc3d-44f2-9a91-24953b4a59dc","_cell_guid":"c18e8a5c-fc4c-42f1-b2ca-6be4097274fa","trusted":true},"cell_type":"code","source":"def main_CV(fold):\n    best_lwlrap = -np.inf\n    early_stop_count = 0\n\n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)\n\n    model = AudioSEDModel(**args.model_param)\n    model = model.to(args.device)\n    \n    if args.pretrain_weights and os.path.exists(args.pretrain_weights):\n        print(\"-- loading pretrain weights -- : \", args.pretrain_weights)\n        checkpoint = torch.load(args.pretrain_weights, map_location=args.device)\n        if type(checkpoint) is dict:\n            model.load_state_dict(checkpoint['state'], strict=False)\n            if 'best' in checkpoint:\n                best_lwlrap = checkpoint['best']\n            print(\"previous best_lwlrap = \", best_lwlrap)\n        else:\n            model.load_state_dict(checkpoint, strict=False)\n        model = model.to(args.device)\n\n    criterion = PANNsLoss() #BCEWithLogitsLoss() #MaskedBCEWithLogitsLoss() #BCEWithLogitsLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr,weight_decay=0.005)\n\n    train_df = pd.read_csv(args.train_csv)\n    if args.DEBUG:\n        train_df = train_df.sample(args.DBG_TRAIN_LEN)\n    train_fold = train_df[train_df.kfold != (fold)]\n    valid_fold = train_df[train_df.kfold == (fold)]\n\n    train_dataset = SedDataset(\n        df = train_fold,\n        period=args.period,\n        audio_transform=train_audio_transform,\n        data_path=args.train_data_path,\n        mode=\"train\"\n    )\n\n    valid_dataset = SedDataset(\n        df = valid_fold,\n        period=args.period,\n        audio_transform=None,\n        data_path=args.train_data_path,\n        mode=\"valid\"\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        shuffle=True,\n        drop_last=True,\n        num_workers=args.num_workers\n    )\n\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n\n    trainfp_df = pd.read_csv(args.trainfp_csv)\n    if args.DEBUG:\n        trainfp_df = trainfp_df.sample(args.DBG_TRAIN_LEN)\n    trainfp_fold = trainfp_df[trainfp_df.kfold != (fold)]\n    validfp_fold = trainfp_df[trainfp_df.kfold == (fold)]\n\n    trainfp_dataset = SedDataset(\n        df = trainfp_fold,\n        period=args.period,\n        audio_transform=train_audio_transform,\n        data_path=args.train_data_path,\n        mode=\"train\"\n    )\n\n\n    num_train_steps = int(len(train_loader) * args.epochs)\n    num_warmup_steps = int(0.1 * args.epochs * len(train_loader))\n    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_train_steps)\n\n    for epoch in range(args.start_epcoh, args.epochs):\n        \n        train_avg, train_loss = train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch, trainfp_dataset)\n        valid_avg, valid_loss = valid_epoch(args, model, valid_loader, criterion, epoch)\n\n        content = f\"\"\"\n                {time.ctime()} \\n\n                Fold:{fold}, Epoch:{epoch}, lr:{optimizer.param_groups[0]['lr']:.7}\\n\n                Train Loss:{train_loss:0.4f} - LWLRAP:{train_avg['lwlrap']:0.4f}\\n\n                Valid Loss:{valid_loss:0.4f} - LWLRAP:{valid_avg['lwlrap']:0.4f}\\n\n        \"\"\"\n        print(content)\n        with open(f'{args.save_path}/log_{args.exp_name}.txt', 'a') as appender:\n            appender.write(content+'\\n')\n        \n        if valid_avg['lwlrap'] > best_lwlrap:\n            print(f\"########## >>>>>>>> Model Improved From {best_lwlrap} ----> {valid_avg['lwlrap']}\")\n            best_lwlrap = valid_avg['lwlrap']\n            early_stop_count = 0\n            torch.save({'state': model.state_dict(), 'best': best_lwlrap, 'seed': args.seed},\n                       os.path.join(args.save_path, f'fold-{fold}.bin'))\n        else:\n            early_stop_count += 1\n\n        # ---------------------\n        if args.early_stop == early_stop_count:\n            print(\"\\n $$$ ---? Ohoo.... we reached early stoping count :\", early_stop_count)\n            break\n\n        if args.epoch_scheduler:\n            scheduler.step()\n        \n    torch.save({'state': model.state_dict(), 'best': best_lwlrap, 'seed': args.seed},\n               os.path.join(args.save_path, f'fold-9999{fold}.bin'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a2909c7-6269-4085-9d43-c166db842efd","_cell_guid":"2160d707-4eac-4b2b-8846-2658a5191531","trusted":true},"cell_type":"markdown","source":"### Config"},{"metadata":{"_uuid":"402a7781-4db7-47de-9d37-9b808b8c8470","_cell_guid":"84da3a76-4b2d-4ad2-bcfd-14755d34ed28","trusted":true},"cell_type":"code","source":"class args:\n    \n    FOLDS = 5\n    if (local):\n        DEBUG = True\n        train_tp_csv = \"C:/kaggle/train_tp.csv\"\n        train_fp_csv = \"C:/kaggle/train_fp.csv\"\n        sample_submission_csv = \"C:/kaggle/sample_submission.csv\"\n        train_csv = \"C:/kaggle/work/train_folds.csv\"\n        trainfp_csv = \"C:/kaggle/work/trainfp_folds.csv\"\n        test_csv = \"C:/kaggle/work/test_df.csv\"\n        output_dir = \"C:/kaggle/work/weights\"\n        input_dir = \"C:/kaggle/modelrfcx\"\n        train_data_path = \"C:/kaggle/train\"\n        test_data_path  = \"C:/kaggle/test\"\n        num_workers = 0\n        epochs = 6 if DEBUG else 50\n        DBG_TRAIN_LEN = 200\n        pretrain_weights = \"C:/kaggle/modelrfcx/latest.bin\"\n    else:\n        DEBUG = False\n        train_tp_csv = \"../input/rfcx-species-audio-detection/train_tp.csv\"\n        train_fp_csv = \"../input/rfcx-species-audio-detection/train_fp.csv\"\n        sample_submission_csv = \"../input/rfcx-species-audio-detection/sample_submission.csv\"\n        train_csv = \"train_folds.csv\"\n        trainfp_csv = \"trainfp_folds.csv\"\n        test_csv = \"test_df.csv\"\n        output_dir = \"weights\"\n        input_dir = \"../input/modelrfcx\"\n        train_data_path = \"../input/rfcx-species-audio-detection/train\"\n        test_data_path = \"../input/rfcx-species-audio-detection/test\"\n        num_workers = 4\n        epochs = 1 if DEBUG else 50\n        DBG_TRAIN_LEN = 400\n        pretrain_weights = \"../input/modelrfcx/latest.bin\"\n    \n    exp_name = \"SED_E0_5F_BASE\"\n    # pretrain_weights = None \n    model_param = {\n        'encoder' : 'resnet18', #'tf_efficientnet_b0_ns',\n        'sample_rate': 16000 if local else 48000,\n        'window_size' : 512 * 2, # 512\n        'hop_size' : 512, \n        'mel_bins' : 128, \n        'fmin' : 0,\n        'fmax' : 16000//2 if local else 48000//2, # 48000 // 2,\n        'classes_num' : 24\n    }\n    finetune = True\n    period = 10\n    seed = 42\n    start_epcoh = 0 \n    lr = 1e-3\n    batch_size = 16\n    early_stop = 150\n    step_scheduler = True\n    epoch_scheduler = False\n\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a49dd869-c726-44a7-81e3-0f2eec5383ef","_cell_guid":"6945a1d9-c674-45a7-9d29-ffaa68fd1540","trusted":true},"cell_type":"markdown","source":"### train folds"},{"metadata":{"_uuid":"b39f9dc6-726d-4ffa-92be-be0071e6cdfa","_cell_guid":"1a48683a-931b-487e-8ac0-41164d2e1557","trusted":true},"cell_type":"code","source":"reseedOnResume()\nif local:\n    os.environ['TORCH_HOME'] = os.path.join(args.output_dir, 'resnet')\n    \nif test_only:\n    seed_everithing(args.seed)\n    main_test()\nelse:\n    seed_everithing(args.seed)\n    create_folds()\n    create_foldsfp()\n    seed_everithing(args.seed)\n    if (local):\n        main_CV(0)\n    else:\n        main_CV(0)\n        # main_CV(1)\n        # main_CV(2)\n        #main_train()\n    \n    seed_everithing(args.seed)\n    # main_test()\n    test_one_model(0)\n    test_one_model(99990)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7305ffa4-1a94-4260-a468-721f94a0d8c3","_cell_guid":"0a055fb7-6192-427f-aea2-b79551823662","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}