{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip -q install timm\n!pip -q install torchlibrosa\n!pip -q install audiomentations","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, glob, random\nimport numpy as np, pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\ntrain = pd.read_csv(\"../input/rfcx-species-audio-detection/train_tp.csv\").sort_values(\"recording_id\")\nss = pd.read_csv(\"../input/rfcx-species-audio-detection/sample_submission.csv\")\n\nFOLDS = 5\nSEED = 42\n\ntrain_gby = train.groupby(\"recording_id\")[[\"species_id\"]].first().reset_index()\ntrain_gby = train_gby.sample(frac=1, random_state=SEED).reset_index(drop=True)\ntrain_gby.loc[:, 'kfold'] = -1\n\nX = train_gby[\"recording_id\"].values\ny = train_gby[\"species_id\"].values\n\nkfold = StratifiedKFold(n_splits=FOLDS)\nfor fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n    train_gby.loc[v_idx, \"kfold\"] = fold\n\ntrain = train.merge(train_gby[['recording_id', 'kfold']], on=\"recording_id\", how=\"left\")\nprint(train.kfold.value_counts())\ntrain.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%writefile pytorch_utils.py\nimport numpy as np\nimport time\nimport torch\nimport torch.nn as nn\n\n\ndef move_data_to_device(x, device):\n    if 'float' in str(x.dtype):\n        x = torch.Tensor(x)\n    elif 'int' in str(x.dtype):\n        x = torch.LongTensor(x)\n    else:\n        return x\n\n    return x.to(device)\n\n\ndef do_mixup(x, mixup_lambda):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes \n    (1, 3, 5, ...).\n\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0 :: 2].transpose(0, -1) * mixup_lambda[0 :: 2] + \\\n        x[1 :: 2].transpose(0, -1) * mixup_lambda[1 :: 2]).transpose(0, -1)\n    return out\n    \n\ndef append_to_dict(dict, key, value):\n    if key in dict.keys():\n        dict[key].append(value)\n    else:\n        dict[key] = [value]\n\n\ndef forward(model, generator, return_input=False, \n    return_target=False):\n    \"\"\"Forward data to a model.\n    \n    Args: \n      model: object\n      generator: object\n      return_input: bool\n      return_target: bool\n\n    Returns:\n      audio_name: (audios_num,)\n      clipwise_output: (audios_num, classes_num)\n      (ifexist) segmentwise_output: (audios_num, segments_num, classes_num)\n      (ifexist) framewise_output: (audios_num, frames_num, classes_num)\n      (optional) return_input: (audios_num, segment_samples)\n      (optional) return_target: (audios_num, classes_num)\n    \"\"\"\n    output_dict = {}\n    device = next(model.parameters()).device\n    time1 = time.time()\n\n    # Forward data to a model in mini-batches\n    for n, batch_data_dict in enumerate(generator):\n        print(n)\n        batch_waveform = move_data_to_device(batch_data_dict['waveform'], device)\n        \n        with torch.no_grad():\n            model.eval()\n            batch_output = model(batch_waveform)\n\n        append_to_dict(output_dict, 'audio_name', batch_data_dict['audio_name'])\n\n        append_to_dict(output_dict, 'clipwise_output', \n            batch_output['clipwise_output'].data.cpu().numpy())\n\n        if 'segmentwise_output' in batch_output.keys():\n            append_to_dict(output_dict, 'segmentwise_output', \n                batch_output['segmentwise_output'].data.cpu().numpy())\n\n        if 'framewise_output' in batch_output.keys():\n            append_to_dict(output_dict, 'framewise_output', \n                batch_output['framewise_output'].data.cpu().numpy())\n            \n        if return_input:\n            append_to_dict(output_dict, 'waveform', batch_data_dict['waveform'])\n            \n        if return_target:\n            if 'target' in batch_data_dict.keys():\n                append_to_dict(output_dict, 'target', batch_data_dict['target'])\n\n        if n % 10 == 0:\n            print(' --- Inference time: {:.3f} s / 10 iterations ---'.format(\n                time.time() - time1))\n            time1 = time.time()\n\n    for key in output_dict.keys():\n        output_dict[key] = np.concatenate(output_dict[key], axis=0)\n\n    return output_dict\n\n\ndef interpolate(x, ratio):\n    \"\"\"Interpolate data in time domain. This is used to compensate the \n    resolution reduction in downsampling of a CNN.\n    \n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output, frames_num):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value \n    is the same as the value of the last frame.\n\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1 :, :].repeat(1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\n\ndef count_flops(model, audio_length):\n    \"\"\"Count flops. Code modified from others' implementation.\n    \"\"\"\n    multiply_adds = True\n    list_conv2d=[]\n    def conv2d_hook(self, input, output):\n        batch_size, input_channels, input_height, input_width = input[0].size()\n        output_channels, output_height, output_width = output[0].size()\n \n        kernel_ops = self.kernel_size[0] * self.kernel_size[1] * (self.in_channels / self.groups) * (2 if multiply_adds else 1)\n        bias_ops = 1 if self.bias is not None else 0\n \n        params = output_channels * (kernel_ops + bias_ops)\n        flops = batch_size * params * output_height * output_width\n \n        list_conv2d.append(flops)\n\n    list_conv1d=[]\n    def conv1d_hook(self, input, output):\n        batch_size, input_channels, input_length = input[0].size()\n        output_channels, output_length = output[0].size()\n \n        kernel_ops = self.kernel_size[0] * (self.in_channels / self.groups) * (2 if multiply_adds else 1)\n        bias_ops = 1 if self.bias is not None else 0\n \n        params = output_channels * (kernel_ops + bias_ops)\n        flops = batch_size * params * output_length\n \n        list_conv1d.append(flops)\n \n    list_linear=[] \n    def linear_hook(self, input, output):\n        batch_size = input[0].size(0) if input[0].dim() == 2 else 1\n \n        weight_ops = self.weight.nelement() * (2 if multiply_adds else 1)\n        bias_ops = self.bias.nelement()\n \n        flops = batch_size * (weight_ops + bias_ops)\n        list_linear.append(flops)\n \n    list_bn=[] \n    def bn_hook(self, input, output):\n        list_bn.append(input[0].nelement() * 2)\n \n    list_relu=[] \n    def relu_hook(self, input, output):\n        list_relu.append(input[0].nelement() * 2)\n \n    list_pooling2d=[]\n    def pooling2d_hook(self, input, output):\n        batch_size, input_channels, input_height, input_width = input[0].size()\n        output_channels, output_height, output_width = output[0].size()\n \n        kernel_ops = self.kernel_size * self.kernel_size\n        bias_ops = 0\n        params = output_channels * (kernel_ops + bias_ops)\n        flops = batch_size * params * output_height * output_width\n \n        list_pooling2d.append(flops)\n\n    list_pooling1d=[]\n    def pooling1d_hook(self, input, output):\n        batch_size, input_channels, input_length = input[0].size()\n        output_channels, output_length = output[0].size()\n \n        kernel_ops = self.kernel_size[0]\n        bias_ops = 0\n        \n        params = output_channels * (kernel_ops + bias_ops)\n        flops = batch_size * params * output_length\n \n        list_pooling2d.append(flops)\n \n    def foo(net):\n        childrens = list(net.children())\n        if not childrens:\n            if isinstance(net, nn.Conv2d):\n                net.register_forward_hook(conv2d_hook)\n            elif isinstance(net, nn.Conv1d):\n                net.register_forward_hook(conv1d_hook)\n            elif isinstance(net, nn.Linear):\n                net.register_forward_hook(linear_hook)\n            elif isinstance(net, nn.BatchNorm2d) or isinstance(net, nn.BatchNorm1d):\n                net.register_forward_hook(bn_hook)\n            elif isinstance(net, nn.ReLU):\n                net.register_forward_hook(relu_hook)\n            elif isinstance(net, nn.AvgPool2d) or isinstance(net, nn.MaxPool2d):\n                net.register_forward_hook(pooling2d_hook)\n            elif isinstance(net, nn.AvgPool1d) or isinstance(net, nn.MaxPool1d):\n                net.register_forward_hook(pooling1d_hook)\n            else:\n                print('Warning: flop of module {} is not counted!'.format(net))\n            return\n        for c in childrens:\n            foo(c)\n\n    # Register hook\n    foo(model)\n    \n    device = device = next(model.parameters()).device\n    input = torch.rand(1, audio_length).to(device)\n\n    out = model(input)\n \n    total_flops = sum(list_conv2d) + sum(list_conv1d) + sum(list_linear) + \\\n        sum(list_bn) + sum(list_relu) + sum(list_pooling2d) + sum(list_pooling1d)\n    \n    return total_flops\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%writefile Datasets.py\nimport random, glob\nimport numpy as np, pandas as pd\nimport soundfile as sf\n\nimport torch\nfrom torch.utils.data import Dataset\nfrom albumentations.pytorch.functional import img_to_tensor\n\n\nclass AudioDataset(Dataset):\n    def __init__(self, df, period=10, transforms=None, data_path=\"train\", train=True):\n        self.period = period\n        self.transforms = transforms\n        self.data_path = data_path\n        self.train = train\n        \n        if train: \n            dfgby = df.groupby(\"recording_id\").agg(lambda x: list(x)).reset_index()\n            self.recording_ids = dfgby[\"recording_id\"].values\n            self.species_ids = dfgby[\"species_id\"].values\n            self.t_mins = dfgby[\"t_min\"].values\n            self.t_maxs = dfgby[\"t_max\"].values\n        else:\n            self.recording_ids = df[\"recording_id\"].values\n\n    \n    def __len__(self):\n        return len(self.recording_ids)\n    \n    def __getitem__(self, idx):\n\n        recording_id = self.recording_ids[idx]\n        if self.train:\n            species_id = self.species_ids[idx]\n            t_min, t_max = self.t_mins[idx], self.t_maxs[idx]\n        else:\n            species_id = [0]\n            t_min, t_max = [0], [0]\n\n        y, sr = sf.read(f\"{self.data_path}/{recording_id}.flac\")\n        \n        len_y = len(y)\n        effective_length = sr * self.period\n        rint = np.random.randint(len(t_min))\n        tmin, tmax = round(sr * t_min[rint]), round(sr * t_max[rint])\n\n        if len_y < effective_length:\n            start = np.random.randint(effective_length - len_y)\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            new_y[start:start+len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            center = round((tmin + tmax) / 2)\n            big = center - effective_length\n            if big < 0:\n                big = 0\n            start = np.random.randint(big, center)\n            y = y[start:start+effective_length]\n            if len(y) < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start1 = np.random.randint(effective_length - len(y))\n                new_y[start1:start1+len(y)] = y\n                y = new_y.astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n            start = 0\n        \n        if self.transforms:\n            y = self.transforms(samples=y, sample_rate=sr)\n            \n        start_time = start / sr\n        end_time = (start + effective_length) / sr\n\n        label = np.zeros(24, dtype='f')\n\n        if self.train:\n            for i in range(len(t_min)):\n                if (t_min[i] >= start_time) & (t_max[i] <= end_time):\n                    label[species_id[i]] = 1\n                elif start_time <= ((t_min[i] + t_max[i]) / 2) <= end_time:\n                    label[species_id[i]] = 1\n        \n        return {\n            \"waveform\" : y,\n            \"target\" : torch.tensor(label, dtype=torch.float),\n            \"id\" : recording_id\n        }\n\nclass TestDataset(Dataset):\n    def __init__(self, df, period=10, transforms=None, data_path=\"train\", train=True):\n        self.period = period\n        self.transforms = transforms\n        self.data_path = data_path\n        self.train = train\n        \n        self.recording_ids = df[\"recording_id\"].values\n\n    \n    def __len__(self):\n        return len(self.recording_ids)\n    \n    def __getitem__(self, idx):\n\n        recording_id = self.recording_ids[idx]\n        \n        y, sr = sf.read(f\"{self.data_path}/{recording_id}.flac\")\n        \n        len_y = len(y)\n        effective_length = sr * self.period\n        \n        y_ = []\n        i = 0\n        while i < len_y:\n            y__ = y[i:i+effective_length]\n            \n            if self.transforms:\n                y__ = self.transforms(samples=y__, sample_rate=sr)\n                \n            y_.append(y__)\n            i = i + effective_length\n        \n        y = np.stack(y_)\n\n        label = np.zeros(24, dtype='f')\n        \n        return {\n            \"waveform\" : y,\n            \"target\" : torch.tensor(label, dtype=torch.float),\n            \"id\" : recording_id\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Models.py\nimport numpy as np\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.modules.dropout import Dropout\nfrom torch.nn.modules.linear import Linear\nfrom torch.nn.modules.pooling import AdaptiveAvgPool2d, AdaptiveMaxPool2d\n\nimport timm\nfrom timm.models.efficientnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, \\\n    tf_efficientnet_b5_ns, tf_efficientnet_b2_ns, tf_efficientnet_b6_ns, tf_efficientnet_b7_ns, tf_efficientnet_b0_ns\n\nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation\n\nfrom pytorch_utils import do_mixup, interpolate, pad_framewise_output\n\nencoder_params = {\n    \"resnest50d\" : {\n        \"features\" : 2048,\n        \"init_op\"  : partial(timm.models.resnest50d, pretrained=True, in_chans=1)\n    },\n    \"densenet201\" : {\n        \"features\": 1920,\n        \"init_op\": partial(timm.models.densenet201, pretrained=True)\n    },\n    \"dpn92\" : {\n        \"features\": 2688,\n        \"init_op\": partial(timm.models.dpn92, pretrained=True)\n    },\n    \"dpn131\": {\n        \"features\": 2688,\n        \"init_op\": partial(timm.models.dpn131, pretrained=True)\n    },\n    \"tf_efficientnet_b0_ns\": {\n        \"features\": 1280,\n        \"init_op\": partial(tf_efficientnet_b0_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b3_ns\": {\n        \"features\": 1536,\n        \"init_op\": partial(tf_efficientnet_b3_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b2_ns\": {\n        \"features\": 1408,\n        \"init_op\": partial(tf_efficientnet_b2_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b4_ns\": {\n        \"features\": 1792,\n        \"init_op\": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b5_ns\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n    \"tf_efficientnet_b6_ns\": {\n        \"features\": 2304,\n        \"init_op\": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.2, in_chans=1)\n    },\n}\n\n\nclass AudioClassifier(nn.Module):\n    def __init__(self, encoder, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n        \n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        #self.max_pool = AdaptiveMaxPool2d((1, 1))\n        self.dropout = Dropout(0.3)\n        self.fc = Linear(encoder_params[encoder]['features'], classes_num)\n    \n    def forward(self, input, spec_aug=False, mixup_lambda=None):\n        #print(input.type())\n        x = self.spectrogram_extractor(input.float()) # (batch_size, 1, time_steps, freq_bins)\n        x = self.logmel_extractor(x) # (batch_size, 1, time_steps, mel_bins)\n\n        #if spec_aug:\n        #    x = self.spec_augmenter(x)\n        if self.training:\n            x = self.spec_augmenter(x)\n        \n        # Mixup on spectrogram\n        if mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n            #pass\n        \n        x = self.encoder.forward_features(x)\n        x = self.avg_pool(x).flatten(1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Augmentation.py\nimport audiomentations as A\n\naugmenter = A.Compose([\n    A.AddGaussianNoise(p=0.4),\n    A.AddGaussianSNR(p=0.4),\n    #A.AddBackgroundNoise(\"../input/train_audio/\", p=1)\n    #A.AddImpulseResponse(p=0.1),\n    #A.AddShortNoises(\"../input/train_audio/\", p=1)\n    #A.FrequencyMask(min_frequency_band=0.0,  max_frequency_band=0.2, p=0.05),\n    #A.TimeMask(min_band_part=0.0, max_band_part=0.2, p=0.05),\n    #A.PitchShift(min_semitones=-0.5, max_semitones=0.5, p=0.05),\n    #A.Shift(p=0.1),\n    #A.Normalize(p=0.1),\n    #A.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=1, p=0.05),\n    #A.PolarityInversion(p=0.05),\n    A.Gain(p=0.2)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Utils.py\nimport torch\nimport numpy as np\nfrom sklearn import metrics\nfrom sklearn.metrics import log_loss\n\n# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/198418#1086063\ndef _one_sample_positive_class_precisions(scores, truth):\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n\n    retrieved_classes = np.argsort(scores)[::-1]\n\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\ndef lwlrap(truth, scores):\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = _one_sample_positive_class_precisions(scores[sample_num, :], truth[sample_num, :])\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = precision_at_hits\n\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    return per_class_lwlrap, weight_per_class\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass MetricMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = []\n        self.y_pred = []\n    \n    def update(self, y_true, y_pred):\n        self.y_true.extend(y_true.cpu().detach().numpy().tolist())\n        self.y_pred.extend(torch.sigmoid(y_pred).cpu().detach().numpy().tolist())\n\n    @property\n    def avg(self):\n        \n        score_class, weight = lwlrap(np.array(self.y_true), np.array(self.y_pred))\n        self.score = (score_class * weight).sum()\n\n        return {\n            \"lwlrap\" : self.score\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Losses.py\nfrom torch.nn import BCEWithLogitsLoss, CrossEntropyLoss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Functions.py\nfrom tqdm import tqdm\n\nimport numpy as np\nimport torch, torch.nn as nn\nimport torch.nn.functional as F\n\nfrom Utils import AverageMeter, MetricMeter\n\ndef train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch):\n    losses = AverageMeter()\n    scores = MetricMeter()\n\n    model.train()\n    #scaler = torch.cuda.amp.GradScaler()\n\n    t = tqdm(loader)\n    for i, sample in enumerate(t):\n        optimizer.zero_grad()\n        input = sample['waveform'].to(args.device)\n        target = sample['target'].to(args.device)\n        #print(input.shape)\n        #with torch.cuda.amp.autocast(enabled=args.amp):\n        output = model(input)\n        loss = criterion(output, target)\n        #scaler.scale(loss).backward()\n        #scaler.step(optimizer)\n        #scaler.update()\n        loss.backward()\n        optimizer.step()\n        if scheduler and args.step_scheduler:\n            scheduler.step()\n\n        bs = input.size(0)\n        scores.update(target, output)\n        losses.update(loss.item(), bs)\n\n        t.set_description(f\"Train E:{epoch} - Loss{losses.avg:0.4f}\")\n    t.close()\n    return scores.avg, losses.avg\n\ndef valid_epoch(args, model, loader, criterion, epoch):\n    losses = AverageMeter()\n    scores = MetricMeter()\n\n    model.eval()\n\n    with torch.no_grad():\n        t = tqdm(loader)\n        for i, sample in enumerate(t):\n            input = sample['waveform'].to(args.device)\n            target = sample['target'].to(args.device)\n            output = model(input)\n            loss = criterion(output, target)\n\n            bs = input.size(0)\n            scores.update(target, output)\n            losses.update(loss.item(), bs)\n            t.set_description(f\"Valid E:{epoch} - Loss:{losses.avg:0.4f}\")\n    t.close()\n    return scores.avg, losses.avg\n\ndef test_epoch(args, model, loader):\n    model.eval()\n    pred_list = []\n    id_list = []\n    with torch.no_grad():\n        t = tqdm(loader)\n        for i, sample in enumerate(t):\n            input = sample[\"waveform\"].to(args.device)\n            bs, seq, w = input.shape\n            input = input.reshape(bs*seq, w)\n            id = sample[\"id\"]\n            output = torch.sigmoid(model(input))\n            output = output.reshape(bs, seq, -1)\n            output, _ = torch.max(output, dim=1)\n            output = output.cpu().detach().numpy().tolist()\n            pred_list.extend(output)\n            id_list.extend(id)\n    \n    return pred_list, id_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile Run.py\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os, time, librosa, random\nimport numpy as np, pandas as pd\n\nimport torch, torch.nn as nn\nimport torch.nn.functional as F\n\nfrom transformers import get_linear_schedule_with_warmup\nfrom tqdm import tqdm\n\ntry:\n    import wandb\nexcept:\n    wandb = False\n\nimport Datasets\nimport Models\nimport Losses\nimport Functions\nimport Augmentation\n\nclass args:\n    DEBUG = False\n    amp = False\n    wandb = False\n    exp_name = \"resnest50d_5fold_base\"\n    network = \"AudioClassifier\"\n    pretrain_weights = None\n    model_param = {\n        'encoder' : 'resnest50d',\n        'sample_rate': 48000,\n        'window_size' : 2048,\n        #'win_length' : 1024,\n        'hop_size' : 512,\n        'mel_bins' : 128,\n        'fmin' : 0,\n        'fmax' : 24000,\n        'classes_num' : 24\n    }\n    losses = \"BCEWithLogitsLoss\"\n    lr = 1e-3\n    step_scheduler = True\n    epoch_scheduler = False\n    period = 30\n    seed = 42\n    start_epoch = 0\n    epochs = 50\n    batch_size = 16\n    num_workers = 2\n    early_stop = 10\n\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n    train_csv = \"train_folds.csv\"\n    test_csv = \"test_df.csv\"\n    sub_csv = \"../input/rfcx-species-audio-detection/sample_submission.csv\"\n    output_dir = \"weights\"\n\ndef main(fold):\n\n    # Setting seed\n    seed = args.seed\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n    args.fold = fold\n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)\n\n    train_df = pd.read_csv(args.train_csv)\n    #test_df = pd.read_csv(args.test_csv)\n    sub_df = pd.read_csv(args.sub_csv)\n    if args.DEBUG:\n        train_df = train_df.sample(1000)\n    train_fold = train_df[train_df.kfold != fold]\n    valid_fold = train_df[train_df.kfold == fold]\n\n    train_dataset = Datasets.AudioDataset(\n        df=train_fold,\n        period=args.period,\n        transforms=Augmentation.augmenter,\n        train=True,\n        data_path=\"../input/rfcx-species-audio-detection/train\"\n    )\n    valid_dataset = Datasets.AudioDataset(\n        df=valid_fold,\n        period=args.period,\n        transforms=None,\n        train=True,\n        data_path=\"../input/rfcx-species-audio-detection/train\"\n    )\n    \n    test_dataset = Datasets.TestDataset(\n        df=sub_df,\n        period=args.period,\n        transforms=None,\n        train=False,\n        data_path=\"../input/rfcx-species-audio-detection/test\"\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        shuffle=True,\n        drop_last=True,\n        num_workers=args.num_workers\n    )\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n    \n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=args.batch_size//2,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n\n    model = Models.__dict__[args.network](**args.model_param)\n    model = model.to(args.device)\n\n    if args.pretrain_weights:\n        print(\"---------------------loading pretrain weights\")\n        model.load_state_dict(torch.load(args.pretrain_weights, map_location=args.device)[\"model\"], strict=False)\n        model = model.to(args.device)\n\n    criterion = Losses.__dict__[args.losses]()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)\n    num_train_steps = int(len(train_loader) * args.epochs)\n    num_warmup_steps = int(0.1 * args.epochs * len(train_loader))\n    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_train_steps)\n    \n    \n    best_lwlrap = -np.inf\n    for epoch in range(args.start_epoch, args.epochs):\n        train_avg, train_loss = Functions.train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch)\n        valid_avg, valid_loss = Functions.valid_epoch(args, model, valid_loader, criterion, epoch)\n        \n        if args.epoch_scheduler:\n            scheduler.step()\n\n        content = f\"\"\"\n                {time.ctime()} \\n\n                Fold:{args.fold}, Epoch:{epoch}, lr:{optimizer.param_groups[0]['lr']:.7}\\n\n                Train Loss:{train_loss:0.4f} - LWLRAP:{train_avg['lwlrap']:0.4f}\\n\n                Valid Loss:{valid_loss:0.4f} - LWLRAP:{valid_avg['lwlrap']:0.4f}\\n\n        \"\"\"\n        print(content)\n        with open(f'{args.save_path}/log_{args.exp_name}.txt', 'a') as appender:\n            appender.write(content+'\\n')\n        \n        if valid_avg['lwlrap'] > best_lwlrap:\n            print(f\"########## >>>>>>>> Model Improved From {best_lwlrap} ----> {valid_avg['lwlrap']}\")\n            torch.save(model.state_dict(), os.path.join(args.save_path, f'fold-{args.fold}.bin'))\n            best_lwlrap = valid_avg['lwlrap']\n        #torch.save(model.state_dict(), os.path.join(args.save_path, f'fold-{args.fold}_last.bin'))\n    \n    \n    model.load_state_dict(torch.load(os.path.join(args.save_path, f'fold-{args.fold}.bin'), map_location=args.device))\n    model = model.to(args.device)\n\n    target_cols = sub_df.columns[1:].values.tolist()\n    test_pred, ids = Functions.test_epoch(args, model, test_loader)\n    print(np.array(test_pred).shape)\n    \n    test_pred_df = pd.DataFrame({\n        \"recording_id\" : sub_df.recording_id.values\n    })\n    test_pred_df[target_cols] = test_pred\n    test_pred_df.to_csv(os.path.join(args.save_path, f\"fold-{args.fold}-submission.csv\"), index=False)\n    print(os.path.join(args.save_path, f\"fold-{args.fold}-submission.csv\"))\n\n    #oof_pred, ids = Functions.test_epoch(args, model, valid_loader)\n    #oof_pred_df = pd.DataFrame({\n    #    \"recording_id\" : ids\n    #})\n    #oof_pred_df[target_cols] = oof_pred\n    #oof_pred_df.to_csv(os.path.join(args.save_path, f\"oof-fold-{args.fold}.csv\"), index=False)\n    \nif __name__ == \"__main__\":\n    for fold in range(5):\n        if fold == 0:\n            main(fold)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!python Run.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}