{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip -q install --upgrade pip\n!pip -q install timm\n!pip -q install torchlibrosa\n!pip -q install audiomentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, glob, random, time\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport librosa, librosa.display\nimport soundfile as sf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom tqdm import tqdm\nfrom functools import partial\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom transformers import get_linear_schedule_with_warmup\nfrom torchlibrosa.stft import Spectrogram, LogmelFilterBank\nfrom torchlibrosa.augmentation import SpecAugmentation\n\nimport timm\nfrom timm.models.efficientnet import tf_efficientnet_b0_ns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n            \n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n    \n    \ndef init_weights(model):\n    classname = model.__class__.__name__\n    if classname.find(\"Conv2d\") != -1:\n        nn.init.xavier_uniform_(model.weight, gain=np.sqrt(2))\n        model.bias.data.fill_(0)\n    elif classname.find(\"BatchNorm\") != -1:\n        model.weight.data.normal_(1.0, 0.02)\n        model.bias.data.fill_(0)\n    elif classname.find(\"GRU\") != -1:\n        for weight in model.parameters():\n            if len(weight.size()) > 1:\n                nn.init.orghogonal_(weight.data)\n    elif classname.find(\"Linear\") != -1:\n        model.weight.data.normal_(0, 0.01)\n        model.bias.data.zero_()\n\n        \ndef do_mixup(x: torch.Tensor, mixup_lambda: torch.Tensor):\n    \"\"\"Mixup x of even indexes (0, 2, 4, ...) with x of odd indexes\n    (1, 3, 5, ...).\n    Args:\n      x: (batch_size * 2, ...)\n      mixup_lambda: (batch_size * 2,)\n    Returns:\n      out: (batch_size, ...)\n    \"\"\"\n    out = (x[0::2].transpose(0, -1) * mixup_lambda[0::2] +\n           x[1::2].transpose(0, -1) * mixup_lambda[1::2]).transpose(0, -1)\n    return out\n\n\nclass Mixup(object):\n    def __init__(self, mixup_alpha, random_seed=1234):\n        \"\"\"Mixup coefficient generator.\n        \"\"\"\n        self.mixup_alpha = mixup_alpha\n        self.random_state = np.random.RandomState(random_seed)\n\n    def get_lambda(self, batch_size):\n        \"\"\"Get mixup random coefficients.\n        Args:\n          batch_size: int\n        Returns:\n          mixup_lambdas: (batch_size,)\n        \"\"\"\n        mixup_lambdas = []\n        for n in range(0, batch_size, 2):\n            lam = self.random_state.beta(self.mixup_alpha, self.mixup_alpha, 1)[0]\n            mixup_lambdas.append(lam)\n            mixup_lambdas.append(1. - lam)\n\n        return torch.from_numpy(np.array(mixup_lambdas, dtype=np.float32))\n\n    \ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.conv2 = nn.Conv2d(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=(3, 3),\n            stride=(1, 1),\n            padding=(1, 1),\n            bias=False)\n\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.init_weight()\n\n    def init_weight(self):\n        init_layer(self.conv1)\n        init_layer(self.conv2)\n        init_bn(self.bn1)\n        init_bn(self.bn2)\n\n    def forward(self, input, pool_size=(2, 2), pool_type='avg'):\n\n        x = input\n        x = F.relu_(self.bn1(self.conv1(x)))\n        x = F.relu_(self.bn2(self.conv2(x)))\n        if pool_type == 'max':\n            x = F.max_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg':\n            x = F.avg_pool2d(x, kernel_size=pool_size)\n        elif pool_type == 'avg+max':\n            x1 = F.avg_pool2d(x, kernel_size=pool_size)\n            x2 = F.max_pool2d(x, kernel_size=pool_size)\n            x = x1 + x2\n        else:\n            raise Exception('Incorrect argument!')\n\n        return x\n    \n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 5\nSEED = 42\n\ntrain = pd.read_csv(\"../input/rfcx-species-audio-detection/train_tp.csv\").sort_values(\"recording_id\")\nss = pd.read_csv(\"../input/rfcx-species-audio-detection/sample_submission.csv\")\n\ntrain_gby = train.groupby(\"recording_id\")[[\"species_id\"]].first().reset_index()\ntrain_gby = train_gby.sample(frac=1, random_state=SEED).reset_index(drop=True)\ntrain_gby.loc[:, 'kfold'] = -1\n\nX = train_gby[\"recording_id\"].values\ny = train_gby[\"species_id\"].values\n\nkfold = StratifiedKFold(n_splits=FOLDS)\nfor fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n    train_gby.loc[v_idx, \"kfold\"] = fold\n\ntrain = train.merge(train_gby[['recording_id', 'kfold']], on=\"recording_id\", how=\"left\")\nprint(train.kfold.value_counts())\ntrain.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_params = {\n    \"tf_efficientnet_b0_ns\": {\n        \"features\": 1280,\n        \"init_op\": partial(tf_efficientnet_b0_ns, pretrained=True, drop_path_rate=0.2)\n    }\n}\n\n\nclass AudioSEDModel(nn.Module):\n    def __init__(self, encoder, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):\n        super().__init__()\n\n        window = 'hann'\n        center = True\n        pad_mode = 'reflect'\n        ref = 1.0\n        amin = 1e-10\n        top_db = None\n        self.interpolate_ratio = 30  # Downsampled ratio\n\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size, \n            win_length=window_size, window=window, center=center, pad_mode=pad_mode, \n            freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size, \n            n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db, \n            freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2, \n            freq_drop_width=8, freq_stripes_num=2)\n        \n        # Model Encoder\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.fc1 = nn.Linear(encoder_params[encoder][\"features\"], 1024, bias=True)\n        self.att_block = AttBlock(1024, classes_num, activation=\"sigmoid\")\n        self.bn0 = nn.BatchNorm2d(mel_bins)\n        self.init_weight()\n    \n    def init_weight(self):\n        init_layer(self.fc1)\n        init_bn(self.bn0)\n    \n    def forward(self, input, mixup_lambda=None):\n        \"\"\"Input : (batch_size, data_length)\"\"\"\n\n        x = self.spectrogram_extractor(input)\n        # batch_size x 1 x time_steps x freq_bins\n        x = self.logmel_extractor(x)\n        # batch_size x 1 x time_steps x mel_bins\n\n        frames_num = x.shape[2]\n\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        #print(x.shape)\n\n        if self.training:\n            x = self.spec_augmenter(x)\n        \n        # Mixup on spectrogram\n        if self.training and mixup_lambda is not None:\n            x = do_mixup(x, mixup_lambda)\n        \n        # Output shape (batch size, channels, time, frequency)\n        x = x.expand(x.shape[0], 3, x.shape[2], x.shape[3])\n        #print(x.shape)\n        x = self.encoder.forward_features(x)\n        #print(x.shape)\n        x = torch.mean(x, dim=3)\n        #print(x.shape)\n\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        #print(x.shape)\n\n        x = F.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.5, training=self.training)\n        #print(x.shape)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output' : framewise_output,\n            'logit' : logit,\n            'clipwise_output' : clipwise_output\n        }\n\n        return output_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_or_pad(y, sr, period, record, mode=\"train\"):\n    len_y = len(y)\n    effective_length = sr * period\n    rint = np.random.randint(len(record['t_min']))\n    time_start = record['t_min'][rint] * sr\n    time_end = record['t_max'][rint] * sr\n    if len_y > effective_length:\n        # Positioning sound slice\n        center = np.round((time_start + time_end) / 2)\n        beginning = center - effective_length / 2\n        if beginning < 0:\n            beginning = 0\n        beginning = np.random.randint(beginning, center)\n        ending = beginning + effective_length\n        if ending > len_y:\n            ending = len_y\n        beginning = ending - effective_length\n        y = y[beginning:ending].astype(np.float32)\n    else:\n        y = y.astype(np.float32)\n        beginning = 0\n        ending = effective_length\n\n\n    beginning_time = beginning / sr\n    ending_time = ending / sr\n    label = np.zeros(24, dtype='f')\n\n    for i in range(len(record['t_min'])):\n        if (record['t_min'][i] <= ending_time) and (record['t_max'][i] >= beginning_time):\n            label[record['species_id'][i]] = 1\n    \n    return y, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SedDataset:\n    def __init__(self, df, period=10, stride=5, audio_transform=None, data_path=\"train\", mode=\"train\"):\n\n        self.period = period\n        self.stride = stride\n        self.audio_transform = audio_transform\n        self.data_path = data_path\n        self.mode = mode\n\n        self.df = df.groupby(\"recording_id\").agg(lambda x: list(x)).reset_index()\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        record = self.df.iloc[idx]\n\n        y, sr = sf.read(f\"{self.data_path}/{record['recording_id']}.flac\")\n        \n        if self.mode != \"test\":\n            y, label = crop_or_pad(y, sr, period=self.period, record=record, mode=self.mode)\n\n            if self.audio_transform:\n                y = self.audio_transform(samples=y, sample_rate=sr)\n        else:\n            y_ = []\n            i = 0\n            effective_length = self.period * sr\n            stride = self.stride * sr\n            y = np.stack([y[i:i+effective_length].astype(np.float32) for i in range(0, 60*sr+stride-effective_length, stride)])\n            label = np.zeros(24, dtype='f')\n            if self.mode == \"valid\":\n                for i in record['species_id']:\n                    label[i] = 1\n        \n        return {\n            \"image\" : y,\n            \"target\" : label,\n            \"id\" : record['recording_id']\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import audiomentations as AA\n\ntrain_audio_transform = AA.Compose([\n    AA.AddGaussianNoise(p=0.5),\n    AA.AddGaussianSNR(p=0.5),\n    #AA.AddBackgroundNoise(\"../input/train_audio/\", p=1)\n    #AA.AddImpulseResponse(p=0.1),\n    #AA.AddShortNoises(\"../input/train_audio/\", p=1)\n    #AA.FrequencyMask(min_frequency_band=0.0,  max_frequency_band=0.2, p=0.1),\n    #AA.TimeMask(min_band_part=0.0, max_band_part=0.2, p=0.1),\n    #AA.PitchShift(min_semitones=-0.5, max_semitones=0.5, p=0.1),\n    #AA.Shift(p=0.1),\n    #AA.Normalize(p=0.1),\n    #AA.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=1, p=0.05),\n    #AA.PolarityInversion(p=0.05),\n    #AA.Gain(p=0.2)\n])\n\ndef _lwlrap_sklearn(truth, scores):\n    \"\"\"Reference implementation from https://colab.research.google.com/drive/1AgPdhSp7ttY18O3fEoHOQKlt_3HJDLi8\"\"\"\n    sample_weight = np.sum(truth > 0, axis=1)\n    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)\n    overall_lwlrap = metrics.label_ranking_average_precision_score(\n        truth[nonzero_weight_sample_indices, :] > 0, \n        scores[nonzero_weight_sample_indices, :], \n        sample_weight=sample_weight[nonzero_weight_sample_indices])\n    return overall_lwlrap\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass MetricMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = []\n        self.y_pred = []\n    \n    def update(self, y_true, y_pred):\n        self.y_true.extend(y_true.cpu().detach().numpy().tolist())\n        self.y_pred.extend(y_pred.cpu().detach().numpy().tolist())\n\n    @property\n    def avg(self):\n        #score_class, weight = lwlrap(np.array(self.y_true), np.array(self.y_pred))\n        self.score = _lwlrap_sklearn(np.array(self.y_true), np.array(self.y_pred)) #(score_class * weight).sum()\n        return {\n            \"lwlrap\" : self.score\n        }\n\ndef seed_everithing(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss\n\nclass PANNsLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.bce = nn.BCELoss()\n\n    def forward(self, input, target):\n        input_ = input[\"clipwise_output\"]\n        input_ = torch.where(torch.isnan(input_),\n                             torch.zeros_like(input_),\n                             input_)\n        input_ = torch.where(torch.isinf(input_),\n                             torch.zeros_like(input_),\n                             input_)\n\n        target = target.float()\n\n        return self.bce(input_, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(args, model, loader, criterion, optimizer, scheduler, epoch):\n    losses = AverageMeter()\n    scores = MetricMeter()\n\n    model.train()\n    t = tqdm(loader)\n    for i, sample in enumerate(t):\n        optimizer.zero_grad()\n        input = sample['image'].to(args.device)\n        target = sample['target'].to(args.device)\n        output = model(input)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        if scheduler and args.step_scheduler:\n            scheduler.step()\n\n        bs = input.size(0)\n        scores.update(target, torch.sigmoid(torch.max(output['framewise_output'], dim=1)[0]))\n        losses.update(loss.item(), bs)\n\n        t.set_description(f\"Train E:{epoch} - Loss{losses.avg:0.4f}\")\n    t.close()\n    return scores.avg, losses.avg\n        \ndef valid_epoch(args, model, loader, criterion, epoch):\n    losses = AverageMeter()\n    scores = MetricMeter()\n    model.eval()\n    with torch.no_grad():\n        t = tqdm(loader)\n        for i, sample in enumerate(t):\n            input = sample['image'].to(args.device)\n            target = sample['target'].to(args.device)\n            output = model(input)\n            loss = criterion(output, target)\n\n            bs = input.size(0)\n            scores.update(target, torch.sigmoid(torch.max(output['framewise_output'], dim=1)[0]))\n            losses.update(loss.item(), bs)\n            t.set_description(f\"Valid E:{epoch} - Loss:{losses.avg:0.4f}\")\n    t.close()\n    return scores.avg, losses.avg\n\ndef test_epoch(args, model, loader):\n    model.eval()\n    pred_list = []\n    id_list = []\n    with torch.no_grad():\n        t = tqdm(loader)\n        for i, sample in enumerate(t):\n            input = sample[\"image\"].to(args.device)\n            bs, seq, w = input.shape\n            input = input.reshape(bs*seq, w)\n            id = sample[\"id\"]\n            output = model(input)\n            output = torch.sigmoid(torch.max(output['framewise_output'], dim=1)[0])\n            output = output.reshape(bs, seq, -1)\n            output = torch.sum(output, dim=1)\n            #output, _ = torch.max(output, dim=1)\n            output = output.cpu().detach().numpy().tolist()\n            pred_list.extend(output)\n            id_list.extend(id)\n    \n    return pred_list, id_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import get_cosine_schedule_with_warmup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main(fold):\n    seed_everithing(args.seed)\n\n    args.fold = fold\n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)\n\n    train_df = pd.read_csv(args.train_csv)\n    sub_df = pd.read_csv(args.sub_csv)\n    if args.DEBUG:\n        train_df = train_df.sample(200)\n    train_fold = train_df[train_df.kfold != fold]\n    valid_fold = train_df[train_df.kfold == fold]\n\n    train_dataset = SedDataset(\n        df = train_fold,\n        period=args.period,\n        audio_transform=train_audio_transform,\n        data_path=args.train_data_path,\n        mode=\"train\"\n    )\n\n    valid_dataset = SedDataset(\n        df = valid_fold,\n        period=args.period,\n        stride=5,\n        audio_transform=None,\n        data_path=args.train_data_path,\n        mode=\"valid\"\n    )\n\n    test_dataset = SedDataset(\n        df = sub_df,\n        period=args.period,\n        stride=5,\n        audio_transform=None,\n        data_path=args.test_data_path,\n        mode=\"test\"\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        shuffle=True,\n        drop_last=True,\n        num_workers=args.num_workers\n    )\n\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=args.batch_size,\n        shuffle=False,\n        drop_last=False,\n        num_workers=args.num_workers\n    )\n\n    model = AudioSEDModel(**args.model_param)\n    model = model.to(args.device)\n\n    if args.pretrain_weights:\n        print(\"---------------------loading pretrain weights\")\n        model.load_state_dict(torch.load(args.pretrain_weights, map_location=args.device), strict=False)\n        model = model.to(args.device)\n\n    criterion = PANNsLoss() #BCEWithLogitsLoss() #MaskedBCEWithLogitsLoss() #BCEWithLogitsLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)\n    num_train_steps = int(len(train_loader) * args.epochs)\n    num_warmup_steps = int(0.1 * args.epochs * len(train_loader))\n    # scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_train_steps)\n    scheduler = get_cosine_schedule_with_warmup(optimizer, \n                                                num_warmup_steps=num_warmup_steps, \n                                                num_training_steps=num_train_steps)\n    best_lwlrap = -np.inf\n    early_stop_count = 0\n\n    for epoch in range(args.start_epcoh, args.epochs):\n        train_avg, train_loss = train_epoch(args, model, train_loader, criterion, optimizer, scheduler, epoch)\n        valid_avg, valid_loss = valid_epoch(args, model, valid_loader, criterion, epoch)\n\n        if args.epoch_scheduler:\n            scheduler.step()\n        \n        content = f\"\"\"\n                {time.ctime()} \\n\n                Fold:{args.fold}, Epoch:{epoch}, lr:{optimizer.param_groups[0]['lr']:.7}\\n\n                Train Loss:{train_loss:0.4f} - LWLRAP:{train_avg['lwlrap']:0.4f}\\n\n                Valid Loss:{valid_loss:0.4f} - LWLRAP:{valid_avg['lwlrap']:0.4f}\\n\n        \"\"\"\n        print(content)\n        with open(f'{args.save_path}/log_{args.exp_name}.txt', 'a') as appender:\n            appender.write(content+'\\n')\n        \n        if valid_avg['lwlrap'] > best_lwlrap:\n            print(f\"########## >>>>>>>> Model Improved From {best_lwlrap} ----> {valid_avg['lwlrap']}\")\n            torch.save(model.state_dict(), os.path.join(args.save_path, f'fold-{args.fold}.bin'))\n            best_lwlrap = valid_avg['lwlrap']\n            early_stop_count = 0\n        else:\n            early_stop_count += 1\n        #torch.save(model.state_dict(), os.path.join(args.save_path, f'fold-{args.fold}_last.bin'))\n\n        if args.early_stop == early_stop_count:\n            print(\"\\n $$$ ---? Ohoo.... we reached early stoping count :\", early_stop_count)\n            break\n    \n    model.load_state_dict(torch.load(os.path.join(args.save_path, f'fold-{args.fold}.bin'), map_location=args.device))\n    model = model.to(args.device)\n\n    target_cols = sub_df.columns[1:].values.tolist()\n    test_pred, ids = test_epoch(args, model, test_loader)\n    print(np.array(test_pred).shape)\n\n    test_pred_df = pd.DataFrame({\n        \"recording_id\" : sub_df.recording_id.values\n    })\n    test_pred_df[target_cols] = test_pred\n    test_pred_df.to_csv(os.path.join(args.save_path, f\"fold-{args.fold}-submission.csv\"), index=False)\n    print(os.path.join(args.save_path, f\"fold-{args.fold}-submission.csv\"))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class args:\n    DEBUG = False\n\n    exp_name = \"SED_E0_5F_BASE\"\n    pretrain_weights = None \n    model_param = {\n        'encoder' : 'tf_efficientnet_b0_ns',\n        'sample_rate': 48000,\n        'window_size' : 2048, # 512 * 2, # 512 * 2\n        'hop_size' : 512, # 345 * 2, # 320\n        'mel_bins' : 128, # 60\n        'fmin' : 20,\n        'fmax' : 16000, # 48000 // 2,\n        'classes_num' : 24\n    }\n    period = 10\n    seed = 42\n    start_epcoh = 0 \n    epochs = 50\n    lr = 1e-3\n    batch_size = 16\n    num_workers = 4\n    early_stop = 15\n    step_scheduler = True\n    epoch_scheduler = False\n\n    device = ('cuda' if torch.cuda.is_available() else 'cpu')\n    train_csv = \"train_folds.csv\"\n    test_csv = \"test_df.csv\"\n    sub_csv = \"../input/rfcx-species-audio-detection/sample_submission.csv\"\n    output_dir = \"weights\"\n    train_data_path = \"../input/rfcx-species-audio-detection/train\"\n    test_data_path = \"../input/rfcx-species-audio-detection/test\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    main(fold=i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r effnet_b0_sed_5_fold.zip weights/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls weights/SED_E0_5F_BASE/fold-0-submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(\"effnet_b0_sed_5_fold.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def ensemble(DataFrame1, DataFrame2, ..., DataFrame5):\n#     for i in range(24):\n#         name = \"s\" + str(i)\n#         DataFrame1[name] = 0.2*(DataFrame1[name]+ ... + DataFrame5[name])\n#     return DataFrame1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls weights/SED_E0_5F_BASE/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataframes = []\nfor file in Path(\"weights/SED_E0_5F_BASE/\").iterdir():\n    if file.name.endswith(\"csv\"):\n        df = pd.read_csv(file)\n        dataframes.append(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataframes[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_vals = []\nprediction = None\n\nfor df in dataframes:\n    current_vals = df.values[:, 1:]\n    if prediction is None:\n        prediction = current_vals\n    else:\n        prediction += current_vals\n        \nprediction /= 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vals_df = pd.DataFrame(prediction, columns=df.columns[1:])\nsub_df = pd.concat([df.recording_id, vals_df], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}