{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport IPython\nimport IPython.display\nimport PIL\nimport time\nimport sklearn.metrics\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_FOLD = 5\nNUM_CLASS = 80\nSEED = 42\nNUM_EPOCH = 64*6\nNUM_CYCLE = 64\nBATCH_SIZE = 64\nBATCH_SIZE_VALID = 32\nDO_TRAIN = True\nDO_EVALUATE = True\nDEBUG = False\nEXPERIMENT = False\nEXPERIMENT_FOLD = 1\nFOLD_LIST = [4]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.optim as optim\nimport torch.utils.data\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport torchvision.models as models\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels\nimport pretrainedmodels\nimport pretrainedmodels.utils\nclass ResNet(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ResNet, self).__init__()\n\n        self.num_classes = num_classes\n        self.mode = 'train'\n\n        self.base_model = pretrainedmodels.__dict__['resnet34'](num_classes=num_classes, pretrained=None)\n\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = self.base_model.bn1\n        self.relu = self.base_model.relu\n        self.maxpool = self.base_model.maxpool\n        self.layer1 = self.base_model.layer1\n        self.layer2 = self.base_model.layer2\n        self.layer3 = self.base_model.layer3\n        self.layer4 = self.base_model.layer4\n        self.avgpool = nn.AdaptiveMaxPool2d((1, 1))\n        self.last_linear = nn.Linear(self.base_model.layer4[1].conv1.in_channels, num_classes)\n        self.last_linear = nn.Sequential(\n            nn.Linear(self.base_model.layer4[1].conv1.in_channels, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, NUM_CLASS),\n        )\n        self.last_linear2 = nn.Sequential(\n            nn.Linear(self.base_model.layer4[1].conv1.in_channels, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024),\n            nn.ReLU(),\n            nn.Dropout(p=0.1),\n            nn.Linear(1024, NUM_CLASS),\n        )\n\n\n    def feature(self, input):\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        # x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        return x\n\n    def forward(self, input):\n        bs, ch, h, w = input.size()\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x4).view(bs, -1) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.last_linear(x) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n\n        return x\n    \n    def noisy(self, input):\n        bs, ch, h, w = input.size()\n        x0 = self.conv1(input)  #; print('layer conv1 ',x.size()) # [8, 64, 112, 112]\n        x0 = self.bn1(x0)\n        x0 = self.relu(x0)\n        x1 = self.maxpool(x0)\n        x1 = self.layer1(x1) #  ; print('layer 1 ',x.size()) # [8, 1024, 28, 28])\n        x2 = self.layer2(x1) #  ; print('layer 2 ',x.size()) # [8, 1024, 28, 28])\n        x3 = self.layer3(x2) #  ; print('layer 3 ',x.size()) # [8, 1024, 28, 28])\n        x4 = self.layer4(x3) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.avgpool(x4).view(bs, -1) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n        x = self.last_linear2(x) #  ; print('layer 4 ',x.size()) # [8, 2048, 14, 14])\n\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport librosa\nclass MfccDataset(Dataset):\n    \"\"\"Dataset wrapping images and target labels for Kaggle - Planet Amazon from Space competition.\n\n    Arguments:\n        A CSV file path\n        Path to image folder\n        Extension of images\n        PIL transforms\n    \"\"\"\n\n    def __init__(self, df, load_dir, slice=-1, mixup=False, \n                 cutout=False, cutout_h=False, cutout_w=False,\n                 gain=False, resize=False,\n                flip=False,\n                highpass=False,\n                ):\n        self.X_train = df['fname']\n        self.y_train = df.iloc[:,2:].values\n        self.slice = slice\n        self.mixup = mixup\n        self.cutout = cutout\n        self.cutout_h = cutout_h\n        self.cutout_w = cutout_w\n        self.gain = gain\n        self.resize = resize\n        self.highpass = highpass\n        self.flip = flip\n        self.load_dir = load_dir\n        # print(self.y_train.shape)\n\n    def do_slice(self, img):\n        if self.slice!=-1:\n            img_new = np.zeros([img.shape[0], self.slice], np.float32)\n            if img.shape[1]<self.slice:\n                shift = np.random.randint(0, self.slice - img.shape[1])\n                img_new[:, shift:shift + img.shape[1]] =img\n            elif img.shape[1]==self.slice:\n                img_new = img\n            else:\n                shift = np.random.randint(0, img.shape[1]-self.slice)\n                img_new = img[:, shift:shift+self.slice]\n        else:\n            # print(img_base.shape)\n            img_new = img\n        return img_new\n    \n    def do_highpass(self, img):\n        coord = np.random.randint(0, img.shape[0])\n        img[coord:] = 0\n    \n    def do_mixup(self, img, label):\n        idx = np.random.randint(0,len(self.X_train))\n        img2 = np.load(\"{}/{}.npy\".format(self.load_dir, self.X_train[idx][:-4]))\n        img2 = self.do_slice(img2)\n        \n        label2 = self.y_train[idx].astype(np.float32)\n\n        rate = np.random.random()\n        img = img*rate + img2*(1-rate)\n        label = label*rate + label2*(1-rate)\n        return img, label\n    \n    def do_flip(self, img):\n        return img[:,::-1]\n    \n    \n    def do_cutout_h(self, img, max = 32):\n        coord = np.random.randint(0, img.shape[0])\n        width = np.random.randint(8, max)\n        cut = np.array([coord-width, coord+width])\n        cut = np.clip(cut, 0, img.shape[0])\n        img[cut[0]:cut[1]] = 0\n        return img\n    \n    \n    def do_cutout_w(self, img, max = 32):\n        coord = np.random.randint(0, img.shape[1])\n        width = np.random.randint(8, max)\n        cut = np.array([coord-width, coord+width])\n        cut = np.clip(cut, 0, img.shape[1])\n        img[:,cut[0]:cut[1]] = 0\n        return img\n    \n    def do_highpass(self, img):\n        th = np.random.randint(0, img.shape[0])\n        img[th:] = 0\n        return img\n    \n    def cutout_bug(self, img):\n        coordx = np.sort(np.random.randint(0, self.slice,2))\n        coordy = np.sort(np.random.randint(0, 128, 2))\n        img[coordx[0]:coordx[1]] = 0\n        return img\n        \n    def do_resize(self, img, max=0.1):\n        rate = 1- max + np.random.random() * max * 2\n        img_tmp = cv2.resize(img, (int(self.slice*rate), img.shape[0], ))\n        if rate>1:\n            img_new = img_tmp[:,:img.shape[1]]\n        else:\n            img_new = np.zeros_like(img)\n            img_new[:,:img_tmp.shape[1]] = img_tmp\n        return img\n\n    \n    def do_gain(self, img, max=0.1):\n        rate = 1- max + np.random.random() * max * 2\n        return img * rate\n    \n    def __getitem__(self, index):\n        img = np.load(\"{}/{}.npy\".format(self.load_dir, self.X_train[index][:-4]))\n        img = self.do_slice(img)\n        label = self.y_train[index].astype(np.float32)\n\n        if self.mixup and np.random.random()<0.5:\n            img, label = self.do_mixup(img, label)\n        if self.gain and np.random.random()<0.5:\n             img = self.do_gain(img)\n        if self.resize and np.random.random()<0.5:\n             img = self.do_resize(img)\n        if self.cutout and np.random.random()<0.5:\n            img = self.cutout_bug(img)\n        if self.cutout_h and np.random.random()<0.5:\n            img = self.do_cutout_h(img)\n        if self.cutout_w and np.random.random()<0.5:\n            img = self.do_cutout_w(img)\n        if self.flip and np.random.random()<0.5:\n            img = self.do_flip(img)\n        if self.highpass and np.random.random()<0.5:\n            img = self.do_highpass(img)\n            \n            \n        img = librosa.power_to_db(img)\n        img = (img - img.mean()) / (img.std()+1e-7)\n        img = img.reshape([1, img.shape[0], img.shape[1]])\n        \n        return img, label\n\n    def __len__(self):\n        return len(self.X_train.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/freesound-audio-tagging-2019/train_curated.csv\")\ndf_test = pd.read_csv(\"../input/freesound-audio-tagging-2019/sample_submission.csv\")\ndf_noise = pd.read_csv(\"../input/freesound-audio-tagging-2019/train_noisy.csv\")\nlabels = df_test.columns[1:].tolist()\n\nfor label in labels:\n    print(label)\n    df_train[label] = df_train['labels'].apply(lambda x: label in x)\n    df_noise[label] = df_noise['labels'].apply(lambda x: label in x)\nprint(df_train.shape, df_noise.shape, df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold, StratifiedKFold\n# set log columns\nfolds = list(KFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED).split(np.arange(len(df_train))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import cos, pi\n\n\ndef cycle(iterable):\n    \"\"\"\n    dataloaderをiteratorに変換\n    :param iterable:\n    :return:\n    \"\"\"\n    while True:\n        for x in iterable:\n            yield x\n            \ndef _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n      scores: np.array of (num_classes,) giving the individual classifier scores.\n      truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n      pos_class_indices: np.array of indices of the true classes for this sample.\n      pos_class_precisions: np.array of precisions corresponding to each of those\n        classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample.\n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n            retrieved_cumulative_hits[class_rankings[pos_class_indices]] /\n            (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\n# All-in-one calculation of per-class lwlrap.\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n      truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n        of presence of that class in that sample.\n      scores: np.array of (num_samples, num_classes) giving the classifier-under-\n        test's real-valued score for each class for each sample.\n\n    Returns:\n      per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each\n        class.\n      weight_per_class: np.array of (num_classes,) giving the prior of each\n        class within the truth labels.  Then the overall unbalanced lwlrap is\n        simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n            _one_sample_positive_class_precisions(scores[sample_num, :],\n                                                  truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) /\n                        np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class\n\n\nclass CosineLR(_LRScheduler):\n    \"\"\"SGD with cosine annealing.\n    \"\"\"\n\n    def __init__(self, optimizer, step_size_min=1e-5, t0=100, tmult=2, curr_epoch=-1, last_epoch=-1):\n        self.step_size_min = step_size_min\n        self.t0 = t0\n        self.tmult = tmult\n        self.epochs_since_restart = curr_epoch\n        super(CosineLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        self.epochs_since_restart += 1\n\n        if self.epochs_since_restart > self.t0:\n            self.t0 *= self.tmult\n            self.epochs_since_restart = 0\n\n        lrs = [self.step_size_min + (\n                    0.5 * (base_lr - self.step_size_min) * (1 + cos(self.epochs_since_restart * pi / self.t0)))\n               for base_lr in self.base_lrs]\n\n        # print(lrs)\n\n        return lrs\n\n    \nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(train_loaders, model, optimizer, scheduler, epoch):\n    train_loader, noise_itr = train_loaders\n    bce_avr   = AverageMeter()\n    bce_noise_avr   = AverageMeter()\n    criterion_bce    = nn.BCEWithLogitsLoss().cuda()\n    sigmoid = torch.nn.Sigmoid().cuda()\n\n    # switch to train mode\n    model.train()\n\n    starttime = time.time()\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    preds_noise = np.zeros([0, NUM_CLASS], np.float32)\n    y_true_noise = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(train_loader):\n        input = input.cuda(async=True)\n        target = target.cuda(async=True)\n        input_var = torch.autograd.Variable(input)\n        target_var = torch.autograd.Variable(target)\n\n        input_noise, target_noise = next(noise_itr)  # test dataのバッチ\n        input_noise = torch.autograd.Variable(input_noise.cuda(async=True))\n        target_noise = torch.autograd.Variable(target_noise.cuda(async=True))\n\n        # compute output\n        output = model(input_var)\n        bce = criterion_bce(output, target_var)\n        output_noise = model.noisy(input_noise)\n        bce_noise = criterion_bce(output_noise, target_noise)\n        loss = bce + bce_noise\n        pred = sigmoid(output)\n        pred = pred.data.cpu().numpy()\n        pred_noise = sigmoid(output_noise)\n        pred_noise = pred_noise.data.cpu().numpy()\n        bce_avr.update(bce.data, input.size(0))\n        bce_noise_avr.update(bce_noise.data, input.size(0))\n\n        # compute gradient and do SGD step\n        optimizer.zero_grad()  # # 勾配の初期化\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n        preds_noise = np.concatenate([preds_noise, pred_noise])\n        y_true_noise = np.concatenate([y_true_noise, target_noise.data.cpu().numpy()])\n\n    # print(preds.shape, y_true.shape)\n    # print(y_true[:,:-1].shape, preds[:,:-1].shape)\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true[:,:-1], preds[:,:-1])\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n    # print(y_true_noise[:,:-1].shape, preds_noise[:,:-1].shape)\n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true_noise[:,:-1], preds_noise[:,:-1])\n    lwlrap_noise = np.sum(per_class_lwlrap * weight_per_class)\n    return bce_avr.avg.item(), lwlrap, bce_noise_avr.avg.item(), lwlrap_noise\n\n\ndef validate(val_loader, model):\n    bce_avr = AverageMeter()\n\n    sigmoid = torch.nn.Sigmoid().cuda()\n\n    criterion_bce = nn.BCEWithLogitsLoss().cuda()\n\n    # switch to train mode\n    model.eval()\n\n    starttime = time.time()\n    preds = np.zeros([0, NUM_CLASS], np.float32)\n    y_true = np.zeros([0, NUM_CLASS], np.float32)\n    for i, (input, target) in enumerate(val_loader):\n        input = input.cuda(async=True)\n        target = target.cuda(async=True)\n        input_var = torch.autograd.Variable(input)\n        target_var = torch.autograd.Variable(target)\n        # print(input.size())\n\n        # compute output\n        with torch.no_grad():\n            output = model(input_var)\n            bce = criterion_bce(output, target_var)\n            pred = sigmoid(output)\n            pred = pred.data.cpu().numpy()\n\n        # measure accuracy and record loss\n        bce_avr.update(bce.data, input.size(0))\n        preds = np.concatenate([preds, pred])\n        y_true = np.concatenate([y_true, target.data.cpu().numpy()])\n        \n        \n    per_class_lwlrap, weight_per_class = calculate_per_class_lwlrap(y_true, preds)\n    lwlrap = np.sum(per_class_lwlrap * weight_per_class)\n\n    return bce_avr.avg.item(), lwlrap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# baseline\n\nlog_columns = ['epoch', 'bce', 'lwlrap', 'bce_noise', 'lwlrap_noise', 'val_bce', 'val_lwlrap', 'time']  # 学習ログのカラム名\nfor fold, (ids_train_split, ids_valid_split) in enumerate(folds):\n    print(\"fold: {}\".format(fold + 1))\n    if fold+1 not in FOLD_LIST: continue\n    starttime = time.time()\n    train_log = pd.DataFrame(columns=log_columns)\n\n    # build model\n    model = ResNet(NUM_CLASS).cuda()\n\n    # set generator\n    df_train_fold = df_train.iloc[ids_train_split].reset_index(drop=True)\n    dataset_train = MfccDataset(df_train_fold, \"../input/mel128v3/train/\", \n                                slice=1024, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n                                resize=True,\n                               )\n    train_loader = DataLoader(dataset_train,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True  # CUDA only\n                              )\n\n    df_valid = df_train.iloc[ids_valid_split].reset_index(drop=True)\n    dataset_valid = MfccDataset(df_valid, \"../input/mel128v3/train/\")\n    valid_loader = DataLoader(dataset_valid,\n                              batch_size=1,\n                              shuffle=False,\n                              num_workers=1,  # 1 for CUDA\n                              pin_memory=True  # CUDA only\n                              )\n\n    dataset_noise = MfccDataset(df_noise, \"../input/mel128v3n/noise/\",\n                                slice=1024, \n                                mixup=True, \n                                cutout_h=True,\n                                gain=True,\n                                resize=True,\n                               )\n    noise_loader = DataLoader(dataset_noise,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True,\n                              num_workers=1,  # 1 for CUDA?\n                              pin_memory=True  # CUDA only\n                              )\n    noise_itr = cycle(noise_loader)  # dataloaderをgeneratorに変換\n\n    # set optimizer and loss\n    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)\n    scheduler = CosineLR(optimizer, step_size_min=1e-6, t0=len(train_loader) * NUM_CYCLE, tmult=1)\n#     scheduler = CosineLR(optimizer, step_size_min=1e-6, t0=len(train_loader) * 512, tmult=1)\n    cudnn.benchmark = True\n\n    # training\n    for epoch in range(NUM_EPOCH):\n        # train for one epoch\n        bce, lwlrap, bce_noise, lwlrap_noise = train(\n            (train_loader, noise_itr),\n             model, optimizer, scheduler, epoch\n        )\n\n        # evaluate on validation set\n        val_bce, val_lwlrap = validate(valid_loader, model)\n        # print(lwlrap)\n        # print(val_lwlrap)\n        \n        endtime = time.time() - starttime\n        print(\"Epoch: {}/{} \".format(epoch + 1, NUM_EPOCH)\n              + \"CE: {:.4f} \".format(bce)\n              + \"LwLRAP: {:.4f} \".format(lwlrap)\n              + \"noise CE: {:.4f} \".format(bce_noise)\n              + \"noise LwLRAP: {:.4f} \".format(lwlrap_noise)\n              + \"Valid CE: {:.4f} \".format(val_bce)\n              + \"Valid LwLRAP: {:.4f} \".format(val_lwlrap)\n              + \"sec: {:.1f}\".format(endtime)\n              )\n        train_log_epoch = pd.DataFrame([[epoch+1, bce, lwlrap, bce_noise, lwlrap_noise, val_bce, val_lwlrap, endtime]],\n                               columns=log_columns)\n        train_log = pd.concat([train_log, train_log_epoch])\n        train_log.to_csv(\"train_log_fold{}.csv\".format(fold+1), index=False)\n        if (epoch+1)%NUM_CYCLE==0:\n            torch.save(model.state_dict(), \"weight_fold_{}_epoch_{}.pth\".format(fold+1, epoch+1))\n    torch.save(optimizer.state_dict(), 'optimizer_fold_{}_epoch_{}.pth'.format(fold+1, epoch+1))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}