{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install timm\n!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport copy\nimport time\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = \"../input/cassava-leaf-disease-classification\"\nTRAIN_DIR = \"../input/cassava-leaf-disease-classification/train_images\"\nTEST_DIR = \"../input/cassava-leaf-disease-classification/test_images\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    model_name = 'tf_efficientnet_b4_ns'\n    img_size = 512\n    scheduler = 'CosineAnnealingWarmRestarts'\n    T_max = 10\n    T_0 = 10\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 16\n    weight_decay = 1e-6\n    seed = 42\n    num_classes = 5\n    num_epochs = 10\n    n_fold = 5\n    smoothing = 0.2\n    t1 = 0.8\n    t2 = 1.4\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CFG.n_fold)\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n    df.loc[val_ , \"kfold\"] = int(fold)\n    \ndf['kfold'] = df['kfold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLeafDataset(nn.Module):\n    def __init__(self, root_dir, df, transforms=None):\n        self.root_dir = root_dir\n        self.df = df\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.df.iloc[index, 0])\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.df.iloc[index, 1]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n        A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n        A.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2()], p=1.),\n    \n    \"valid\": A.Compose([\n        A.CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code taken from https://github.com/mlpanda/bi-tempered-loss-pytorch/blob/master/bi_tempered_loss.py\n\n# def log_t(u, t):\n#     \"\"\"Compute log_t for `u`.\"\"\"\n\n#     if t == 1.0:\n#         return torch.log(u)\n#     else:\n#         return (u ** (1.0 - t) - 1.0) / (1.0 - t)\n\n\n# def exp_t(u, t):\n#     \"\"\"Compute exp_t for `u`.\"\"\"\n\n#     if t == 1.0:\n#         return torch.exp(u)\n#     else:\n#         return torch.relu(1.0 + (1.0 - t) * u) ** (1.0 / (1.0 - t))\n\n\n# def compute_normalization_fixed_point(activations, t, num_iters=5):\n#     \"\"\"Returns the normalization value for each example (t > 1.0).\n#     Args:\n#     activations: A multi-dimensional tensor with last dimension `num_classes`.\n#     t: Temperature 2 (> 1.0 for tail heaviness).\n#     num_iters: Number of iterations to run the method.\n#     Return: A tensor of same rank as activation with the last dimension being 1.\n#     \"\"\"\n\n#     mu = torch.max(activations, dim=-1).values.view(-1, 1)\n#     normalized_activations_step_0 = activations - mu\n\n#     normalized_activations = normalized_activations_step_0\n#     i = 0\n#     while i < num_iters:\n#         i += 1\n#         logt_partition = torch.sum(exp_t(normalized_activations, t), dim=-1).view(-1, 1)\n#         normalized_activations = normalized_activations_step_0 * (logt_partition ** (1.0 - t))\n\n#     logt_partition = torch.sum(exp_t(normalized_activations, t), dim=-1).view(-1, 1)\n\n#     return -log_t(1.0 / logt_partition, t) + mu\n\n\n# def compute_normalization(activations, t, num_iters=5):\n#     \"\"\"Returns the normalization value for each example.\n#     Args:\n#     activations: A multi-dimensional tensor with last dimension `num_classes`.\n#     t: Temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n#     num_iters: Number of iterations to run the method.\n#     Return: A tensor of same rank as activation with the last dimension being 1.\n#     \"\"\"\n\n#     if t < 1.0:\n#         return None # not implemented as these values do not occur in the authors experiments...\n#     else:\n#         return compute_normalization_fixed_point(activations, t, num_iters)\n\n\n# def tempered_softmax(activations, t, num_iters=5):\n#     \"\"\"Tempered softmax function.\n#     Args:\n#     activations: A multi-dimensional tensor with last dimension `num_classes`.\n#     t: Temperature tensor > 0.0.\n#     num_iters: Number of iterations to run the method.\n#     Returns:\n#     A probabilities tensor.\n#     \"\"\"\n\n#     if t == 1.0:\n#         normalization_constants = torch.log(torch.sum(torch.exp(activations), dim=-1))\n#     else:\n#         normalization_constants = compute_normalization(activations, t, num_iters)\n\n#     return exp_t(activations - normalization_constants, t)\n\n\n# def bi_tempered_logistic_loss(activations, labels, t1, t2, label_smoothing=0.0, num_iters=5):\n\n#     \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n#     Args:\n#     activations: A multi-dimensional tensor with last dimension `num_classes`.\n#     labels: A tensor with shape and dtype as activations.\n#     t1: Temperature 1 (< 1.0 for boundedness).\n#     t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n#     label_smoothing: Label smoothing parameter between [0, 1).\n#     num_iters: Number of iterations to run the method.\n#     Returns:\n#     A loss tensor.\n#     \"\"\"\n\n#     if label_smoothing > 0.0:\n#         num_classes = labels.shape[-1]\n#         labels = (1 - num_classes / (num_classes - 1) * label_smoothing) * labels + label_smoothing / (num_classes - 1)\n\n#     probabilities = tempered_softmax(activations, t2, num_iters)\n\n#     temp1 = (log_t(labels + 1e-10, t1) - log_t(probabilities, t1)) * labels\n#     temp2 = (1 / (2 - t1)) * (torch.pow(labels, 2 - t1) - torch.pow(probabilities, 2 - t1))\n#     loss_values = temp1 - temp2\n\n#     return torch.sum(loss_values, dim=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code taken from https://github.com/fhopfmueller/bi-tempered-loss-pytorch/blob/master/bi_tempered_loss_pytorch.py\n\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold):\n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    history = defaultdict(list)\n    scaler = amp.GradScaler()\n\n    for epoch in range(1,num_epochs+1):\n        print('Epoch {}/{}'.format(epoch, num_epochs))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train','valid']:\n            if(phase == 'train'):\n                model.train() # Set model to training mode\n            else:\n                model.eval() # Set model to evaluation mode\n            \n            running_loss = 0.0\n            running_corrects = 0.0\n            \n            # Iterate over data\n            for inputs,labels in tqdm(dataloaders[phase]):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    with amp.autocast():\n                        outputs = model(inputs)\n                        _, preds = torch.max(outputs,1)\n                        loss = bi_tempered_logistic_loss(outputs, labels, t1=CFG.t1, t2=CFG.t2, label_smoothing=CFG.smoothing)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        scaler.scale(loss).backward()\n                        scaler.step(optimizer)\n                        scaler.update()\n\n\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data).double().item()\n\n            \n            epoch_loss = running_loss/dataset_sizes[phase]\n            epoch_acc = running_corrects/dataset_sizes[phase]\n\n            history[phase + ' loss'].append(epoch_loss)\n            history[phase + ' acc'].append(epoch_acc)\n\n            if phase == 'train' and scheduler != None:\n                scheduler.step()\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n            \n            # deep copy the model\n            if phase=='valid' and epoch_acc >= best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n                PATH = f\"Fold{fold}_{best_acc}_epoch{epoch}.bin\"\n                torch.save(model.state_dict(), PATH)\n\n        print()\n\n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Accuracy \",best_acc)\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_fold(model, optimizer, scheduler, device, fold, num_epochs=10):\n    valid_df = df[df.kfold == fold]\n    train_df = df[df.kfold != fold]\n    \n    train_data = CassavaLeafDataset(TRAIN_DIR, train_df, transforms=data_transforms[\"train\"])\n    valid_data = CassavaLeafDataset(TRAIN_DIR, valid_df, transforms=data_transforms[\"valid\"])\n    \n    dataset_sizes = {\n        'train' : len(train_data),\n        'valid' : len(valid_data)\n    }\n    \n    train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=True)\n    valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=False)\n    \n    dataloaders = {\n        'train' : train_loader,\n        'valid' : valid_loader\n    }\n\n    model, history = train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold)\n    \n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = timm.create_model(CFG.model_name, pretrained=True)\nnum_features = model.classifier.in_features\n# for param in model.parameters():\n#     param.requires_grad = False\nmodel.classifier = nn.Linear(num_features, CFG.num_classes)\nmodel.to(CFG.device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)\n    elif CFG.scheduler == None:\n        return None\n        \n    return scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scheduler = fetch_scheduler(optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model, history = run_fold(model, optimizer, scheduler, device=CFG.device, fold=0, num_epochs=CFG.num_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}