{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Pytorch training only with efficientnet-b6\n\n- Using mixed precision training\n- Using gradient accumulation\n- Using group CV\n\nReached score of *8.49* on LB\n\nThis notebook is based on this one:\n\nhttps://www.kaggle.com/leighplt/pytorch-fcn-resnet50/comments\n\nTrained on private VM with NVIDIA Tesla T4 on Google Cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install segmentation-models-pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\n\nfrom tqdm.notebook import tqdm\n\nimport sys, os, random, time\nimport numba, cv2, gc\nimport pickle\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as D\n\nimport torchvision\nfrom torchvision import transforms as T\n\nfrom segmentation_models_pytorch.unetplusplus import UnetPlusPlus\nfrom segmentation_models_pytorch.unet import Unet\nimport segmentation_models_pytorch as smp\n\nimport rasterio\nfrom rasterio.windows import Window\n\nimport albumentations as A\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import GroupKFold\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seeds(seed = 42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    \nset_seeds()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup training variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = Path('/kaggle/input/hubmap-kidney-segmentation/')\nassert DATA_PATH.exists()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_decode(mask_rle, shape=(256, 256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    splits = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (splits[0:][::2], splits[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype='uint8')\n    for lo, hi in zip(starts, ends):\n        img[lo: hi] = 1\n    return img.reshape(shape, order='F') # Fortran order reshaping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@numba.njit()\ndef rle_numba(pixels):\n    size = len(pixels)\n    points = []\n    if pixels[0] == 1: points.append(1)\n    for i in range(1, size):\n        if pixels[i] != pixels[i-1]:\n            if len(points) % 2 == 0:\n                points.append(i+1)\n            else:\n                points.append(i+1 - points[-1])\n    if pixels[-1] == 1: points.append(size-points[-1]+1)    \n    return points","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check run length encoding starting with 0\nassert rle_numba([0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1]) == [2, 2, 5, 1, 7, 4, 12, 1]\n# Check run length encoding starting with 0\nassert rle_numba([1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1]) == [1, 3, 5, 1, 7, 4, 12, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_numba_encode(image):\n    pixels = image.flatten(order = 'F')\n    points = rle_numba(pixels)\n    return ' '.join(str(x) for x in points)\n\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(DATA_PATH / 'train.csv', index_col=[0])\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MASK_PATH = Path('/kaggle/working/ds_cache')\n!mkdir {MASK_PATH}\n\nimport shutil\n\ndef reset_mask_path():\n    shutil.rmtree(MASK_PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"WINDOW = 1024 # tile size\nMIN_OVERLAP = 32\nNEW_SIZE = 512 # size after re-size which are fed to the model\nTHRESHOLD = 0 # How many mask 1 pixels necessary to take image into training set","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This method below (*is_tile_contains_info*) was inspired by https://www.kaggle.com/mariazorkaltseva/hubmap-seresnext50-unet-dice-loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Used to filter tiles with enough color information in it\ndef is_tile_contains_info(img, pixel_limits = (50, 220), content_threshold = 0.08, expected_shape = (WINDOW, WINDOW, 3)):\n    \"\"\"\n    img: np.array\n    pixel_limits: tuple\n    content_threshold: float percents\n    expected_shape: tuple\n    \"\"\"\n    \n    left_limit = np.prod(img > pixel_limits[0], axis=-1)\n    right_limit =  np.prod(img < pixel_limits[1], axis=-1)\n\n    if img.shape != expected_shape:\n        print('img.shape != expected_shape', img.shape)\n        return False, 0.\n\n    percent_of_pixels = np.sum(left_limit*right_limit) / (img.shape[0] * img.shape[1])\n    return  percent_of_pixels > content_threshold, percent_of_pixels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\nidentity = None\n# normalize_transform = T.Normalize([0.625, 0.448, 0.688], [0.131, 0.177, 0.101])\n# normalize_transform = T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n# normalize_transform = T.Normalize([0.65459856,0.48386562,0.69428385], [0.15167958,0.23584107,0.13146145])\nnormalize_transform = T.Normalize([0.6130, 0.4126, 0.6595], [0.1417, 0.2045, 0.1237])\n\ndef read_from_slice(dataset, x1, x2, y1, y2):\n    image = dataset.read([1,2,3],\n                    window=Window.from_slices((x1,x2),(y1,y2)))\n    image = np.moveaxis(image, 0, -1)\n    return image\n\n\nclass HubDataset(D.Dataset):\n    '''\n    Dataset with caching support using pickle files for better training performance\n    '''\n    def __init__(self, root_dir, transform, window=256, overlap=32, threshold = THRESHOLD):\n        self.path = root_dir\n        assert self.path.exists()\n        self.overlap, self.window, self.transform, self.threshold = overlap, window, transform, threshold\n        self.csv = pd.read_csv(self.path / 'train.csv', index_col=[0])\n        self.build_slices()\n        self.len = len(self.slices)\n        self.as_tensor = T.Compose([\n            T.ToTensor(),\n            normalize_transform,\n        ])\n        \n        \n    def build_slices(self):\n        self.masks = []; self.files = []; self.slices = []\n        self.skipped = 0\n        slices_path = MASK_PATH/f'slices.pkl'\n        files_path = MASK_PATH/f'files.pkl'\n        masks_path = MASK_PATH/f'masks.pkl'\n        if not slices_path.exists():\n            for i, filename in tqdm(enumerate(self.csv.index), total = len(self.csv)):\n                filepath = self.path/'train'/f'{filename}.tiff'\n                assert filepath.exists()\n                self.files.append(filepath)\n                with rasterio.open(filepath) as dataset:\n                    self.build_slice(dataset, filename, i)\n            with open(slices_path, \"wb\") as filehandler:\n                pickle.dump(self.slices, filehandler)\n            with open(files_path, \"wb\") as filehandler:\n                pickle.dump(self.files, filehandler)\n            with open(masks_path, \"wb\") as filehandler:\n                pickle.dump(self.masks, filehandler)\n        else:\n            print('Reading cached slices, files and masks')\n            with open(slices_path,'rb') as file:\n                self.slices = pickle.load(file)\n            with open(files_path,'rb') as file:\n                self.files = pickle.load(file)\n            with open(masks_path,'rb') as file:\n                self.masks = pickle.load(file)\n                        \n    def build_slice(self, dataset, filename, i):\n        dataset_shape = dataset.shape\n        self.masks.append(rle_decode(self.csv.loc[filename, 'encoding'], dataset_shape))\n        slices = make_grid(dataset_shape, window = self.window, min_overlap = self.overlap)\n\n        # Simple way to create more data below by shifting slices\n        # Please note, that due to caching and file system quota, this does not work on Kaggle notebook\n        # Shifting slices to the right and bottom and adding to the original slices\n#         slices_copy = slices.copy()\n#         slices_copy_y = slices.copy()\n        # horizontal\n#         slices_copy[:,(0,1)] += WINDOW // 2 # shift\n#         slices = np.concatenate ([slices, slices_copy])\n        # vertical\n#         slices_copy_y[:,(2,3)] += WINDOW // 2\n#         slices = np.concatenate ([slices, slices_copy_y])\n#         slices = slices[~(slices[:,1] > dataset.shape[0]),:] # filter those outside of the screen\n#         slices = slices[~(slices[:,3] > dataset.shape[1]),:] # filter those outside of the screen\n\n        # Only including slices above a specific threshold\n        # Note: we are potentially throwing away some data here\n        for slc in slices:\n            x1, x2, y1, y2 = slc\n            image = read_from_slice(dataset, x1, x2 , y1, y2)\n            contains_info = is_tile_contains_info(image)\n            if self.masks[-1][x1:x2,y1:y2].sum() > self.threshold and contains_info[0]:\n                self.slices.append([i,x1,x2,y1,y2])\n            else:\n                self.skipped += 1\n                        \n                        \n    def apply_transform(self, image, mask):\n        augments = self.transform(image=image, mask=mask)\n        image = self.as_tensor(augments['image'])\n        mask = augments['mask'][None]\n        return image, mask\n        \n    def __getitem__(self, index):\n        image_path = MASK_PATH/f'image_{index}'\n        mask_path = MASK_PATH/f'mask_{index}'\n        if not image_path.exists():\n            idx = self.slices[index][0]\n            filename = self.files[idx]\n            x1, x2, y1, y2 = self.slices[index][1:]\n            with rasterio.open(filename) as dataset:\n                image = read_from_slice(dataset, x1, x2, y1, y2).astype('uint8')\n            mask = self.masks[idx][x1:x2,y1:y2]\n            with open(image_path, \"wb\") as filehandler:\n                pickle.dump(image, filehandler)\n                if index % 100 == 0:\n                    print(f'Writing to {image_path}')\n            with open(mask_path, \"wb\") as filehandler:\n                pickle.dump(mask, filehandler)\n            return self.apply_transform(image, mask)\n        else:\n            with open(image_path,'rb') as file:\n                image = pickle.load(file)\n            with open(mask_path,'rb') as file:\n                mask = pickle.load(file)\n            return self.apply_transform(image, mask)\n    \n    def __len__(self):\n        return self.len\n    \n    def __repr__(self):\n        return f'total: {len(self)}, skipped: {self.skipped}'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_ds(size):\n    trfm = A.Compose([\n        A.Resize(size, size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ColorJitter (brightness=0.07, contrast=0.07,\n                       saturation=0.1, hue=0.1, always_apply=False, p=0.3),\n        A.OneOf(\n            [\n                A.RandomContrast(p=1),\n                A.HueSaturationValue(p=1),\n                A.RandomBrightness(p=1)\n            ],\n            p=0.5,\n        )\n    ])\n\n    return HubDataset(DATA_PATH, window=WINDOW, overlap=MIN_OVERLAP, transform=trfm)\n\nds = generate_ds(NEW_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with rasterio.open(DATA_PATH/'train/2f6ecfcdf.tiff') as raster:\n    img = raster.read([1,2,3], window=Window.from_slices((3909, 4933),(11464,12488)))\n    img = np.moveaxis(img, 0, -1)\n    print(img.shape)\n    crs = raster.crs\n\nplt.figure(figsize = (20,20))\nplt.imshow(img)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_mask_img(idx):\n    image, mask = ds[idx]\n    plt.figure(figsize=(16,8))\n    plt.subplot(121)\n    plt.imshow(mask[0], cmap='gray')\n    plt.subplot(122)\n    plt.imshow(np.moveaxis(image.numpy(), 0, -1));\n\ndisplay_mask_img(2)\ndisplay_mask_img(1)\ndisplay_mask_img(0)\ndisplay_mask_img(3)\ndisplay_mask_img(5)\ndisplay_mask_img(6)\ndisplay_mask_img(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image, mask = ds[0]\n\n_ = rle_numba_encode(mask[0]) # compile function with numba","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Images and its corresponding masks are saved with the same filename.\ngroups = [ds.slices[i][0] for i in range(len(ds))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_kfold = GroupKFold(n_splits = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_split_on_index(img_index = 7):\n    valid_idx, train_idx = [], []\n    for i in range(len(ds)):\n        if ds.slices[i][0] == img_index:\n            valid_idx.append(i)\n        else:\n            train_idx.append(i)\n    return valid_idx, train_idx","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Data loaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 10\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_train_valid_dls(ds, train_idx, valid_idx):\n    train_ds = D.Subset(ds, train_idx)\n    valid_ds = D.Subset(ds, valid_idx)\n\n    # define training and validation data loaders\n    train_dl = D.DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\n\n    valid_dl = D.DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n    return train_dl, valid_dl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"ENCODER_NAME = 'efficientnet-b6'\n\nclass HuBMAPModel(nn.Module):\n    def __init__(self):\n        super(HuBMAPModel, self).__init__()\n        self.model = Unet(encoder_name = ENCODER_NAME, \n                          encoder_weights = 'imagenet',\n                          classes = 1,\n                          activation = None)\n        \n        \n    def forward(self, images):\n        img_masks = self.model(images)\n        return img_masks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    model = HuBMAPModel()\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 1e-3\nWD = 1e-3\nBEST_MODEL = f'best_model_unet_1024_double_shift_grad_acc_{ENCODER_NAME}.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_optimizer_scheduler(model, train_dl, epochs):\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WD)\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR,\n                                                    steps_per_epoch=len(train_dl), epochs=epochs)\n    return optimizer, scheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"dice_loss = smp.utils.losses.DiceLoss()\n\ndef loss_fn(y_pred, y_true):\n    return dice_loss(y_pred.sigmoid(), y_true)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jaccard_loss = smp.utils.losses.JaccardLoss()\n\ndef loss_fn(y_pred, y_true):\n    return jaccard_loss(y_pred.sigmoid(), y_true)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_metric(y_pred, y, epsilon = 1.0):\n    dims=(-2,-1)\n    x = (y_pred > 0).float()\n    dc = (2 * (x * y).sum(dims) + epsilon) / ((x + y).sum(dims) + epsilon)\n    return dc.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iou_metric = smp.utils.metrics.IoU(threshold=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\ndef train_epoch(model, dataloader, optim, criterion, scheduler, device=\"cpu\", grad_accu_steps=4):\n    model.train()\n    \n    train_loss = []\n    labels = []\n    outs = []\n    lrs = []\n    \n    tbar = tqdm(dataloader)\n    scaler = torch.cuda.amp.GradScaler() # mixed precision support\n    scale = None\n    for step, (image, target) in enumerate(tbar):\n        image, target = image.to(DEVICE), target.float().to(DEVICE)\n        \n        with torch.cuda.amp.autocast():\n            output = model(image)\n            loss = criterion(output, target) / grad_accu_steps\n        \n        scaler.scale(loss).backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        \n        if (step + 1) % grad_accu_steps == 0:\n            scaler.step(optim)\n            scale = scaler.get_scale()\n            scaler.update()\n            optim.zero_grad()\n        \n        skip_lr_sched = (scale != scaler.get_scale())\n        if not skip_lr_sched:\n            scheduler.step()\n        \n        loss_val = loss.item() * grad_accu_steps\n        train_loss.append(loss_val)\n        lrs.append(get_lr(optim))\n        \n        tbar.set_description('loss - {:.4f}'.format(loss_val))\n        \n    print(f'Train loss: {np.array(train_loss).mean()}')\n    return train_loss, lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def val_epoch(model, dataloader, criterion, epoch, device=\"cpu\"):\n    model.eval()\n\n    valid_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    dice_metrics = []\n    iou_metrics = []\n\n    for item in dataloader:\n        image, target = item\n        image, target = image.to(DEVICE), target.float().to(DEVICE)\n\n        with torch.no_grad():\n            output = model(image)\n            loss = criterion(output, target)\n            dice_metrics.append(dice_metric(output, target))\n            iou_metrics.append(iou_metric(output, target))\n        valid_loss.append(loss.item())\n\n    avg_loss = np.array(valid_loss).mean()\n    print(f'Epoch {epoch} - valid loss: {avg_loss}')\n    dice_metrics = [x.item() for x in dice_metrics]\n    iou_metrics = [x.item() for x in iou_metrics]\n    dice_metric_mean = np.array(dice_metrics).mean()\n    iou_metric_mean = np.array(iou_metrics).mean()\n    return dice_metrics, valid_loss, dice_metric_mean, avg_loss, iou_metric_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(epochs, train_dl, valid_dl, optimizer, scheduler, patience = 6):\n    best_loss = 100.0\n    best_metric = 0\n    train_losses = []\n    valid_losses = []\n    accumulated_lrs = []\n    accumulated_dice_metrics = []\n    early_stop_counter = 0\n\n    for epoch in tqdm(range(epochs)):\n        train_loss, lrs = train_epoch(model, train_dl, optimizer, loss_fn, scheduler, DEVICE)\n        dice_metrics, valid_loss, dice_metric_mean, avg_loss, iou_metric_mean = val_epoch(model, valid_dl, loss_fn, epoch, DEVICE)\n        train_losses += train_loss\n        valid_losses.append(np.array(valid_loss).mean())\n        accumulated_lrs += lrs\n        accumulated_dice_metrics.append(np.array(dice_metrics).mean())\n        if best_metric < iou_metric_mean:\n            best_metric = iou_metric_mean\n            print('Saving model')\n            torch.save(model.state_dict(), BEST_MODEL)\n            early_stop_counter = 0\n        else:\n            early_stop_counter += 1\n        if best_loss > avg_loss:\n            best_loss = avg_loss\n        print(f'Epoch {epoch} - val best loss {best_loss} dice metric ({dice_metric_mean}) iou metric ({iou_metric_mean}).')\n        if early_stop_counter >= patience:\n            print('Stopping early')\n            break\n        \n    return train_losses, valid_losses, accumulated_lrs, accumulated_dice_metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reset_mask_path()\n# !mkdir {MASK_PATH}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 10\nPATIENCE = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_split(fold_info, fold):\n    global model\n    \n    print(f'Processing fold {fold}')\n    model = get_model()\n    model.to(DEVICE)\n    train_idx, valid_idx = fold_info[fold]\n    f'Proportions valid / train: {len(valid_idx) / len(train_idx)}'\n    train_dl, valid_dl = generate_train_valid_dls(ds, train_idx, valid_idx)\n    optimizer, scheduler = create_optimizer_scheduler(model, train_dl, EPOCHS)\n    train_losses, valid_losses, accumulated_lrs, accumulated_dice_metrics = train(EPOCHS, train_dl, valid_dl, optimizer, scheduler, patience = PATIENCE)\n    return train_losses, valid_losses, accumulated_lrs, accumulated_dice_metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold_info = [(train_idx, valid_idx) for fold, (train_idx, valid_idx) in tqdm(enumerate(group_kfold.split(ds.slices, \n                                                        groups = groups)), total=FOLDS)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx, valid_idx = fold_info[0]\ntrain_dl, valid_dl = generate_train_valid_dls(ds, train_idx, valid_idx)\nfor image, target in tqdm(train_dl):\n    pass\nfor image, target in tqdm(valid_dl):\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_split_and_move(fold_info, fold):\n    train_losses, valid_losses, accumulated_lrs, accumulated_dice_metrics = train_split(fold_info, fold)\n    !mv {BEST_MODEL} {fold}_{BEST_MODEL}\n    stats_df = pd.DataFrame({'train_losses': train_losses, 'accumulated_lrs': accumulated_lrs})\n    stats_df[['train_losses']].plot()\n    val_stats_df = pd.DataFrame({'valid_losses': valid_losses})\n    val_stats_df[['valid_losses']].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_split_and_move(fold_info, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_split_and_move(fold_info, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_split_and_move(fold_info, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_split_and_move(fold_info, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}