{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip3 install timm","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:11:56.164073Z","iopub.execute_input":"2022-09-04T16:11:56.164691Z","iopub.status.idle":"2022-09-04T16:12:09.171473Z","shell.execute_reply.started":"2022-09-04T16:11:56.164568Z","shell.execute_reply":"2022-09-04T16:12:09.170321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.callback.wandb import *\nfrom torch.utils.data import Dataset\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\n\nimport wandb\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nfrom sklearn.model_selection import StratifiedKFold # maintain distribution\n\nimport timm\nfrom timm import create_model\n\nimport gc\nfrom torch.autograd import Variable\n\n\nimport  os","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:09.174951Z","iopub.execute_input":"2022-09-04T16:12:09.175272Z","iopub.status.idle":"2022-09-04T16:12:14.073537Z","shell.execute_reply.started":"2022-09-04T16:12:09.175240Z","shell.execute_reply":"2022-09-04T16:12:14.072414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Login to wandb","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb_secret\")","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:14.075468Z","iopub.execute_input":"2022-09-04T16:12:14.076117Z","iopub.status.idle":"2022-09-04T16:12:14.380332Z","shell.execute_reply.started":"2022-09-04T16:12:14.076051Z","shell.execute_reply":"2022-09-04T16:12:14.379326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.login(key = secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:14.383063Z","iopub.execute_input":"2022-09-04T16:12:14.383473Z","iopub.status.idle":"2022-09-04T16:12:15.927154Z","shell.execute_reply.started":"2022-09-04T16:12:14.383434Z","shell.execute_reply":"2022-09-04T16:12:15.926155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    SEED = 0\n    folds = 5 \n    kfold_shuffle = True\n    \n    num_worker = 0\n    dl_shuffle = True\n    device = 'cuda'\n    project_name = 'HB-Competition'\n    entity = 'deltaechov'\n    exp_name = 'resnet50d-Baseline-003-acts-nn.Leakyrelu-1024-128-256patch_cx'\n    wandb_conf = dict(\n        seed = SEED,\n        lr_ft = 3e-04,\n        lr = 1e-04,\n        wd = 0.,\n        batch_size = 2,\n        architecture ='',\n        epochs = 200,\n        epochs_ft = 10,\n        optimizer ='AdamW',\n        run_id = 1,\n        kfolds = folds,\n        lr_schedular = 'OneCycleLR',\n        Loss_Function ='',\n        #'lr_patience' : cfg.lr_patience,\n        #'lr_pct_start':cfg.lr_pct_start,\n        #'lr_div_factor':cfg.lr_div_factor,\n        #'lr_final_div_factor':cfg.lr_final_div_factor,\n        #'dropout':cfg.dropout,\n        \n        Note = 'resnet50d dice_bceloss, act = nn.LeakyRelu, corrected mask, randomcrop, using pretrained  resnet50d-Baseline-001-acts-nn.Leakyrelu-1024-128patch '\n        \n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.928237Z","iopub.execute_input":"2022-09-04T16:12:15.929366Z","iopub.status.idle":"2022-09-04T16:12:15.937444Z","shell.execute_reply.started":"2022-09-04T16:12:15.929326Z","shell.execute_reply":"2022-09-04T16:12:15.936264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths","metadata":{}},{"cell_type":"code","source":"train_csv = Path(\"../input/hubmap-organ-segmentation/train.csv\")\ntrain_images_path = Path(\"../input/dataset-resize-hack-human-body/\")","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.939151Z","iopub.execute_input":"2022-09-04T16:12:15.939807Z","iopub.status.idle":"2022-09-04T16:12:15.947645Z","shell.execute_reply.started":"2022-09-04T16:12:15.939752Z","shell.execute_reply":"2022-09-04T16:12:15.946597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{}},{"cell_type":"code","source":"train_transform = A.Compose([\n    #A.Resize(224,224),\n    A.RandomResizedCrop(256,256),\n    A.Flip(),\n    A.RandomGridShuffle(),\n    A.ChannelDropout(),\n    A.ChannelShuffle(),\n    A.ColorJitter(),\n    A.Equalize(),\n    A.RandomBrightnessContrast(),\n    A.CLAHE(),\n    A.Normalize(),\n    ToTensorV2()\n])\n\nvalid_transform = A.Compose([\n    A.Resize(256,256),\n    A.Normalize(),\n    ToTensorV2()\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.948956Z","iopub.execute_input":"2022-09-04T16:12:15.949425Z","iopub.status.idle":"2022-09-04T16:12:15.958221Z","shell.execute_reply.started":"2022-09-04T16:12:15.949384Z","shell.execute_reply":"2022-09-04T16:12:15.957139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper function","metadata":{}},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] =str(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n# needed for Dataloader\ndef seed_worker(worker_id):\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.959870Z","iopub.execute_input":"2022-09-04T16:12:15.960484Z","iopub.status.idle":"2022-09-04T16:12:15.972274Z","shell.execute_reply.started":"2022-09-04T16:12:15.960443Z","shell.execute_reply":"2022-09-04T16:12:15.971291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Schedular function","metadata":{}},{"cell_type":"code","source":"def get_schedular(schedular,cfg, dls):\n    if schedular == 'OneCycleLR':\n        return True,torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr = cfg.lr,epochs = cfg.EPOCHS, steps_per_epoch=len(dls.train),\n                                                       pct_start=cfg.lr_pct_start,div_factor=cfg.lr_div_factor,final_div_factor=cfg.lr_final_div_factor)\n    elif schedular == 'ReduceLROnPlateau':\n        return False,torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=cfg.lr_patience)\n    elif schedular == 'CosineAnnealingWarmRestarts':\n        return True,torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0 =cfg.lr_T_0,T_mult=cfg.lr_T_mult, eta_min=cfg.lr_eta_min)\n    else:\n        return False,None","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.975175Z","iopub.execute_input":"2022-09-04T16:12:15.976298Z","iopub.status.idle":"2022-09-04T16:12:15.983690Z","shell.execute_reply.started":"2022-09-04T16:12:15.976209Z","shell.execute_reply":"2022-09-04T16:12:15.982739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read CSV","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(train_csv)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:15.988403Z","iopub.execute_input":"2022-09-04T16:12:15.989388Z","iopub.status.idle":"2022-09-04T16:12:16.367667Z","shell.execute_reply.started":"2022-09-04T16:12:15.989345Z","shell.execute_reply":"2022-09-04T16:12:16.366716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class HBDataset(Dataset):\n    def __init__(self,image_dir, df, transforms):\n        self.df = df\n        self.transforms = transforms\n        self.image_dir = image_dir\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        image_id = \"image/\"+str(self.df.id.iloc[idx]) +'.png'\n        mask_id = \"mask/\"+str(self.df.id.iloc[idx]) +'_mask.png'\n        \n        image = np.array(Image.open(self.image_dir / image_id), dtype = np.uint8)\n        mask = np.array(Image.open(self.image_dir / mask_id), dtype = np.uint8)\n        #mask = mask[None,:,:]\n        \n        \n        if self.transforms is not None:\n            transformed = self.transforms(image=image, mask=mask)\n            image = transformed['image']\n            mask = transformed['mask']\n        \n       \n        return image.type(torch.float), mask.type(torch.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.369191Z","iopub.execute_input":"2022-09-04T16:12:16.369545Z","iopub.status.idle":"2022-09-04T16:12:16.378989Z","shell.execute_reply.started":"2022-09-04T16:12:16.369509Z","shell.execute_reply":"2022-09-04T16:12:16.377857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HBDataset_new(Dataset):\n    def __init__(self,image_dir, image_list, transforms):\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.image_list = image_list\n        \n    def __len__(self):\n        return len(self.image_list)\n    \n    def __getitem__(self, idx):\n        image_id = \"train/\"+str(self.image_list[idx])\n        mask_id = \"masks/\"+str(self.image_list[idx]) \n        \n        image = np.array(Image.open(self.image_dir / image_id), dtype = np.uint8)\n        mask = np.array(Image.open(self.image_dir / mask_id), dtype = np.uint8)\n        #mask = mask[None,:,:]\n        \n        \n        if self.transforms is not None:\n            transformed = self.transforms(image=image, mask=mask)\n            image = transformed['image']\n            mask = transformed['mask']\n        \n       \n        return image.type(torch.float), mask.type(torch.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.380947Z","iopub.execute_input":"2022-09-04T16:12:16.381335Z","iopub.status.idle":"2022-09-04T16:12:16.393314Z","shell.execute_reply.started":"2022-09-04T16:12:16.381300Z","shell.execute_reply":"2022-09-04T16:12:16.392298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_DICE(input, target):\n    #input = input[:,1,:,:]\n    input = input[:,1,:,:]\n    input = torch.sigmoid(input)\n    input[input >= 0.7] = 1\n    input[input < 0.7] = 0\n    \n    inter = (input*target).float().sum().item()\n    union = (input+target).float().sum().item()\n    \n    dice = (2 * inter) / union\n    return dice","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.394709Z","iopub.execute_input":"2022-09-04T16:12:16.395308Z","iopub.status.idle":"2022-09-04T16:12:16.404075Z","shell.execute_reply.started":"2022-09-04T16:12:16.395273Z","shell.execute_reply":"2022-09-04T16:12:16.403220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# hinge loss","metadata":{}},{"cell_type":"code","source":"from __future__ import print_function, division\n\nimport torch\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nimport numpy as np\ntry:\n    from itertools import  ifilterfalse\nexcept ImportError: # py3k\n    from itertools import  filterfalse as ifilterfalse\n\n\ndef lovasz_grad(gt_sorted):\n    \"\"\"\n    Computes gradient of the Lovasz extension w.r.t sorted errors\n    See Alg. 1 in paper\n    \"\"\"\n    p = len(gt_sorted)\n    gts = gt_sorted.sum()\n    intersection = gts - gt_sorted.float().cumsum(0)\n    union = gts + (1 - gt_sorted).float().cumsum(0)\n    jaccard = 1. - intersection / union\n    if p > 1: # cover 1-pixel case\n        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]\n    return jaccard\n\n\ndef iou_binary(preds, labels, EMPTY=1., ignore=None, per_image=True):\n    \"\"\"\n    IoU for foreground class\n    binary: 1 foreground, 0 background\n    \"\"\"\n    if not per_image:\n        preds, labels = (preds,), (labels,)\n    ious = []\n    for pred, label in zip(preds, labels):\n        intersection = ((label == 1) & (pred == 1)).sum()\n        union = ((label == 1) | ((pred == 1) & (label != ignore))).sum()\n        if not union:\n            iou = EMPTY\n        else:\n            iou = float(intersection) / float(union)\n        ious.append(iou)\n    iou = mean(ious)    # mean accross images if per_image\n    return 100 * iou\n\n\ndef iou(preds, labels, C, EMPTY=1., ignore=None, per_image=False):\n    \"\"\"\n    Array of IoU for each (non ignored) class\n    \"\"\"\n    if not per_image:\n        preds, labels = (preds,), (labels,)\n    ious = []\n    for pred, label in zip(preds, labels):\n        iou = []    \n        for i in range(C):\n            if i != ignore: # The ignored label is sometimes among predicted classes (ENet - CityScapes)\n                intersection = ((label == i) & (pred == i)).sum()\n                union = ((label == i) | ((pred == i) & (label != ignore))).sum()\n                if not union:\n                    iou.append(EMPTY)\n                else:\n                    iou.append(float(intersection) / float(union))\n        ious.append(iou)\n    ious = [mean(iou) for iou in zip(*ious)] # mean accross images if per_image\n    return 100 * np.array(ious)\n\n\n# --------------------------- BINARY LOSSES ---------------------------\n\n\ndef lovasz_hinge(logits, labels, per_image=True, ignore=None):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n      per_image: compute the loss per image instead of per batch\n      ignore: void class id\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_hinge_flat(*flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))\n                          for log, lab in zip(logits, labels))\n    else:\n        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n    return loss\n\n\ndef lovasz_hinge_flat(logits, labels):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n      labels: [P] Tensor, binary ground truth labels (0 or 1)\n      ignore: label to ignore\n    \"\"\"\n    if len(labels) == 0:\n        # only void pixels, the gradients should be 0\n        return logits.sum() * 0.\n    signs = 2. * labels.float() - 1.\n    errors = (1. - logits * Variable(signs))\n    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)\n    perm = perm.data\n    gt_sorted = labels[perm]\n    grad = lovasz_grad(gt_sorted)\n    loss = torch.dot(F.relu(errors_sorted), Variable(grad))\n    return loss\n\n\ndef flatten_binary_scores(scores, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch (binary case)\n    Remove labels equal to 'ignore'\n    \"\"\"\n    scores = scores.view(-1)\n    labels = labels.view(-1)\n    if ignore is None:\n        return scores, labels\n    valid = (labels != ignore)\n    vscores = scores[valid]\n    vlabels = labels[valid]\n    return vscores, vlabels\n\n\nclass StableBCELoss(torch.nn.modules.Module):\n    def __init__(self):\n         super(StableBCELoss, self).__init__()\n    def forward(self, input, target):\n         neg_abs = - input.abs()\n         loss = input.clamp(min=0) - input * target + (1 + neg_abs.exp()).log()\n         return loss.mean()\n\n\ndef binary_xloss(logits, labels, ignore=None):\n    \"\"\"\n    Binary Cross entropy loss\n      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n      ignore: void class id\n    \"\"\"\n    logits, labels = flatten_binary_scores(logits, labels, ignore)\n    loss = StableBCELoss()(logits, Variable(labels.float()))\n    return loss\n\n\n# --------------------------- MULTICLASS LOSSES ---------------------------\n\n\ndef lovasz_softmax(probas, labels, classes='present', per_image=False, ignore=None):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [B, C, H, W] Variable, class probabilities at each prediction (between 0 and 1).\n              Interpreted as binary (sigmoid) output with outputs of size [B, H, W].\n      labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1)\n      classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.\n      per_image: compute the loss per image instead of per batch\n      ignore: void class labels\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_softmax_flat(*flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), classes=classes)\n                          for prob, lab in zip(probas, labels))\n    else:\n        loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore), classes=classes)\n    return loss\n\n\ndef lovasz_softmax_flat(probas, labels, classes='present'):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1)\n      labels: [P] Tensor, ground truth labels (between 0 and C - 1)\n      classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.\n    \"\"\"\n    if probas.numel() == 0:\n        # only void pixels, the gradients should be 0\n        return probas * 0.\n    C = probas.size(1)\n    losses = []\n    class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes\n    for c in class_to_sum:\n        fg = (labels == c).float() # foreground for class c\n        if (classes is 'present' and fg.sum() == 0):\n            continue\n        if C == 1:\n            if len(classes) > 1:\n                raise ValueError('Sigmoid output possible only with 1 class')\n            class_pred = probas[:, 0]\n        else:\n            class_pred = probas[:, c]\n        errors = (Variable(fg) - class_pred).abs()\n        errors_sorted, perm = torch.sort(errors, 0, descending=True)\n        perm = perm.data\n        fg_sorted = fg[perm]\n        losses.append(torch.dot(errors_sorted, Variable(lovasz_grad(fg_sorted))))\n    return mean(losses)\n\n\ndef flatten_probas(probas, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch\n    \"\"\"\n    if probas.dim() == 3:\n        # assumes output of a sigmoid layer\n        B, H, W = probas.size()\n        probas = probas.view(B, 1, H, W)\n    B, C, H, W = probas.size()\n    probas = probas.permute(0, 2, 3, 1).contiguous().view(-1, C)  # B * H * W, C = P, C\n    labels = labels.view(-1)\n    if ignore is None:\n        return probas, labels\n    valid = (labels != ignore)\n    vprobas = probas[valid.nonzero().squeeze()]\n    vlabels = labels[valid]\n    return vprobas, vlabels\n\ndef xloss(logits, labels, ignore=None):\n    \"\"\"\n    Cross entropy loss\n    \"\"\"\n    return F.cross_entropy(logits, Variable(labels), ignore_index=255)\n\n\n# --------------------------- HELPER FUNCTIONS ---------------------------\ndef isnan(x):\n    return x != x\n    \n    \ndef mean(l, ignore_nan=False, empty=0):\n    \"\"\"\n    nanmean compatible with generators.\n    \"\"\"\n    l = iter(l)\n    if ignore_nan:\n        l = ifilterfalse(isnan, l)\n    try:\n        n = 1\n        acc = next(l)\n    except StopIteration:\n        if empty == 'raise':\n            raise ValueError('Empty mean')\n        return empty\n    for n, v in enumerate(l, 2):\n        acc += v\n    if n == 1:\n        return acc\n    return acc / n","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.405631Z","iopub.execute_input":"2022-09-04T16:12:16.406165Z","iopub.status.idle":"2022-09-04T16:12:16.442755Z","shell.execute_reply.started":"2022-09-04T16:12:16.406073Z","shell.execute_reply":"2022-09-04T16:12:16.441749Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LovaszHingeLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True):\n        super(LovaszHingeLoss, self).__init__()\n\n    def forward(self, inputs, targets):\n        inputs = torch.sigmoid(inputs)    \n        Lovasz = lovasz_hinge(inputs, targets, per_image=False)                       \n        return Lovasz","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.446546Z","iopub.execute_input":"2022-09-04T16:12:16.447405Z","iopub.status.idle":"2022-09-04T16:12:16.457075Z","shell.execute_reply.started":"2022-09-04T16:12:16.447375Z","shell.execute_reply":"2022-09-04T16:12:16.456129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PyTorch\nclass DiceLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True):\n        super(DiceLoss, self).__init__()\n\n    def forward(self, inputs, targets, smooth=1):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = torch.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).sum()                            \n        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)  \n        \n        return 1 - dice","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.458537Z","iopub.execute_input":"2022-09-04T16:12:16.459615Z","iopub.status.idle":"2022-09-04T16:12:16.468425Z","shell.execute_reply.started":"2022-09-04T16:12:16.459579Z","shell.execute_reply":"2022-09-04T16:12:16.467271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c =  BCEWithLogitsLossFlat()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.471018Z","iopub.execute_input":"2022-09-04T16:12:16.471532Z","iopub.status.idle":"2022-09-04T16:12:16.478656Z","shell.execute_reply.started":"2022-09-04T16:12:16.471489Z","shell.execute_reply":"2022-09-04T16:12:16.477593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dice_bceloss(input, target):\n    bce_loss = BCEWithLogitsLossFlat(axis = 1)\n    dice_loss = DiceLoss()\n    \n    return bce_loss(input, target) + dice_loss(input, target)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.480453Z","iopub.execute_input":"2022-09-04T16:12:16.481120Z","iopub.status.idle":"2022-09-04T16:12:16.489957Z","shell.execute_reply.started":"2022-09-04T16:12:16.481056Z","shell.execute_reply":"2022-09-04T16:12:16.489127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hinge_bce_loss(input, target):\n    lh_loss = LovaszHingeLoss()\n    bce_loss = BCEWithLogitsLossFlat(axis = 1)\n    \n    return lh_loss(input, target) + bce_loss(input, target)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.491427Z","iopub.execute_input":"2022-09-04T16:12:16.492190Z","iopub.status.idle":"2022-09-04T16:12:16.501871Z","shell.execute_reply.started":"2022-09-04T16:12:16.492149Z","shell.execute_reply":"2022-09-04T16:12:16.500875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":" train_df, valid_df = train_test_split(df, test_size=0.2, random_state=CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.503589Z","iopub.execute_input":"2022-09-04T16:12:16.504306Z","iopub.status.idle":"2022-09-04T16:12:16.517613Z","shell.execute_reply.started":"2022-09-04T16:12:16.504270Z","shell.execute_reply":"2022-09-04T16:12:16.516633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = HBDataset(image_dir = train_images_path, df = train_df, transforms = train_transform)\nvalid_ds = HBDataset(image_dir = train_images_path, df = valid_df, transforms = valid_transform)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.521026Z","iopub.execute_input":"2022-09-04T16:12:16.521611Z","iopub.status.idle":"2022-09-04T16:12:16.530356Z","shell.execute_reply.started":"2022-09-04T16:12:16.521576Z","shell.execute_reply":"2022-09-04T16:12:16.529422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = DataLoader(train_ds, bs = CFG.wandb_conf['batch_size'], shuffle = CFG.dl_shuffle,\n                      num_workers = CFG.num_worker, pin_memory = True, worker_init_fn = seed_worker,)\nvalid_dl = DataLoader(valid_ds, bs = CFG.wandb_conf['batch_size'], shuffle = CFG.dl_shuffle, \n                      num_workers = CFG.num_worker, pin_memory = True,worker_init_fn = seed_worker)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.531576Z","iopub.execute_input":"2022-09-04T16:12:16.531833Z","iopub.status.idle":"2022-09-04T16:12:16.542046Z","shell.execute_reply.started":"2022-09-04T16:12:16.531809Z","shell.execute_reply":"2022-09-04T16:12:16.541048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = DataLoaders(train_dl,valid_dl,device = CFG.device)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.543859Z","iopub.execute_input":"2022-09-04T16:12:16.544695Z","iopub.status.idle":"2022-09-04T16:12:16.552279Z","shell.execute_reply.started":"2022-09-04T16:12:16.544660Z","shell.execute_reply":"2022-09-04T16:12:16.551028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#resnet50_model = resnet50()\n# remove last layers\nmodel = create_model('resnet50d',  pretrained=True)\nmodel = nn.Sequential(*list(model.children())[:-2])\nunet_model = DynamicUnet(model, 2, (256,256), norm_type=None,  act_cls = nn.LeakyReLU)\nunet_model.load_state_dict(torch.load(Path(f'../input/hubmapmodelweight/resnet50d-Baseline-003-acts-nn.Leakyrelu-1024-128patch_cx-1.pth'),map_location=torch.device(CFG.device)))\nunet_model.to(CFG.device);","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:16.566940Z","iopub.execute_input":"2022-09-04T16:12:16.567921Z","iopub.status.idle":"2022-09-04T16:12:34.692403Z","shell.execute_reply.started":"2022-09-04T16:12:16.567885Z","shell.execute_reply":"2022-09-04T16:12:34.691414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = Learner(dls, unet_model, loss_func=CrossEntropyLossFlat(axis =1),opt_func=Adam, metrics = custom_DICE, cbs = ShowGraphCallback())","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:34.693926Z","iopub.execute_input":"2022-09-04T16:12:34.694576Z","iopub.status.idle":"2022-09-04T16:12:34.701730Z","shell.execute_reply.started":"2022-09-04T16:12:34.694535Z","shell.execute_reply":"2022-09-04T16:12:34.700555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:34.703040Z","iopub.execute_input":"2022-09-04T16:12:34.704154Z","iopub.status.idle":"2022-09-04T16:12:34.713055Z","shell.execute_reply.started":"2022-09-04T16:12:34.704117Z","shell.execute_reply":"2022-09-04T16:12:34.711936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.init(project=CFG.project_name, entity = CFG.entity, group=CFG.exp_name, config = CFG.wandb_conf)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:34.718834Z","iopub.execute_input":"2022-09-04T16:12:34.719780Z","iopub.status.idle":"2022-09-04T16:12:38.545685Z","shell.execute_reply.started":"2022-09-04T16:12:34.719738Z","shell.execute_reply":"2022-09-04T16:12:38.544260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_seed(CFG.SEED)\nlearn.fit(n_epoch = CFG.wandb_conf['epochs'], lr=CFG.wandb_conf['lr'], wd=CFG.wandb_conf['wd'],  cbs = WandbCallback())\n\ntorch.save(unet_model.state_dict(), CFG.exp_name+'-'+str(CFG.wandb_conf['run_id'])+\".pth\")\nwandb.save(CFG.exp_name+'-'+str(CFG.wandb_conf['run_id'])+\".pth\") ","metadata":{"execution":{"iopub.status.busy":"2022-09-04T16:12:38.547904Z","iopub.execute_input":"2022-09-04T16:12:38.548856Z","iopub.status.idle":"2022-09-04T18:00:44.591041Z","shell.execute_reply.started":"2022-09-04T16:12:38.548813Z","shell.execute_reply":"2022-09-04T18:00:44.585141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = next(iter(valid_dl))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:00:44.592497Z","iopub.execute_input":"2022-09-04T18:00:44.592946Z","iopub.status.idle":"2022-09-04T18:00:46.078193Z","shell.execute_reply.started":"2022-09-04T18:00:44.592898Z","shell.execute_reply":"2022-09-04T18:00:46.076219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f =  unet_model(temp[0])","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:01:17.576528Z","iopub.execute_input":"2022-09-04T18:01:17.576992Z","iopub.status.idle":"2022-09-04T18:01:18.722574Z","shell.execute_reply.started":"2022-09-04T18:01:17.576942Z","shell.execute_reply":"2022-09-04T18:01:18.721538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f[0][1].max(), f[0][1].min()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:01:18.724671Z","iopub.execute_input":"2022-09-04T18:01:18.725375Z","iopub.status.idle":"2022-09-04T18:01:19.618157Z","shell.execute_reply.started":"2022-09-04T18:01:18.725334Z","shell.execute_reply":"2022-09-04T18:01:19.617135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.sigmoid(f[0][1]).min(), torch.sigmoid(f[0][1]).max()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:01:23.765218Z","iopub.execute_input":"2022-09-04T18:01:23.765723Z","iopub.status.idle":"2022-09-04T18:01:25.000168Z","shell.execute_reply.started":"2022-09-04T18:01:23.765676Z","shell.execute_reply":"2022-09-04T18:01:24.998341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = torch.sigmoid(f[1][1])\nc[c >= 0.7] = 1\nc[c < 0.7] = 0","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:07:22.075514Z","iopub.execute_input":"2022-09-04T18:07:22.075964Z","iopub.status.idle":"2022-09-04T18:07:23.126926Z","shell.execute_reply.started":"2022-09-04T18:07:22.075927Z","shell.execute_reply":"2022-09-04T18:07:23.125931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c.max()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:07:23.129015Z","iopub.execute_input":"2022-09-04T18:07:23.129786Z","iopub.status.idle":"2022-09-04T18:07:24.157862Z","shell.execute_reply.started":"2022-09-04T18:07:23.129743Z","shell.execute_reply":"2022-09-04T18:07:24.156880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.imshow(c.permute(1,2,0).detach().cpu().numpy())\nplt.imshow(c.detach().cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:07:24.159927Z","iopub.execute_input":"2022-09-04T18:07:24.160644Z","iopub.status.idle":"2022-09-04T18:07:25.544134Z","shell.execute_reply.started":"2022-09-04T18:07:24.160605Z","shell.execute_reply":"2022-09-04T18:07:25.542991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(temp[1][1].detach().cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:07:29.401714Z","iopub.execute_input":"2022-09-04T18:07:29.402172Z","iopub.status.idle":"2022-09-04T18:07:30.966977Z","shell.execute_reply.started":"2022-09-04T18:07:29.402132Z","shell.execute_reply":"2022-09-04T18:07:30.965993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(torch.sigmoid(f[1][1]).detach().cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2022-09-04T18:07:37.366804Z","iopub.execute_input":"2022-09-04T18:07:37.367293Z","iopub.status.idle":"2022-09-04T18:07:38.817000Z","shell.execute_reply.started":"2022-09-04T18:07:37.367251Z","shell.execute_reply":"2022-09-04T18:07:38.816016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}