{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":6942858,"sourceType":"datasetVersion","datasetId":3987147}],"dockerImageVersionId":30580,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q segmentation_models_pytorch","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:37:32.452398Z","iopub.execute_input":"2023-12-06T21:37:32.453131Z","iopub.status.idle":"2023-12-06T21:37:52.805051Z","shell.execute_reply.started":"2023-12-06T21:37:32.453086Z","shell.execute_reply":"2023-12-06T21:37:52.803968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nimport gc\nimport time\nfrom collections import defaultdict\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nimport copy\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nimport torch.nn.functional as F\nfrom torch.cuda import amp\nimport torch.optim as optim\nimport albumentations as A\nimport segmentation_models_pytorch as smp\nfrom sklearn.model_selection import train_test_split\nfrom colorama import Fore, Back, Style\nc_ = Fore.GREEN\nsr_ = Style.RESET_ALL\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-06T21:37:52.807141Z","iopub.execute_input":"2023-12-06T21:37:52.807448Z","iopub.status.idle":"2023-12-06T21:38:01.186785Z","shell.execute_reply.started":"2023-12-06T21:37:52.807421Z","shell.execute_reply":"2023-12-06T21:38:01.185963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n seed = 42\n debug = False # set debug=False for Full Training\n exp_name = 'baseline'\n comment = 'unet-efficientnet_b1-512x512'\n output_dir = './'\n model_name = 'Unet'\n backbone = 'resnext50_32x4d'\n train_bs = 5\n valid_bs = 5\n img_size = [512, 512]\n epochs = 8\n n_accumulate = max(1, 64//train_bs)\n lr = 1e-3\n scheduler = 'CosineAnnealingLR'\n min_lr = 1e-6\n T_max = int(2279/(train_bs*n_accumulate)*epochs)+50\n T_0 = 25\n warmup_epochs = 0\n wd = 1e-6\n n_fold = 5\n num_classes = 1\n device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n gt_df = \"/kaggle/input/sennet-hoa-gt-data/gt.csv\"\n data_root = \"/kaggle/input\"\n train_groups = [\"kidney_1_dense\", 'kidney_2']\n valid_groups = [\"kidney_3_sparse\"]\n loss_func = \"DiceLoss\"\n\n data_transforms = {\n \"train\": A.Compose([\n A.Resize(*img_size, interpolation=cv2.INTER_NEAREST),\n A.HorizontalFlip(p=0.5),\n ], p=1.0),\n \n \"valid\": A.Compose([\n A.Resize(*img_size, interpolation=cv2.INTER_NEAREST),\n ], p=1.0)\n }","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:01.1881Z","iopub.execute_input":"2023-12-06T21:38:01.188419Z","iopub.status.idle":"2023-12-06T21:38:01.268854Z","shell.execute_reply.started":"2023-12-06T21:38:01.188392Z","shell.execute_reply":"2023-12-06T21:38:01.26762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\nset_seed(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:01.270249Z","iopub.execute_input":"2023-12-06T21:38:01.270595Z","iopub.status.idle":"2023-12-06T21:38:01.316522Z","shell.execute_reply.started":"2023-12-06T21:38:01.270553Z","shell.execute_reply":"2023-12-06T21:38:01.315746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataLoader","metadata":{}},{"cell_type":"code","source":"def load_img(path):\n img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n img = np.tile(img[...,None], [1, 1, 3]) # gray to rgb\n img = img.astype('float32') # original is uint16\n mx = np.max(img)\n if mx:\n img/=mx # scale image to [0, 1]\n return img\n\ndef load_msk(path):\n msk = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n msk = msk.astype('float32')\n msk/=255.0\n return msk","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:01.319472Z","iopub.execute_input":"2023-12-06T21:38:01.319746Z","iopub.status.idle":"2023-12-06T21:38:01.327699Z","shell.execute_reply.started":"2023-12-06T21:38:01.319722Z","shell.execute_reply":"2023-12-06T21:38:01.326778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BuildDataset(torch.utils.data.Dataset):\n def __init__(self, img_paths, msk_paths=[], transforms=None):\n self.img_paths = img_paths\n self.msk_paths = msk_paths\n self.transforms = transforms\n \n def __len__(self):\n return len(self.img_paths)\n \n def __getitem__(self, index):\n img_path = self.img_paths[index]\n img = load_img(img_path)\n \n if len(self.msk_paths)>0:\n msk_path = self.msk_paths[index]\n msk = load_msk(msk_path)\n if self.transforms:\n data = self.transforms(image=img, mask=msk)\n img = data['image']\n msk = data['mask']\n img = np.transpose(img, (2, 0, 1))\n return torch.tensor(img), torch.tensor(msk)\n else:\n orig_size = img.shape\n if self.transforms:\n data = self.transforms(image=img)\n img = data['image']\n img = np.transpose(img, (2, 0, 1))\n return torch.tensor(img), torch.tensor(np.array([orig_size[0], orig_size[1]]))","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:01.328991Z","iopub.execute_input":"2023-12-06T21:38:01.329407Z","iopub.status.idle":"2023-12-06T21:38:01.341656Z","shell.execute_reply.started":"2023-12-06T21:38:01.329375Z","shell.execute_reply":"2023-12-06T21:38:01.339405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_groups = CFG.train_groups\nvalid_groups = CFG.valid_groups\ngt_df = pd.read_csv(CFG.gt_df)\ngt_df[\"img_path\"] = gt_df[\"img_path\"].apply(lambda x: os.path.join(CFG.data_root, x))\ngt_df[\"msk_path\"] = gt_df[\"msk_path\"].apply(lambda x: os.path.join(CFG.data_root, x))\ntrain_df = gt_df.query(\"group in @train_groups\").reset_index(drop=True)\nvalid_df = gt_df.query(\"group in @valid_groups\").reset_index(drop=True)\ntrain_img_paths = train_df[\"img_path\"].values.tolist()\ntrain_msk_paths = train_df[\"msk_path\"].values.tolist()\nvalid_img_paths = valid_df[\"img_path\"].values.tolist()\nvalid_msk_paths = valid_df[\"msk_path\"].values.tolist()\nif CFG.debug:\n train_img_paths = train_img_paths[:CFG.train_bs*5]\n train_msk_paths = train_msk_paths[:CFG.train_bs*5]\n valid_img_paths = valid_img_paths[:CFG.valid_bs*3]\n valid_msk_paths = valid_msk_paths[:CFG.valid_bs*3]","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:01.343657Z","iopub.execute_input":"2023-12-06T21:38:01.344056Z","iopub.status.idle":"2023-12-06T21:38:02.742152Z","shell.execute_reply.started":"2023-12-06T21:38:01.344031Z","shell.execute_reply":"2023-12-06T21:38:02.741363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = BuildDataset(train_img_paths, train_msk_paths, transforms=CFG.data_transforms['train'])\nvalid_dataset = BuildDataset(valid_img_paths, valid_msk_paths, transforms=CFG.data_transforms['valid'])\ntrain_loader = DataLoader(train_dataset, batch_size=CFG.train_bs, num_workers=0, shuffle=True, pin_memory=True, drop_last=False)\nvalid_loader = DataLoader(valid_dataset, batch_size=CFG.valid_bs, num_workers=0, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:02.743355Z","iopub.execute_input":"2023-12-06T21:38:02.743669Z","iopub.status.idle":"2023-12-06T21:38:02.750014Z","shell.execute_reply.started":"2023-12-06T21:38:02.743643Z","shell.execute_reply":"2023-12-06T21:38:02.74889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Chack Augmentation","metadata":{}},{"cell_type":"code","source":"sample_ids = [random.randint(0, len(train_img_paths)) for _ in range(5)]\nfor sample_id in sample_ids:\n data_name = train_df.loc[sample_id][\"id\"]\n img, msk = train_dataset[sample_id]\n img = img.permute((1, 2, 0)).numpy()*255.0\n img = img.astype('uint8')\n msk = (msk*255).numpy().astype('uint8')\n plt.figure(figsize=(9, 4))\n print(data_name)\n plt.axis('off')\n plt.subplot(1,3,1)\n plt.imshow(img)\n plt.subplot(1,3,2)\n plt.imshow(msk)\n plt.subplot(1,3,3)\n plt.imshow(img, cmap='bone')\n plt.imshow(msk, alpha=0.5)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-06T21:38:02.751082Z","iopub.execute_input":"2023-12-06T21:38:02.751363Z","iopub.status.idle":"2023-12-06T21:38:06.33742Z","shell.execute_reply.started":"2023-12-06T21:38:02.751322Z","shell.execute_reply":"2023-12-06T21:38:06.336509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class attention_block(nn.Module):\n def __init__(self, F_g, F_l, n_coefficients):\n super(attention_block, self).__init__()\n\n self.bypass = nn.Sequential(\n nn.Conv2d(F_g, n_coefficients, kernel_size=1, stride=1, padding=0, bias=True),\n nn.BatchNorm2d(n_coefficients)\n )\n\n self.upsample = nn.Sequential(\n nn.Conv2d(F_l, n_coefficients, kernel_size=1, stride=1, padding=0, bias=True),\n nn.BatchNorm2d(n_coefficients)\n )\n\n self.psi = nn.Sequential(\n nn.Conv2d(n_coefficients, 1, kernel_size=1, stride=1, padding=0, bias=True),\n nn.BatchNorm2d(1),\n nn.Sigmoid()\n )\n\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, gate, skip_connection):\n \"\"\"\n :param gate: gating signal from previous layer\n :param skip_connection: activation from corresponding encoder layer\n :return: output activations\n \"\"\"\n g1 = self.upsample(gate)\n x1 = self.bypass(skip_connection)\n psi = self.relu(g1 + x1)\n psi = self.psi(psi)\n out = skip_connection * psi\n return out\n\nclass MyUNet(nn.Module):\n def contracting_block(self, in_channels, out_channels, kernel_size=3):\n block = torch.nn.Sequential(\n torch.nn.Conv2d(kernel_size=kernel_size, in_channels=in_channels, out_channels=out_channels, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(out_channels),\n torch.nn.Conv2d(kernel_size=kernel_size, in_channels=out_channels, out_channels=out_channels, padding=1),\n torch.nn.ReLU(),\n )\n return block\n def expansive_block(self, in_channels, mid_channel, out_channels, kernel_size=3):\n block = torch.nn.Sequential(\n torch.nn.Conv2d(kernel_size=kernel_size, in_channels=in_channels, out_channels=mid_channel, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(mid_channel),\n torch.nn.Conv2d(kernel_size=kernel_size, in_channels=mid_channel, out_channels=mid_channel, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(mid_channel),\n torch.nn.ConvTranspose2d(in_channels=mid_channel, out_channels=out_channels, kernel_size=3, stride=2, padding=1, output_padding=1)\n )\n return block\n\n def final_block(self, in_channels, mid_channel, out_channels, kernel_size=1):\n block = torch.nn.Sequential(\n \n torch.nn.Conv2d(kernel_size=3, in_channels=in_channels, out_channels=mid_channel, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(mid_channel),\n torch.nn.Conv2d(kernel_size=kernel_size, in_channels=mid_channel, out_channels=out_channels, padding=0),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(out_channels),\n )\n return block\n\n def __init__(self, in_channel, out_channel):\n super(MyUNet, self).__init__()\n #Encode\n self.conv_encode1 = self.contracting_block(in_channels=in_channel, out_channels=64)\n self.conv_maxpool1 = torch.nn.MaxPool2d(kernel_size=2)\n self.conv_encode2 = self.contracting_block(64, 128)\n self.conv_maxpool2 = torch.nn.MaxPool2d(kernel_size=2)\n self.conv_encode3 = self.contracting_block(128, 256)\n self.conv_maxpool3 = torch.nn.MaxPool2d(kernel_size=2)\n # Bottleneck\n self.bottleneck = torch.nn.Sequential(\n torch.nn.Conv2d(kernel_size=3, in_channels=256, out_channels=512, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(512),\n torch.nn.Conv2d(kernel_size=3, in_channels=512, out_channels=512, padding=1),\n torch.nn.ReLU(),\n torch.nn.BatchNorm2d(512),\n torch.nn.ConvTranspose2d(in_channels=512, out_channels=256, kernel_size=3, stride=2, padding=1, output_padding=1)\n )\n\n # Decode\n self.attention3 = attention_block(256, 256, 128)\n self.conv_decode3 = self.expansive_block(512, 256, 128)\n self.attention2 = attention_block(128, 128, 64)\n self.conv_decode2 = self.expansive_block(256, 128, 64)\n self.attention1 = attention_block(64, 64, 32)\n self.final_layer = self.final_block(128, 64, out_channel)\n \n def forward(self, x):\n # Encode\n encode_block1 = self.conv_encode1(x)\n encode_pool1 = self.conv_maxpool1(encode_block1)\n encode_block2 = self.conv_encode2(encode_pool1)\n encode_pool2 = self.conv_maxpool2(encode_block2)\n encode_block3 = self.conv_encode3(encode_pool2)\n encode_pool3 = self.conv_maxpool3(encode_block3)\n # Bottleneck\n bottle_neck1 = self.bottleneck(encode_pool3)\n # Decode\n att_3 = self.attention3(bottle_neck1, encode_block3) \n decode_block1 = torch.cat((bottle_neck1, att_3), 1)\n cat_layer2 = self.conv_decode3(decode_block1)\n \n att_2 = self.attention2(cat_layer2, encode_block2) \n decode_block2 = torch.cat((cat_layer2, att_2), 1)\n cat_layer1 = self.conv_decode2(decode_block2)\n \n \n att_1 = self.attention1(cat_layer1, encode_block1) \n decode_block3 = torch.cat((cat_layer1, att_1), 1)\n final_layer = self.final_layer(decode_block3)\n return final_layer\ndef build_model(backbone, num_classes, device):\n model = MyUNet(in_channel=3, out_channel=num_classes)\n model.to(device)\n return model","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:51.938984Z","iopub.execute_input":"2023-12-07T00:26:51.939372Z","iopub.status.idle":"2023-12-07T00:26:51.966253Z","shell.execute_reply.started":"2023-12-07T00:26:51.939332Z","shell.execute_reply":"2023-12-07T00:26:51.965243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(CFG.backbone, CFG.num_classes, CFG.device)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:51.96801Z","iopub.execute_input":"2023-12-07T00:26:51.9683Z","iopub.status.idle":"2023-12-07T00:26:52.058728Z","shell.execute_reply.started":"2023-12-07T00:26:51.968258Z","shell.execute_reply":"2023-12-07T00:26:52.057976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Function","metadata":{}},{"cell_type":"code","source":"DiceLoss = smp.losses.DiceLoss(mode='binary')\nBCELoss = nn.BCEWithLogitsLoss()\ndef criterion(y_pred, y_true):\n if CFG.loss_func == \"DiceLoss\":\n return DiceLoss(y_pred, y_true)\n elif CFG.loss_func == \"BCELoss\":\n y_true = y_true.unsqueeze(1)\n return BCELoss(y_pred, y_true)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.060018Z","iopub.execute_input":"2023-12-07T00:26:52.060585Z","iopub.status.idle":"2023-12-07T00:26:52.066768Z","shell.execute_reply.started":"2023-12-07T00:26:52.060549Z","shell.execute_reply":"2023-12-07T00:26:52.065823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"def dice_coef(y_true, y_pred, thr=0.5, dim=(2,3), epsilon=0.001):\n y_true = y_true.unsqueeze(1).to(torch.float32)\n y_pred = (y_pred>thr).to(torch.float32)\n inter = (y_true*y_pred).sum(dim=dim)\n den = y_true.sum(dim=dim) + y_pred.sum(dim=dim)\n dice = ((2*inter+epsilon)/(den+epsilon)).mean(dim=(1,0))\n return dice\n\ndef iou_coef(y_true, y_pred, thr=0.5, dim=(2,3), epsilon=0.001):\n y_true = y_true.unsqueeze(1).to(torch.float32)\n y_pred = (y_pred>thr).to(torch.float32)\n inter = (y_true*y_pred).sum(dim=dim)\n union = (y_true + y_pred - y_true*y_pred).sum(dim=dim)\n iou = ((inter+epsilon)/(union+epsilon)).mean(dim=(1,0))\n return iou","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.068803Z","iopub.execute_input":"2023-12-07T00:26:52.069102Z","iopub.status.idle":"2023-12-07T00:26:52.081336Z","shell.execute_reply.started":"2023-12-07T00:26:52.069078Z","shell.execute_reply":"2023-12-07T00:26:52.080544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizer","metadata":{}},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CFG.scheduler == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CFG.T_max, \n eta_min=CFG.min_lr)\n elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CFG.T_0, \n eta_min=CFG.min_lr)\n elif CFG.scheduler == 'ReduceLROnPlateau':\n scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,\n mode='min',\n factor=0.1,\n patience=7,\n threshold=0.0001,\n min_lr=CFG.min_lr,)\n elif CFG.scheduer == 'ExponentialLR':\n scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.85)\n elif CFG.scheduler == None:\n return None\n \n return scheduler\noptimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.wd)\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.082332Z","iopub.execute_input":"2023-12-07T00:26:52.082581Z","iopub.status.idle":"2023-12-07T00:26:52.093348Z","shell.execute_reply.started":"2023-12-07T00:26:52.082558Z","shell.execute_reply":"2023-12-07T00:26:52.092436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check LR","metadata":{}},{"cell_type":"code","source":"_optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.wd)\n_scheduler = fetch_scheduler(_optimizer)\nlr_list = []\nfor e in range(CFG.epochs):\n for step in range(len(train_loader)):\n lr_list.append(_optimizer.param_groups[0]['lr'])\n if (step + 1) % CFG.n_accumulate == 0:\n _optimizer.step()\n _scheduler.step()\nplt.plot(np.array(range(len(lr_list))), np.array(lr_list))\nplt.show()\ndel _optimizer, _scheduler","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.094734Z","iopub.execute_input":"2023-12-07T00:26:52.095349Z","iopub.status.idle":"2023-12-07T00:26:52.380236Z","shell.execute_reply.started":"2023-12-07T00:26:52.095313Z","shell.execute_reply":"2023-12-07T00:26:52.379208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n model.train()\n scaler = amp.GradScaler()\n \n dataset_size = 0\n running_loss = 0.0\n \n pbar = tqdm(enumerate(dataloader), total=len(dataloader), desc='Train ')\n for step, (images, masks) in pbar: \n images = images.to(device, dtype=torch.float)\n masks = masks.to(device, dtype=torch.float)\n \n batch_size = images.size(0)\n \n with amp.autocast(enabled=True):\n y_pred = model(images)\n loss = criterion(y_pred, masks)\n loss = loss / CFG.n_accumulate\n \n scaler.scale(loss).backward()\n \n if (step + 1) % CFG.n_accumulate == 0:\n scaler.step(optimizer)\n scaler.update()\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n if scheduler is not None:\n scheduler.step()\n \n running_loss += (loss.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n \n mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0\n current_lr = optimizer.param_groups[0]['lr']\n pbar.set_postfix( epoch=f'{epoch}',\n train_loss=f'{epoch_loss:0.4f}',\n lr=f'{current_lr:0.5f}',\n gpu_mem=f'{mem:0.2f} GB')\n torch.cuda.empty_cache()\n gc.collect()\n return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.381816Z","iopub.execute_input":"2023-12-07T00:26:52.382131Z","iopub.status.idle":"2023-12-07T00:26:52.392881Z","shell.execute_reply.started":"2023-12-07T00:26:52.382104Z","shell.execute_reply":"2023-12-07T00:26:52.391249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef valid_one_epoch(model, dataloader, device, epoch):\n model.eval()\n \n dataset_size = 0\n running_loss = 0.0\n \n val_scores = []\n \n pbar = tqdm(enumerate(dataloader), total=len(dataloader), desc='Valid ')\n for step, (images, masks) in pbar: \n images = images.to(device, dtype=torch.float)\n masks = masks.to(device, dtype=torch.float)\n \n batch_size = images.size(0)\n \n y_pred = model(images)\n loss = criterion(y_pred, masks)\n \n running_loss += (loss.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n \n y_pred = nn.Sigmoid()(y_pred)\n val_dice = dice_coef(masks, y_pred).cpu().detach().numpy()\n val_jaccard = iou_coef(masks, y_pred).cpu().detach().numpy()\n val_scores.append([val_dice, val_jaccard])\n \n mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0\n current_lr = optimizer.param_groups[0]['lr']\n pbar.set_postfix(valid_loss=f'{epoch_loss:0.4f}',\n lr=f'{current_lr:0.5f}',\n gpu_memory=f'{mem:0.2f} GB')\n val_scores = np.mean(val_scores, axis=0)\n torch.cuda.empty_cache()\n gc.collect()\n return epoch_loss, val_scores","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.394799Z","iopub.execute_input":"2023-12-07T00:26:52.395134Z","iopub.status.idle":"2023-12-07T00:26:52.406909Z","shell.execute_reply.started":"2023-12-07T00:26:52.395107Z","shell.execute_reply":"2023-12-07T00:26:52.406065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs): \n if torch.cuda.is_available():\n print(\"cuda: {}\\n\".format(torch.cuda.get_device_name()))\n \n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_loss = np.inf\n best_epoch = -1\n history = defaultdict(list)\n \n for epoch in range(1, num_epochs + 1): \n gc.collect()\n print(f'Epoch {epoch}/{num_epochs}', end='')\n train_loss = train_one_epoch(model, optimizer, scheduler, \n dataloader=train_loader, \n device=CFG.device, epoch=epoch)\n \n val_loss, val_scores = valid_one_epoch(model, valid_loader, \n device=CFG.device, \n epoch=epoch)\n val_dice, val_jaccard = val_scores\n history['Train Loss'].append(train_loss)\n history['Valid Loss'].append(val_loss)\n history['Valid Dice'].append(val_dice)\n history['Valid Jaccard'].append(val_jaccard) \n print(f'Valid Dice: {val_dice:0.4f} | Valid Jaccard: {val_jaccard:0.4f}')\n print(f'Valid Loss: {val_loss}')\n \n # deep copy the model\n if val_loss <= best_loss:\n print(f\"{c_}Valid loss Improved ({best_loss} ---> {val_loss})\")\n best_dice = val_dice\n best_jaccard = val_jaccard\n best_loss = val_loss\n best_epoch = epoch\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"best_epoch.bin\"\n torch.save(model.state_dict(), PATH)\n print(f\"Model Saved{sr_}\")\n \n last_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"last_epoch.bin\"\n torch.save(model.state_dict(), PATH)\n \n print(); print()\n \n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best Loss: {:.4f}\".format(best_loss))\n \n # load best model weights\n model.load_state_dict(best_model_wts)\n return model, history","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.409795Z","iopub.execute_input":"2023-12-07T00:26:52.410152Z","iopub.status.idle":"2023-12-07T00:26:52.424133Z","shell.execute_reply.started":"2023-12-07T00:26:52.410123Z","shell.execute_reply":"2023-12-07T00:26:52.423347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n device=CFG.device,\n num_epochs=CFG.epochs)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:26:52.425275Z","iopub.execute_input":"2023-12-07T00:26:52.425548Z","iopub.status.idle":"2023-12-07T00:27:03.626242Z","shell.execute_reply.started":"2023-12-07T00:26:52.425523Z","shell.execute_reply":"2023-12-07T00:27:03.624828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"\"\"\"Surface Dice metric for HuBMAP 4.\"\"\"\n\nimport numpy as np\nimport pandas as pd\nimport pandas.api.types\nfrom numba import jit\nfrom scipy.ndimage import label, generate_binary_structure\nfrom skimage.transform import resize\nfrom typing import Optional, Tuple, Union\n\n\nclass ParticipantVisibleError(Exception):\n pass\n\n\ndef score(\n solution: pd.DataFrame,\n submission: pd.DataFrame,\n row_id_column_name: str,\n rle_column_name: str,\n tolerance: float = 1.0,\n image_id_column_name: Optional[str] = None,\n slice_id_column_name: Optional[str] = None,\n resize_fraction: float = 1.0,\n) -> float:\n \"\"\"Mean Surface Dice over collections of 2D or 3D data.\n\n This metric is adapted from the Google DeepMind surface dice metric as found here:\n https://github.com/google-deepmind/surface-distance/tree/master.\n\n Can be used with either 2D or 3D data. When used with 3D data, each row in the solution and\n submission files should contain run-length encoded masks of a width x height slice.\n\n Parameters\n ----------\n solution : Pandas dataframe, the ground truth values.\n\n submission : Pandas dataframe, the predicted values.\n\n row_id_column_name : str, the name of the ID column used by Kaggle's preprocessing code to\n align the solution and submission dataframes.\n\n rle_column_name : str, the name of column containing run-length encoded masks.\n\n tolerance : float, the distance, in millimeters, the predicted mask surfaces are allowed to\n vary from the ground-truth masks.\n\n image_id_column_name : str (optional), for 3D data, the name of the column identifying\n the image a slice belongs to.\n\n slice_id_column_name : str (optional), for 3D data, the name of the column enumerating\n the slices within each image.\n\n resize_fraction : float, the fraction by which to resize the decoded masks. Useful for memory\n efficiency in the case of large images.\n\n Returns\n -------\n mean_surface_dice : float\n\n Examples\n --------\n No groups (2D images).\n >>> solution = pd.DataFrame({\n ... 'id': [0, 1],\n ... 'rle': ['1 12 20 2', '1 6'],\n ... 'width': [5, 5],\n ... 'height': [5, 5],\n ... })\n\n Perfect submission.\n >>> submission = pd.DataFrame({\n ... 'id': [0, 1],\n ... 'rle': ['1 12 20 2', '1 6'],\n ... })\n >>> score(solution, submission, 'id', 'rle', 0.0)\n 1.0\n\n One group with two slices.\n >>> solution = pd.DataFrame({\n ... 'id': [0, 1],\n ... 'rle': ['1 12 20 2', '1 6'],\n ... 'width': [5, 5],\n ... 'height': [5, 5],\n ... 'group': ['a', 'a'],\n ... 'slice': [0, 1],\n ... })\n\n Perfect submission.\n >>> submission = pd.DataFrame({\n ... 'id': [0, 1],\n ... 'rle': ['1 12 20 2', '1 6'],\n ... })\n >>> score(solution, submission, 'id', 'rle', 0.0, 'group', 'slice')\n 1.0\n\n Null submission.\n >>> submission = pd.DataFrame({\n ... 'id': [0, 1],\n ... 'rle': ['', ''],\n ... })\n >>> score(solution, submission, 'id', 'rle', 0.0, 'group', 'slice')\n 0.0\n\n Two groups with multiple slices.\n >>> solution = pd.DataFrame({\n ... 'id': [0, 1, 2, 3, 4],\n ... 'rle': ['1 12', '1 12 ', '1 12', '20 5', '22 2'],\n ... 'width': [5, 5, 5, 8, 8],\n ... 'height': [5, 5, 5, 4, 4],\n ... 'group': ['a', 'a', 'a', 'b', 'b'],\n ... 'slice': [0, 1, 2, 0, 1],\n ... })\n >>> submission = pd.DataFrame({\n ... 'id': [0, 1, 2, 3, 4],\n ... 'rle': ['1 12', '1 12 ', '1 12', '20 5', '22 2'],\n ... })\n >>> score(solution, submission, 'id', 'rle', 0.0, 'group', 'slice')\n 1.0\n\n >>> submission = pd.DataFrame({\n ... 'id': [0, 1, 2, 3, 4],\n ... 'rle': ['1 11', '1 12 ', '1 13', '20 4', '22 3'],\n ... })\n\n With non-zero tolerance.\n >>> score(solution, submission, 'id', 'rle', 5.0, 'group', 'slice')\n 1.0\n\n Cubes, adapted from https://github.com/google-deepmind/surface-distance/blob/master/surface_distance_test.py#L307\n >>> solution = pd.DataFrame({\n ... 'id': np.arange(100),\n ... 'rle': ['1 10000' if k <= 49 else '' for k in np.arange(100)],\n ... 'width': [100] * 100,\n ... 'height': [100] * 100,\n ... 'group': ['a'] * 100,\n ... 'slice': np.arange(100),\n ... })\n >>> submission = pd.DataFrame({\n ... 'id': np.arange(100),\n ... 'rle': ['1 10000' if k <= 50 else '' for k in np.arange(100)],\n ... })\n >>> score(solution, submission, 'id', 'rle', 0.0, 'group', 'slice')\n 0.750877...\n\n \"\"\"\n solution = solution.set_index(row_id_column_name)\n submission = submission.set_index(row_id_column_name)\n\n # Check that both are defined or neither are\n if (image_id_column_name is None) != (slice_id_column_name is None):\n raise ValueError(\"If one of `image_id_column_name` or `slice_id_column_name` is given, the other must be given also.\")\n\n if tolerance < 0.0:\n raise ValueError(\"`tolerance` must be non-negative.\")\n\n # Setup spacing_mm\n if image_id_column_name is None:\n spacing_mm = (1, 1) # (height, width)\n else:\n spacing_mm = (1, 1, 1) # (height, width, depth)\n\n # Create joined dataframe to iterate over\n joined = solution.join(submission,\n lsuffix='_sol',\n rsuffix='_sub')\n\n # Compute surface dice for each group of slices and total\n total_dice = 0.0\n group_col = row_id_column_name if image_id_column_name is None else image_id_column_name\n for group, df in joined.groupby(group_col):\n # Make indexing easier\n df = df.reset_index(drop=True)\n\n # Check that we're stacking slices in order\n if image_id_column_name is not None:\n assert df.loc[:, slice_id_column_name].is_monotonic_increasing\n\n group_mask_sol, group_mask_sub = [], []\n height, width = df.loc[0, 'height'], df.loc[0, 'width']\n assert (df.loc[:, 'height'].nunique() == 1) and (df.loc[:, 'width'].nunique() == 1),\\\n \"Height and width must be constant within each group.\"\n\n # Decode slice RLEs to arrays\n for row in df.itertuples():\n # Mask transformation\n shape = (height, width)\n group_mask_sol.append(make_mask(row.rle_sol, shape, resize_fraction))\n group_mask_sub.append(make_mask(row.rle_sub, shape, resize_fraction))\n\n # Stack slices to create 3D arrays\n group_mask_sol = np.stack(group_mask_sol, axis=-1)\n assert group_mask_sol.ndim == 3\n group_mask_sub = np.stack(group_mask_sub, axis=-1)\n assert group_mask_sub.ndim == 3\n\n # Minimize dims: 2D or 3D\n group_mask_sol = group_mask_sol.squeeze()\n group_mask_sub = group_mask_sub.squeeze()\n\n # Compute surface distance\n surface_dist = compute_surface_distances(\n mask_gt=group_mask_sol,\n mask_pred=group_mask_sub,\n spacing_mm=spacing_mm,\n )\n # Compute surface dice and add to running total\n total_dice += compute_surface_dice_at_tolerance(\n surface_dist,\n tolerance_mm=tolerance,\n )\n\n # Compute mean from running total\n if image_id_column_name is None:\n ngroups = len(joined)\n else:\n ngroups = joined.loc[:, image_id_column_name].nunique()\n mean_surface_dice = total_dice / ngroups\n return mean_surface_dice\n\n\ndef make_mask(rle, shape, resize_fraction):\n if resize_fraction < 0.0 or resize_fraction > 1.0:\n raise ValueError(\"`resize_fraction` must be between 0.0 and 1.0, inclusive.\")\n\n mask = rle_decode(rle, shape)\n if resize_fraction < 1.0:\n new_shape = int(shape[0] * resize_fraction), int(shape[1] * resize_fraction)\n mask = voting_resize(mask, new_shape)\n mask = mask.astype(bool)\n return mask\n\n\ndef voting_resize(mask, new_shape):\n interpolated_mask = resize(mask, new_shape, order=1, preserve_range=True) # Using bilinear interpolation\n voting_mask = (interpolated_mask > 0.5).astype(np.uint8)\n return voting_mask\n\n\n# Below is code from https://www.kaggle.com/code/paulorzp/run-length-encode-and-decode\ndef rle_encode(img):\n '''\n img: numpy array, 1 - mask, 0 - background\n Returns run length as string formated\n '''\n pixels = img.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\n\ndef rle_decode(mask_rle, shape):\n '''\n mask_rle: run-length as string formated (start length)\n shape: (height,width) of array to return\n Returns numpy array, 1 - mask, 0 - background\n\n '''\n s = mask_rle.split()\n starts, lengths = [\n np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])\n ]\n starts -= 1\n ends = starts + lengths\n img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape)\n\n\n# Below is adapted from https://github.com/google-deepmind/surface-distance\n\n# Copyright 2018 Google Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS-IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# surface_distance/lookup_tables.py\n\"\"\"Lookup tables used by surface distance metrics.\"\"\"\n\nimport math\nimport numpy as np\n\nENCODE_NEIGHBOURHOOD_3D_KERNEL = np.array([[[128, 64], [32, 16]],\n [[8, 4], [2, 1]]])\n\n# _NEIGHBOUR_CODE_TO_NORMALS is a lookup table.\n# For every binary neighbour code\n# (2x2x2 neighbourhood = 8 neighbours = 8 bits = 256 codes)\n# it contains the surface normals of the triangles (called \"surfel\" for\n# \"surface element\" in the following). The length of the normal\n# vector encodes the surfel area.\n#\n# created using the marching_cube algorithm\n# see e.g. https://en.wikipedia.org/wiki/Marching_cubes\n# pylint: disable=line-too-long\n_NEIGHBOUR_CODE_TO_NORMALS = [[[0, 0, 0]], [[0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [0.25, 0.25, -0.0]],\n [[0.125, -0.125, 0.125]],\n [[-0.25, -0.0, -0.25], [0.25, 0.0, 0.25]],\n [[0.125, -0.125, 0.125], [-0.125, -0.125,\n 0.125]],\n [[0.5, 0.0, -0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125]], [[-0.125, 0.125,\n 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, 0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [-0.25, 0.0, 0.25]],\n [[0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.25, -0.25, 0.0], [0.25, -0.25, 0.0]],\n [[0.5, 0.0, 0.0], [0.25, -0.25, 0.25],\n [-0.125, 0.125, -0.125]],\n [[-0.5, 0.0, 0.0], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]],\n [[0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25]],\n [[-0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[0.0, -0.5, 0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, -0.125, 0.125], [0.125, -0.125,\n -0.125]],\n [[0.0, 0.0, -0.5], [0.25, 0.25, 0.25],\n [-0.125, -0.125, -0.125]],\n [[-0.125, -0.125, 0.125], [0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[-0.125, -0.125, -0.125], [-0.25, -0.25, -0.25],\n [0.25, 0.25, 0.25], [0.125, 0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125,\n -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [-0.25, 0.0, 0.25],\n [0.125, -0.125, -0.125]],\n [[0.125, 0.125, 0.125], [0.375, 0.375, 0.375],\n [0.0, -0.25, 0.25], [-0.25, 0.0, 0.25]],\n [[0.125, -0.125, -0.125], [0.25, -0.25, 0.0],\n [0.25, -0.25, 0.0]],\n [[0.375, 0.375, 0.375], [0.0, 0.25, -0.25],\n [-0.125, -0.125, -0.125], [-0.25, 0.25, 0.0]],\n [[-0.5, 0.0, 0.0], [-0.125, -0.125, -0.125],\n [-0.25, -0.25, -0.25], [0.125, 0.125, 0.125]],\n [[-0.5, 0.0, 0.0], [-0.125, -0.125, -0.125],\n [-0.25, -0.25, -0.25]], [[0.125, -0.125,\n 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, 0.25, -0.25]],\n [[0.0, -0.5, 0.0], [0.125, 0.125, -0.125],\n [0.25, 0.25, -0.25]],\n [[0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.125, -0.125, 0.125], [-0.25, -0.0, -0.25],\n [0.25, 0.0, 0.25]],\n [[0.0, -0.25, 0.25], [0.0, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[-0.375, -0.375, 0.375], [-0.0, 0.25, 0.25],\n [0.125, 0.125, -0.125], [-0.25, -0.0, -0.25]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, 0.125, 0.125]],\n [[-0.0, 0.0, 0.5], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.25, 0.25, -0.25], [0.25, 0.25, -0.25],\n [0.125, 0.125, -0.125], [-0.125, -0.125,\n 0.125]],\n [[0.125, -0.125, 0.125], [0.25, -0.25, 0.0],\n [0.25, -0.25, 0.0]],\n [[0.5, 0.0, 0.0], [0.25, -0.25, 0.25],\n [-0.125, 0.125, -0.125], [0.125, -0.125,\n 0.125]],\n [[0.0, 0.25, -0.25], [0.375, -0.375, -0.375],\n [-0.125, 0.125, 0.125], [0.25, 0.25, 0.0]],\n [[-0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.25, -0.25, 0.0], [-0.25, 0.25, 0.0]],\n [[0.0, 0.5, 0.0], [-0.25, 0.25, 0.25],\n [0.125, -0.125, -0.125]],\n [[0.0, 0.5, 0.0], [0.125, -0.125, 0.125],\n [-0.25, 0.25, -0.25]],\n [[0.0, 0.5, 0.0], [0.0, -0.5, 0.0]],\n [[0.25, -0.25, 0.0], [-0.25, 0.25, 0.0],\n [0.125, -0.125, 0.125]],\n [[-0.375, -0.375, -0.375], [-0.25, 0.0, 0.25],\n [-0.125, -0.125, -0.125], [-0.25, 0.25, 0.0]],\n [[0.125, 0.125, 0.125], [0.0, -0.5, 0.0],\n [-0.25, -0.25, -0.25], [-0.125, -0.125,\n -0.125]],\n [[0.0, -0.5, 0.0], [-0.25, -0.25, -0.25],\n [-0.125, -0.125, -0.125]],\n [[-0.125, 0.125, 0.125], [0.25, -0.25, 0.0],\n [-0.25, 0.25, 0.0]],\n [[0.0, 0.5, 0.0], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.375, 0.375, -0.375], [-0.25, -0.25, 0.0],\n [-0.125, 0.125, -0.125], [-0.25, 0.0, 0.25]],\n [[0.0, 0.5, 0.0], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125]],\n [[0.25, -0.25, 0.0], [-0.25, 0.25, 0.0],\n [0.25, -0.25, 0.0], [0.25, -0.25, 0.0]],\n [[-0.25, -0.25, 0.0], [-0.25, -0.25, 0.0],\n [-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.25, -0.25, 0.0],\n [-0.25, -0.25, 0.0]],\n [[-0.25, -0.25, 0.0], [-0.25, -0.25, 0.0]],\n [[-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.25, -0.25, 0.0],\n [0.25, 0.25, -0.0]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25]],\n [[0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.375, -0.375, 0.375], [0.0, -0.25, -0.25],\n [-0.125, 0.125, -0.125], [0.25, 0.25, 0.0]],\n [[-0.125, -0.125, 0.125], [-0.125, 0.125,\n 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [-0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.25, 0.0, 0.25],\n [-0.25, 0.0, 0.25]],\n [[0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.0, 0.5, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[-0.25, 0.25, -0.25], [-0.25, 0.25, -0.25],\n [-0.125, 0.125, -0.125],\n [-0.125, 0.125, -0.125]],\n [[-0.25, 0.0, -0.25], [0.375, -0.375, -0.375],\n [0.0, 0.25, -0.25], [-0.125, 0.125, 0.125]],\n [[0.5, 0.0, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [0.25, 0.0, -0.25]],\n [[-0.0, 0.0, 0.5], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.25, 0.0, 0.25],\n [0.25, 0.0, -0.25]],\n [[-0.25, -0.0, -0.25], [-0.375, 0.375, 0.375],\n [-0.25, -0.25, 0.0], [-0.125, 0.125, 0.125]],\n [[0.0, 0.0, -0.5], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125]],\n [[-0.0, 0.0, 0.5], [0.0, 0.0, 0.5]],\n [[0.125, 0.125, 0.125], [0.125, 0.125, 0.125],\n [0.25, 0.25, 0.25], [0.0, 0.0, 0.5]],\n [[0.125, 0.125, 0.125], [0.25, 0.25, 0.25],\n [0.0, 0.0, 0.5]],\n [[-0.25, 0.0, 0.25], [0.25, 0.0, -0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [-0.25, 0.0, 0.25],\n [-0.25, 0.0, 0.25], [0.25, 0.0, -0.25]],\n [[0.125, -0.125, 0.125], [0.25, 0.0, 0.25],\n [0.25, 0.0, 0.25]],\n [[0.25, 0.0, 0.25], [-0.375, -0.375, 0.375],\n [-0.25, 0.25, 0.0], [-0.125, -0.125, 0.125]],\n [[-0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.25, 0.0, 0.25],\n [0.25, 0.0, 0.25]],\n [[0.25, 0.0, 0.25], [0.25, 0.0, 0.25]],\n [[-0.125, -0.125, 0.125], [0.125, -0.125,\n 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [0.0, -0.25, 0.25],\n [0.0, 0.25, -0.25]],\n [[0.0, -0.5, 0.0], [0.125, 0.125, -0.125],\n [0.25, 0.25, -0.25], [-0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25],\n [0.0, -0.25, 0.25], [0.0, 0.25, -0.25]],\n [[0.0, 0.25, 0.25], [0.0, 0.25, 0.25],\n [0.125, -0.125, -0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125], [0.125, 0.125, 0.125]],\n [[-0.0, 0.0, 0.5], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[-0.0, 0.5, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, -0.125]],\n [[0.5, 0.0, -0.0], [0.25, -0.25, -0.25],\n [0.125, -0.125, -0.125]],\n [[-0.25, 0.25, 0.25], [-0.125, 0.125, 0.125],\n [-0.25, 0.25, 0.25], [0.125, -0.125, -0.125]],\n [[0.375, -0.375, 0.375], [0.0, 0.25, 0.25],\n [-0.125, 0.125, -0.125], [-0.25, 0.0, 0.25]],\n [[0.0, -0.5, 0.0], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.375, -0.375, 0.375], [0.25, -0.25, 0.0],\n [0.0, 0.25, 0.25], [-0.125, -0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [-0.25, 0.25, 0.25],\n [0.0, 0.0, 0.5]],\n [[0.125, 0.125, 0.125], [0.0, 0.25, 0.25],\n [0.0, 0.25, 0.25]],\n [[0.0, 0.25, 0.25], [0.0, 0.25, 0.25]],\n [[0.5, 0.0, -0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125], [0.125, 0.125, 0.125]],\n [[0.125, -0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, 0.125, 0.125]],\n [[-0.25, -0.0, -0.25], [0.25, 0.0, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [0.25, 0.25, -0.0],\n [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125]], [[0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [0.25, 0.25, -0.0],\n [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[-0.25, -0.0, -0.25], [0.25, 0.0, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, -0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, 0.125, 0.125]],\n [[0.5, 0.0, -0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125], [0.125, 0.125, 0.125]],\n [[0.0, 0.25, 0.25], [0.0, 0.25, 0.25]],\n [[0.125, 0.125, 0.125], [0.0, 0.25, 0.25],\n [0.0, 0.25, 0.25]],\n [[-0.125, 0.125, 0.125], [-0.25, 0.25, 0.25],\n [0.0, 0.0, 0.5]],\n [[-0.375, -0.375, 0.375], [0.25, -0.25, 0.0],\n [0.0, 0.25, 0.25], [-0.125, -0.125, 0.125]],\n [[0.0, -0.5, 0.0], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[0.375, -0.375, 0.375], [0.0, 0.25, 0.25],\n [-0.125, 0.125, -0.125], [-0.25, 0.0, 0.25]],\n [[-0.25, 0.25, 0.25], [-0.125, 0.125, 0.125],\n [-0.25, 0.25, 0.25], [0.125, -0.125, -0.125]],\n [[0.5, 0.0, -0.0], [0.25, -0.25, -0.25],\n [0.125, -0.125, -0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[-0.0, 0.5, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[-0.0, 0.0, 0.5], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125], [0.125, 0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[0.0, 0.25, 0.25], [0.0, 0.25, 0.25],\n [0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25],\n [0.0, 0.25, 0.25], [0.0, 0.25, 0.25]],\n [[0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.0, -0.5, 0.0], [0.125, 0.125, -0.125],\n [0.25, 0.25, -0.25], [-0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [0.0, -0.25, 0.25],\n [0.0, 0.25, -0.25]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [0.125, -0.125,\n 0.125]],\n [[0.25, 0.0, 0.25], [0.25, 0.0, 0.25]],\n [[0.125, 0.125, 0.125], [0.25, 0.0, 0.25],\n [0.25, 0.0, 0.25]],\n [[-0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.25, 0.0, 0.25], [-0.375, -0.375, 0.375],\n [-0.25, 0.25, 0.0], [-0.125, -0.125, 0.125]],\n [[0.125, -0.125, 0.125], [0.25, 0.0, 0.25],\n [0.25, 0.0, 0.25]],\n [[-0.25, -0.0, -0.25], [0.25, 0.0, 0.25],\n [0.25, 0.0, 0.25], [0.25, 0.0, 0.25]],\n [[-0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [0.25, 0.0, -0.25],\n [-0.125, 0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.25, 0.25, 0.25],\n [0.0, 0.0, 0.5]],\n [[0.125, 0.125, 0.125], [0.125, 0.125, 0.125],\n [0.25, 0.25, 0.25], [0.0, 0.0, 0.5]],\n [[-0.0, 0.0, 0.5], [0.0, 0.0, 0.5]],\n [[0.0, 0.0, -0.5], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125]],\n [[-0.25, -0.0, -0.25], [-0.375, 0.375, 0.375],\n [-0.25, -0.25, 0.0], [-0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.25, 0.0, 0.25],\n [0.25, 0.0, -0.25]],\n [[-0.0, 0.0, 0.5], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [0.25, 0.0, -0.25]],\n [[0.5, 0.0, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[-0.25, 0.0, -0.25], [0.375, -0.375, -0.375],\n [0.0, 0.25, -0.25], [-0.125, 0.125, 0.125]],\n [[-0.25, 0.25, -0.25], [-0.25, 0.25, -0.25],\n [-0.125, 0.125, -0.125],\n [-0.125, 0.125, -0.125]],\n [[-0.0, 0.5, 0.0], [-0.25, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.25, 0.0, 0.25],\n [-0.25, 0.0, 0.25]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125],\n [-0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [-0.125, 0.125,\n 0.125]],\n [[0.375, -0.375, 0.375], [0.0, -0.25, -0.25],\n [-0.125, 0.125, -0.125], [0.25, 0.25, 0.0]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.0, 0.0, 0.5], [0.25, -0.25, 0.25],\n [0.125, -0.125, 0.125]],\n [[0.0, -0.25, 0.25], [0.0, -0.25, 0.25]],\n [[-0.125, -0.125, 0.125], [-0.25, -0.25, 0.0],\n [0.25, 0.25, -0.0]],\n [[-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [-0.125, -0.125, 0.125]],\n [[-0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [-0.25, -0.25, 0.0]],\n [[0.125, 0.125, 0.125], [-0.25, -0.25, 0.0],\n [-0.25, -0.25, 0.0]],\n [[-0.25, -0.25, 0.0], [-0.25, -0.25, 0.0],\n [-0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [-0.25, -0.25, 0.0],\n [-0.25, -0.25, 0.0], [0.25, 0.25, -0.0]],\n [[0.0, 0.5, 0.0], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125]],\n [[-0.375, 0.375, -0.375], [-0.25, -0.25, 0.0],\n [-0.125, 0.125, -0.125], [-0.25, 0.0, 0.25]],\n [[0.0, 0.5, 0.0], [0.25, 0.25, -0.25],\n [-0.125, -0.125, 0.125],\n [-0.125, -0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.25, -0.25, 0.0],\n [-0.25, 0.25, 0.0]],\n [[0.0, -0.5, 0.0], [-0.25, -0.25, -0.25],\n [-0.125, -0.125, -0.125]],\n [[0.125, 0.125, 0.125], [0.0, -0.5, 0.0],\n [-0.25, -0.25, -0.25], [-0.125, -0.125,\n -0.125]],\n [[-0.375, -0.375, -0.375], [-0.25, 0.0, 0.25],\n [-0.125, -0.125, -0.125], [-0.25, 0.25, 0.0]],\n [[0.25, -0.25, 0.0], [-0.25, 0.25, 0.0],\n [0.125, -0.125, 0.125]],\n [[0.0, 0.5, 0.0], [0.0, -0.5, 0.0]],\n [[0.0, 0.5, 0.0], [0.125, -0.125, 0.125],\n [-0.25, 0.25, -0.25]],\n [[0.0, 0.5, 0.0], [-0.25, 0.25, 0.25],\n [0.125, -0.125, -0.125]],\n [[0.25, -0.25, 0.0], [-0.25, 0.25, 0.0]],\n [[-0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.0, 0.25, -0.25], [0.375, -0.375, -0.375],\n [-0.125, 0.125, 0.125], [0.25, 0.25, 0.0]],\n [[0.5, 0.0, 0.0], [0.25, -0.25, 0.25],\n [-0.125, 0.125, -0.125], [0.125, -0.125,\n 0.125]],\n [[0.125, -0.125, 0.125], [0.25, -0.25, 0.0],\n [0.25, -0.25, 0.0]],\n [[0.25, 0.25, -0.25], [0.25, 0.25, -0.25],\n [0.125, 0.125, -0.125], [-0.125, -0.125,\n 0.125]],\n [[-0.0, 0.0, 0.5], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125],\n [-0.125, 0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[-0.375, -0.375, 0.375], [-0.0, 0.25, 0.25],\n [0.125, 0.125, -0.125], [-0.25, -0.0, -0.25]],\n [[0.0, -0.25, 0.25], [0.0, 0.25, -0.25],\n [0.125, -0.125, 0.125]],\n [[0.125, -0.125, 0.125], [-0.25, -0.0, -0.25],\n [0.25, 0.0, 0.25]],\n [[0.125, -0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.0, -0.5, 0.0], [0.125, 0.125, -0.125],\n [0.25, 0.25, -0.25]],\n [[0.0, -0.25, 0.25], [0.0, 0.25, -0.25]],\n [[0.125, 0.125, 0.125], [0.125, -0.125, 0.125]],\n [[0.125, -0.125, 0.125]],\n [[-0.5, 0.0, 0.0], [-0.125, -0.125, -0.125],\n [-0.25, -0.25, -0.25]],\n [[-0.5, 0.0, 0.0], [-0.125, -0.125, -0.125],\n [-0.25, -0.25, -0.25], [0.125, 0.125, 0.125]],\n [[0.375, 0.375, 0.375], [0.0, 0.25, -0.25],\n [-0.125, -0.125, -0.125], [-0.25, 0.25, 0.0]],\n [[0.125, -0.125, -0.125], [0.25, -0.25, 0.0],\n [0.25, -0.25, 0.0]],\n [[0.125, 0.125, 0.125], [0.375, 0.375, 0.375],\n [0.0, -0.25, 0.25], [-0.25, 0.0, 0.25]],\n [[-0.25, 0.0, 0.25], [-0.25, 0.0, 0.25],\n [0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[-0.125, 0.125, 0.125], [0.125, -0.125,\n -0.125]],\n [[-0.125, -0.125, -0.125], [-0.25, -0.25, -0.25],\n [0.25, 0.25, 0.25], [0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125], [0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[0.0, 0.0, -0.5], [0.25, 0.25, 0.25],\n [-0.125, -0.125, -0.125]],\n [[0.125, -0.125, 0.125], [0.125, -0.125,\n -0.125]],\n [[0.0, -0.5, 0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125]],\n [[-0.125, -0.125, 0.125],\n [0.125, -0.125, -0.125]],\n [[0.0, -0.25, -0.25], [0.0, 0.25, 0.25]],\n [[0.125, -0.125, -0.125]],\n [[0.5, 0.0, 0.0], [0.5, 0.0, 0.0]],\n [[-0.5, 0.0, 0.0], [-0.25, 0.25, 0.25],\n [-0.125, 0.125, 0.125]],\n [[0.5, 0.0, 0.0], [0.25, -0.25, 0.25],\n [-0.125, 0.125, -0.125]],\n [[0.25, -0.25, 0.0], [0.25, -0.25, 0.0]],\n [[0.5, 0.0, 0.0], [-0.25, -0.25, 0.25],\n [-0.125, -0.125, 0.125]],\n [[-0.25, 0.0, 0.25], [-0.25, 0.0, 0.25]],\n [[0.125, 0.125, 0.125], [-0.125, 0.125, 0.125]],\n [[-0.125, 0.125, 0.125]],\n [[0.5, 0.0, -0.0], [0.25, 0.25, 0.25],\n [0.125, 0.125, 0.125]],\n [[0.125, -0.125, 0.125], [-0.125, -0.125,\n 0.125]],\n [[-0.25, -0.0, -0.25], [0.25, 0.0, 0.25]],\n [[0.125, -0.125, 0.125]],\n [[-0.25, -0.25, 0.0], [0.25, 0.25, -0.0]],\n [[-0.125, -0.125, 0.125]],\n [[0.125, 0.125, 0.125]], [[0, 0, 0]]]\n# pylint: enable=line-too-long\n\n\ndef create_table_neighbour_code_to_surface_area(spacing_mm):\n \"\"\"Returns an array mapping neighbourhood code to the surface elements area.\n\n Note that the normals encode the initial surface area. This function computes\n the area corresponding to the given `spacing_mm`.\n\n Args:\n spacing_mm: 3-element list-like structure. Voxel spacing in x0, x1 and x2\n direction.\n \"\"\"\n # compute the area for all 256 possible surface elements\n # (given a 2x2x2 neighbourhood) according to the spacing_mm\n neighbour_code_to_surface_area = np.zeros([256])\n for code in range(256):\n normals = np.array(_NEIGHBOUR_CODE_TO_NORMALS[code])\n sum_area = 0\n for normal_idx in range(normals.shape[0]):\n # normal vector\n n = np.zeros([3])\n n[0] = normals[normal_idx, 0] * spacing_mm[1] * spacing_mm[2]\n n[1] = normals[normal_idx, 1] * spacing_mm[0] * spacing_mm[2]\n n[2] = normals[normal_idx, 2] * spacing_mm[0] * spacing_mm[1]\n area = np.linalg.norm(n)\n sum_area += area\n neighbour_code_to_surface_area[code] = sum_area\n\n return neighbour_code_to_surface_area\n\n\n# In the neighbourhood, points are ordered: top left, top right, bottom left,\n# bottom right.\nENCODE_NEIGHBOURHOOD_2D_KERNEL = np.array([[8, 4], [2, 1]])\n\n\ndef create_table_neighbour_code_to_contour_length(spacing_mm):\n \"\"\"Returns an array mapping neighbourhood code to the contour length.\n\n For the list of possible cases and their figures, see page 38 from:\n https://nccastaff.bournemouth.ac.uk/jmacey/MastersProjects/MSc14/06/thesis.pdf\n\n In 2D, each point has 4 neighbors. Thus, are 16 configurations. A\n configuration is encoded with '1' meaning \"inside the object\" and '0' \"outside\n the object\". The points are ordered: top left, top right, bottom left, bottom\n right.\n\n The x0 axis is assumed vertical downward, and the x1 axis is horizontal to the\n right:\n (0, 0) --> (0, 1)\n |\n (1, 0)\n\n Args:\n spacing_mm: 2-element list-like structure. Voxel spacing in x0 and x1\n directions.\n \"\"\"\n neighbour_code_to_contour_length = np.zeros([16])\n\n vertical = spacing_mm[0]\n horizontal = spacing_mm[1]\n diag = 0.5 * math.sqrt(spacing_mm[0]**2 + spacing_mm[1]**2)\n # pyformat: disable\n neighbour_code_to_contour_length[int(\"00\" \"01\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"00\" \"10\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"00\" \"11\", 2)] = horizontal\n\n neighbour_code_to_contour_length[int(\"01\" \"00\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"01\" \"01\", 2)] = vertical\n\n neighbour_code_to_contour_length[int(\"01\" \"10\", 2)] = 2 * diag\n\n neighbour_code_to_contour_length[int(\"01\" \"11\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"10\" \"00\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"10\" \"01\", 2)] = 2 * diag\n\n neighbour_code_to_contour_length[int(\"10\" \"10\", 2)] = vertical\n\n neighbour_code_to_contour_length[int(\"10\" \"11\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"11\" \"00\", 2)] = horizontal\n\n neighbour_code_to_contour_length[int(\"11\" \"01\", 2)] = diag\n\n neighbour_code_to_contour_length[int(\"11\" \"10\", 2)] = diag\n # pyformat: enable\n\n return neighbour_code_to_contour_length\n\n\n# surface_distance/metrics.py\n\"\"\"Module exposing surface distance based measures.\"\"\"\n\nimport numpy as np\nfrom scipy import ndimage\n\n\ndef _assert_is_numpy_array(name, array):\n \"\"\"Raises an exception if `array` is not a numpy array.\"\"\"\n if not isinstance(array, np.ndarray):\n raise ValueError(\"The argument {!r} should be a numpy array, not a \"\n \"{}\".format(name, type(array)))\n\n\ndef _check_nd_numpy_array(name, array, num_dims):\n \"\"\"Raises an exception if `array` is not a `num_dims`-D numpy array.\"\"\"\n if len(array.shape) != num_dims:\n raise ValueError(\"The argument {!r} should be a {}D array, not of \"\n \"shape {}\".format(name, num_dims, array.shape))\n\n\ndef _check_2d_numpy_array(name, array):\n _check_nd_numpy_array(name, array, num_dims=2)\n\n\ndef _check_3d_numpy_array(name, array):\n _check_nd_numpy_array(name, array, num_dims=3)\n\n\ndef _assert_is_bool_numpy_array(name, array):\n _assert_is_numpy_array(name, array)\n if array.dtype != bool:\n raise ValueError(\n \"The argument {!r} should be a numpy array of type bool, \"\n \"not {}\".format(name, array.dtype))\n\n\ndef _compute_bounding_box(mask):\n \"\"\"Computes the bounding box of the masks.\n\n This function generalizes to arbitrary number of dimensions great or equal\n to 1.\n\n Args:\n mask: The 2D or 3D numpy mask, where '0' means background and non-zero means\n foreground.\n\n Returns:\n A tuple:\n - The coordinates of the first point of the bounding box (smallest on all\n axes), or `None` if the mask contains only zeros.\n - The coordinates of the second point of the bounding box (greatest on all\n axes), or `None` if the mask contains only zeros.\n \"\"\"\n num_dims = len(mask.shape)\n bbox_min = np.zeros(num_dims, np.int64)\n bbox_max = np.zeros(num_dims, np.int64)\n\n # max projection to the x0-axis\n proj_0 = np.amax(mask, axis=tuple(range(num_dims))[1:])\n idx_nonzero_0 = np.nonzero(proj_0)[0]\n if len(idx_nonzero_0) == 0: # pylint: disable=g-explicit-length-test\n return None, None\n\n bbox_min[0] = np.min(idx_nonzero_0)\n bbox_max[0] = np.max(idx_nonzero_0)\n\n # max projection to the i-th-axis for i in {1, ..., num_dims - 1}\n for axis in range(1, num_dims):\n max_over_axes = list(range(num_dims)) # Python 3 compatible\n max_over_axes.pop(axis) # Remove the i-th dimension from the max\n max_over_axes = tuple(max_over_axes) # numpy expects a tuple of ints\n proj = np.amax(mask, axis=max_over_axes)\n idx_nonzero = np.nonzero(proj)[0]\n bbox_min[axis] = np.min(idx_nonzero)\n bbox_max[axis] = np.max(idx_nonzero)\n\n return bbox_min, bbox_max\n\n\ndef _crop_to_bounding_box(mask, bbox_min, bbox_max):\n \"\"\"Crops a 2D or 3D mask to the bounding box specified by `bbox_{min,max}`.\"\"\"\n # we need to zeropad the cropped region with 1 voxel at the lower,\n # the right (and the back on 3D) sides. This is required to obtain the\n # \"full\" convolution result with the 2x2 (or 2x2x2 in 3D) kernel.\n # TODO: This is correct only if the object is interior to the\n # bounding box.\n cropmask = np.zeros((bbox_max - bbox_min) + 2, np.uint8)\n\n num_dims = len(mask.shape)\n # pyformat: disable\n if num_dims == 2:\n cropmask[0:-1, 0:-1] = mask[bbox_min[0]:bbox_max[0] + 1,\n bbox_min[1]:bbox_max[1] + 1]\n elif num_dims == 3:\n cropmask[0:-1, 0:-1, 0:-1] = mask[bbox_min[0]:bbox_max[0] + 1,\n bbox_min[1]:bbox_max[1] + 1,\n bbox_min[2]:bbox_max[2] + 1]\n # pyformat: enable\n else:\n assert False\n\n return cropmask\n\n\ndef _sort_distances_surfels(distances, surfel_areas):\n \"\"\"Sorts the two list with respect to the tuple of (distance, surfel_area).\n\n Args:\n distances: The distances from A to B (e.g. `distances_gt_to_pred`).\n surfel_areas: The surfel areas for A (e.g. `surfel_areas_gt`).\n\n Returns:\n A tuple of the sorted (distances, surfel_areas).\n \"\"\"\n sorted_surfels = np.array(sorted(zip(distances, surfel_areas)))\n return sorted_surfels[:, 0], sorted_surfels[:, 1]\n\n\ndef compute_surface_distances(mask_gt, mask_pred, spacing_mm):\n \"\"\"Computes closest distances from all surface points to the other surface.\n\n This function can be applied to 2D or 3D tensors. For 2D, both masks must be\n 2D and `spacing_mm` must be a 2-element list. For 3D, both masks must be 3D\n and `spacing_mm` must be a 3-element list. The description is done for the 2D\n case, and the formulation for the 3D case is present is parenthesis,\n introduced by \"resp.\".\n\n Finds all contour elements (resp surface elements \"surfels\" in 3D) in the\n ground truth mask `mask_gt` and the predicted mask `mask_pred`, computes their\n length in mm (resp. area in mm^2) and the distance to the closest point on the\n other contour (resp. surface). It returns two sorted lists of distances\n together with the corresponding contour lengths (resp. surfel areas). If one\n of the masks is empty, the corresponding lists are empty and all distances in\n the other list are `inf`.\n\n Args:\n mask_gt: 2-dim (resp. 3-dim) bool Numpy array. The ground truth mask.\n mask_pred: 2-dim (resp. 3-dim) bool Numpy array. The predicted mask.\n spacing_mm: 2-element (resp. 3-element) list-like structure. Voxel spacing\n in x0 anx x1 (resp. x0, x1 and x2) directions.\n\n Returns:\n A dict with:\n \"distances_gt_to_pred\": 1-dim numpy array of type float. The distances in mm\n from all ground truth surface elements to the predicted surface,\n sorted from smallest to largest.\n \"distances_pred_to_gt\": 1-dim numpy array of type float. The distances in mm\n from all predicted surface elements to the ground truth surface,\n sorted from smallest to largest.\n \"surfel_areas_gt\": 1-dim numpy array of type float. The length of the\n of the ground truth contours in mm (resp. the surface elements area in\n mm^2) in the same order as distances_gt_to_pred.\n \"surfel_areas_pred\": 1-dim numpy array of type float. The length of the\n of the predicted contours in mm (resp. the surface elements area in\n mm^2) in the same order as distances_gt_to_pred.\n\n Raises:\n ValueError: If the masks and the `spacing_mm` arguments are of incompatible\n shape or type. Or if the masks are not 2D or 3D.\n \"\"\"\n # The terms used in this function are for the 3D case. In particular, surface\n # in 2D stands for contours in 3D. The surface elements in 3D correspond to\n # the line elements in 2D.\n\n _assert_is_bool_numpy_array(\"mask_gt\", mask_gt)\n _assert_is_bool_numpy_array(\"mask_pred\", mask_pred)\n\n if not len(mask_gt.shape) == len(mask_pred.shape) == len(spacing_mm):\n raise ValueError(\n \"The arguments must be of compatible shape. Got mask_gt \"\n \"with {} dimensions ({}) and mask_pred with {} dimensions \"\n \"({}), while the spacing_mm was {} elements.\".format(\n len(mask_gt.shape), mask_gt.shape, len(mask_pred.shape),\n mask_pred.shape, len(spacing_mm)))\n\n num_dims = len(spacing_mm)\n if num_dims == 2:\n _check_2d_numpy_array(\"mask_gt\", mask_gt)\n _check_2d_numpy_array(\"mask_pred\", mask_pred)\n\n # compute the area for all 16 possible surface elements\n # (given a 2x2 neighbourhood) according to the spacing_mm\n neighbour_code_to_surface_area = (\n create_table_neighbour_code_to_contour_length(spacing_mm))\n kernel = ENCODE_NEIGHBOURHOOD_2D_KERNEL\n full_true_neighbours = 0b1111\n elif num_dims == 3:\n _check_3d_numpy_array(\"mask_gt\", mask_gt)\n _check_3d_numpy_array(\"mask_pred\", mask_pred)\n\n # compute the area for all 256 possible surface elements\n # (given a 2x2x2 neighbourhood) according to the spacing_mm\n neighbour_code_to_surface_area = (\n create_table_neighbour_code_to_surface_area(spacing_mm))\n kernel = ENCODE_NEIGHBOURHOOD_3D_KERNEL\n full_true_neighbours = 0b11111111\n else:\n raise ValueError(\"Only 2D and 3D masks are supported, not \"\n \"{}D.\".format(num_dims))\n\n # compute the bounding box of the masks to trim the volume to the smallest\n # possible processing subvolume\n bbox_min, bbox_max = _compute_bounding_box(mask_gt | mask_pred)\n # Both the min/max bbox are None at the same time, so we only check one.\n if bbox_min is None:\n return {\n \"distances_gt_to_pred\": np.array([]),\n \"distances_pred_to_gt\": np.array([]),\n \"surfel_areas_gt\": np.array([]),\n \"surfel_areas_pred\": np.array([]),\n }\n\n # crop the processing subvolume.\n cropmask_gt = _crop_to_bounding_box(mask_gt, bbox_min, bbox_max)\n cropmask_pred = _crop_to_bounding_box(mask_pred, bbox_min, bbox_max)\n\n # compute the neighbour code (local binary pattern) for each voxel\n # the resulting arrays are spacially shifted by minus half a voxel in each\n # axis.\n # i.e. the points are located at the corners of the original voxels\n neighbour_code_map_gt = ndimage.correlate(cropmask_gt.astype(np.uint8),\n kernel,\n mode=\"constant\",\n cval=0)\n neighbour_code_map_pred = ndimage.correlate(cropmask_pred.astype(np.uint8),\n kernel,\n mode=\"constant\",\n cval=0)\n\n # create masks with the surface voxels\n borders_gt = ((neighbour_code_map_gt != 0) &\n (neighbour_code_map_gt != full_true_neighbours))\n borders_pred = ((neighbour_code_map_pred != 0) &\n (neighbour_code_map_pred != full_true_neighbours))\n\n # compute the distance transform (closest distance of each voxel to the\n # surface voxels)\n if borders_gt.any():\n distmap_gt = distance_transform_edt(~borders_gt)\n\n else:\n distmap_gt = np.Inf * np.ones(borders_gt.shape)\n\n distances_pred_to_gt = distmap_gt[borders_pred]\n del distmap_gt\n\n if borders_pred.any():\n distmap_pred = distance_transform_edt(~borders_pred)\n\n else:\n distmap_pred = np.Inf * np.ones(borders_pred.shape)\n\n distances_gt_to_pred = distmap_pred[borders_gt]\n del distmap_pred\n\n # compute the area of each surface element\n surface_area_map_gt = neighbour_code_to_surface_area[neighbour_code_map_gt]\n surfel_areas_gt = surface_area_map_gt[borders_gt]\n del neighbour_code_map_gt, surface_area_map_gt, borders_gt\n\n surface_area_map_pred = neighbour_code_to_surface_area[neighbour_code_map_pred]\n surfel_areas_pred = surface_area_map_pred[borders_pred]\n del neighbour_code_map_pred, surface_area_map_pred, borders_pred\n\n # sort them by distance\n if distances_gt_to_pred.shape != (0, ):\n distances_gt_to_pred, surfel_areas_gt = _sort_distances_surfels(\n distances_gt_to_pred, surfel_areas_gt)\n\n if distances_pred_to_gt.shape != (0, ):\n distances_pred_to_gt, surfel_areas_pred = _sort_distances_surfels(\n distances_pred_to_gt, surfel_areas_pred)\n\n return {\n \"distances_gt_to_pred\": distances_gt_to_pred,\n \"distances_pred_to_gt\": distances_pred_to_gt,\n \"surfel_areas_gt\": surfel_areas_gt,\n \"surfel_areas_pred\": surfel_areas_pred,\n }\n\n\ndef compute_average_surface_distance(surface_distances):\n \"\"\"Returns the average surface distance.\n\n Computes the average surface distances by correctly taking the area of each\n surface element into account. Call compute_surface_distances(...) before, to\n obtain the `surface_distances` dict.\n\n Args:\n surface_distances: dict with \"distances_gt_to_pred\", \"distances_pred_to_gt\"\n \"surfel_areas_gt\", \"surfel_areas_pred\" created by\n compute_surface_distances()\n\n Returns:\n A tuple with two float values:\n - the average distance (in mm) from the ground truth surface to the\n predicted surface\n - the average distance from the predicted surface to the ground truth\n surface.\n \"\"\"\n distances_gt_to_pred = surface_distances[\"distances_gt_to_pred\"]\n distances_pred_to_gt = surface_distances[\"distances_pred_to_gt\"]\n surfel_areas_gt = surface_distances[\"surfel_areas_gt\"]\n surfel_areas_pred = surface_distances[\"surfel_areas_pred\"]\n average_distance_gt_to_pred = (\n np.sum(distances_gt_to_pred * surfel_areas_gt) /\n np.sum(surfel_areas_gt))\n average_distance_pred_to_gt = (\n np.sum(distances_pred_to_gt * surfel_areas_pred) /\n np.sum(surfel_areas_pred))\n return (average_distance_gt_to_pred, average_distance_pred_to_gt)\n\n\ndef compute_robust_hausdorff(surface_distances, percent):\n \"\"\"Computes the robust Hausdorff distance.\n\n Computes the robust Hausdorff distance. \"Robust\", because it uses the\n `percent` percentile of the distances instead of the maximum distance. The\n percentage is computed by correctly taking the area of each surface element\n into account.\n\n Args:\n surface_distances: dict with \"distances_gt_to_pred\", \"distances_pred_to_gt\"\n \"surfel_areas_gt\", \"surfel_areas_pred\" created by\n compute_surface_distances()\n percent: a float value between 0 and 100.\n\n Returns:\n a float value. The robust Hausdorff distance in mm.\n \"\"\"\n distances_gt_to_pred = surface_distances[\"distances_gt_to_pred\"]\n distances_pred_to_gt = surface_distances[\"distances_pred_to_gt\"]\n surfel_areas_gt = surface_distances[\"surfel_areas_gt\"]\n surfel_areas_pred = surface_distances[\"surfel_areas_pred\"]\n if len(distances_gt_to_pred) > 0: # pylint: disable=g-explicit-length-test\n surfel_areas_cum_gt = np.cumsum(surfel_areas_gt) / np.sum(\n surfel_areas_gt)\n idx = np.searchsorted(surfel_areas_cum_gt, percent / 100.0)\n perc_distance_gt_to_pred = distances_gt_to_pred[min(\n idx,\n len(distances_gt_to_pred) - 1)]\n else:\n perc_distance_gt_to_pred = np.Inf\n\n if len(distances_pred_to_gt) > 0: # pylint: disable=g-explicit-length-test\n surfel_areas_cum_pred = (np.cumsum(surfel_areas_pred) /\n np.sum(surfel_areas_pred))\n idx = np.searchsorted(surfel_areas_cum_pred, percent / 100.0)\n perc_distance_pred_to_gt = distances_pred_to_gt[min(\n idx,\n len(distances_pred_to_gt) - 1)]\n else:\n perc_distance_pred_to_gt = np.Inf\n\n return max(perc_distance_gt_to_pred, perc_distance_pred_to_gt)\n\n\ndef compute_surface_overlap_at_tolerance(surface_distances, tolerance_mm):\n \"\"\"Computes the overlap of the surfaces at a specified tolerance.\n\n Computes the overlap of the ground truth surface with the predicted surface\n and vice versa allowing a specified tolerance (maximum surface-to-surface\n distance that is regarded as overlapping). The overlapping fraction is\n computed by correctly taking the area of each surface element into account.\n\n Args:\n surface_distances: dict with \"distances_gt_to_pred\", \"distances_pred_to_gt\"\n \"surfel_areas_gt\", \"surfel_areas_pred\" created by\n compute_surface_distances()\n tolerance_mm: a float value. The tolerance in mm\n\n Returns:\n A tuple of two float values. The overlap fraction in [0.0, 1.0] of the\n ground truth surface with the predicted surface and vice versa.\n \"\"\"\n distances_gt_to_pred = surface_distances[\"distances_gt_to_pred\"]\n distances_pred_to_gt = surface_distances[\"distances_pred_to_gt\"]\n surfel_areas_gt = surface_distances[\"surfel_areas_gt\"]\n surfel_areas_pred = surface_distances[\"surfel_areas_pred\"]\n rel_overlap_gt = (\n np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm]) /\n np.sum(surfel_areas_gt))\n rel_overlap_pred = (\n np.sum(surfel_areas_pred[distances_pred_to_gt <= tolerance_mm]) /\n np.sum(surfel_areas_pred))\n return (rel_overlap_gt, rel_overlap_pred)\n\n\ndef compute_surface_dice_at_tolerance(surface_distances, tolerance_mm):\n \"\"\"Computes the _surface_ DICE coefficient at a specified tolerance.\n\n Computes the _surface_ DICE coefficient at a specified tolerance. Not to be\n confused with the standard _volumetric_ DICE coefficient. The surface DICE\n measures the overlap of two surfaces instead of two volumes. A surface\n element is counted as overlapping (or touching), when the closest distance to\n the other surface is less or equal to the specified tolerance. The DICE\n coefficient is in the range between 0.0 (no overlap) to 1.0 (perfect overlap).\n\n Args:\n surface_distances: dict with \"distances_gt_to_pred\", \"distances_pred_to_gt\"\n \"surfel_areas_gt\", \"surfel_areas_pred\" created by\n compute_surface_distances()\n tolerance_mm: a float value. The tolerance in mm\n\n Returns:\n A float value. The surface DICE coefficient in [0.0, 1.0].\n \"\"\"\n distances_gt_to_pred = surface_distances[\"distances_gt_to_pred\"]\n distances_pred_to_gt = surface_distances[\"distances_pred_to_gt\"]\n surfel_areas_gt = surface_distances[\"surfel_areas_gt\"]\n surfel_areas_pred = surface_distances[\"surfel_areas_pred\"]\n overlap_gt = np.sum(surfel_areas_gt[distances_gt_to_pred <= tolerance_mm])\n overlap_pred = np.sum(\n surfel_areas_pred[distances_pred_to_gt <= tolerance_mm])\n surface_dice = (overlap_gt + overlap_pred) / (np.sum(surfel_areas_gt) +\n np.sum(surfel_areas_pred))\n return surface_dice\n\n\ndef compute_dice_coefficient(mask_gt, mask_pred):\n \"\"\"Computes soerensen-dice coefficient.\n\n compute the soerensen-dice coefficient between the ground truth mask `mask_gt`\n and the predicted mask `mask_pred`.\n\n Args:\n mask_gt: 3-dim Numpy array of type bool. The ground truth mask.\n mask_pred: 3-dim Numpy array of type bool. The predicted mask.\n\n Returns:\n the dice coeffcient as float. If both masks are empty, the result is NaN.\n \"\"\"\n volume_sum = mask_gt.sum() + mask_pred.sum()\n if volume_sum == 0:\n return np.NaN\n volume_intersect = (mask_gt & mask_pred).sum()\n return 2 * volume_intersect / volume_sum\n\n\n# Below is from https://github.com/Image-Py/imagepy/blob/master/imagepy/ipyalg/hydrology/edt.py\n# Copyright (c) <2016>, \n# All rights reserved.\n\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are met:\n# 1. Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n# 2. Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n# 3. All advertising materials mentioning features or use of this software\n# must display the following acknowledgement:\n# This product includes software developed by the .\n# 4. Neither the name of the nor the\n# names of its contributors may be used to endorse or promote products\n# derived from this software without specific prior written permission.\n\n# THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY\n# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n# DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY\n# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\ndef neighbors(shape):\n dim = len(shape)\n block = generate_binary_structure(dim, 1)\n block[tuple([1]*dim)] = 0\n idx = np.where(block>0)\n idx = np.array(idx, dtype=np.uint8).T\n idx = np.array(idx-[1]*dim)\n acc = np.cumprod((1,)+shape[::-1][:-1])\n return np.dot(idx, acc[::-1])\n\n\n@jit(nopython=True)\ndef dist(idx1, idx2, acc):\n dis = 0\n for i in range(len(acc)):\n c1 = idx1//acc[i]\n c2 = idx2//acc[i]\n dis += (c1-c2)**2\n idx1 -= c1*acc[i]\n idx2 -= c2*acc[i]\n return dis\n\n\n@jit(nopython=True)\ndef _step(dis, pts, roots, s, level, nbs, acc, scale):\n cur = 0\n while curlevel:\n cur += 1\n continue\n for dp in nbs:\n cp = p+dp\n if dis[cp]=0xffff-1: continue # edge or back\n for dp in nbs:\n if dis[p+dp]==0xffff-1:\n pts[cur] = p\n root[cur] = p\n cur += 1\n break\n return cur\n\n\n@jit(nopython=True)\ndef bufjit(line):\n for i in range(len(line)):\n line[i] = 1 if line[i]==0 else 0xffff\n\n\ndef buffer(img, dtype):\n buf = np.ones(tuple(np.array(img.shape)+2), dtype=dtype)\n buf[tuple([slice(1,-1)]*buf.ndim)] = img\n bufjit(buf.ravel())\n buf[tuple([slice(1,-1)]*buf.ndim)] -= 1\n return buf\n\n\ndef distance_transform_edt(img, output=np.float32, scale=1):\n dis = buffer(img, output)\n nbs = neighbors(dis.shape)\n acc = np.cumprod((1,)+dis.shape[::-1][:-1])[::-1]\n line = dis.ravel()\n pts = np.zeros(max(line.size//4, 1024**2), dtype=np.int64)\n roots = np.zeros(max(line.size//4, 1024**2), dtype=np.int64)\n s = collect(line, nbs, pts, roots)\n for level in range(10000):\n s, c = clear(pts, roots, s, 0)\n s = _step(line, pts, roots, s, level, nbs, acc, scale)\n if s==0:break\n return dis[(slice(1,-1),)*img.ndim]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-12-07T00:27:03.628197Z","iopub.status.idle":"2023-12-07T00:27:03.628691Z","shell.execute_reply.started":"2023-12-07T00:27:03.628435Z","shell.execute_reply":"2023-12-07T00:27:03.628466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(img):\n '''\n img: numpy array, 1 - mask, 0 - background\n Returns run length as string formated\n '''\n pixels = img.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n rle = ' '.join(str(x) for x in runs)\n if rle=='':\n rle = '1 0'\n return rle","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.629804Z","iopub.status.idle":"2023-12-07T00:27:03.630278Z","shell.execute_reply.started":"2023-12-07T00:27:03.630044Z","shell.execute_reply":"2023-12-07T00:27:03.630067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = BuildDataset(valid_img_paths, [], transforms=CFG.data_transforms['valid'])\ntest_loader = DataLoader(test_dataset, batch_size=CFG.valid_bs, num_workers=0, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.631589Z","iopub.status.idle":"2023-12-07T00:27:03.632082Z","shell.execute_reply.started":"2023-12-07T00:27:03.631827Z","shell.execute_reply":"2023-12-07T00:27:03.631867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Inference\nrles = []\npbar = tqdm(enumerate(test_loader), total=len(test_loader), desc='Inference ')\nfor step, (images, shapes) in pbar:\n shapes = shapes.numpy()\n images = images.to(CFG.device, dtype=torch.float)\n with torch.no_grad():\n preds = model(images)\n preds = (nn.Sigmoid()(preds)>0.5).double()\n preds = preds.cpu().numpy().astype(np.uint8)\n\n for pred, shape in zip(preds, shapes):\n pred = cv2.resize(pred[0], (shape[1], shape[0]), cv2.INTER_NEAREST)\n rle = rle_encode(pred)\n rles.append(rle)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.633498Z","iopub.status.idle":"2023-12-07T00:27:03.633985Z","shell.execute_reply.started":"2023-12-07T00:27:03.633722Z","shell.execute_reply":"2023-12-07T00:27:03.633745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = [f'{p.split(\"/\")[-3]}_{os.path.basename(p).split(\".\")[0]}' for p in valid_img_paths]\nsubmission = pd.DataFrame.from_dict({\n \"id\": ids,\n \"rle\": rles\n})\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.635599Z","iopub.status.idle":"2023-12-07T00:27:03.636101Z","shell.execute_reply.started":"2023-12-07T00:27:03.635825Z","shell.execute_reply":"2023-12-07T00:27:03.635888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_gt_df = pd.merge(gt_df, submission.loc[:, [\"id\"]], on=\"id\").reset_index(drop=True)\n_gt_df","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.637439Z","iopub.status.idle":"2023-12-07T00:27:03.637788Z","shell.execute_reply.started":"2023-12-07T00:27:03.637619Z","shell.execute_reply":"2023-12-07T00:27:03.637636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_score = score(submission, _gt_df, \"id\", \"rle\", 0.0, \"group\", \"slice\")\nprint(val_score)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T00:27:03.63875Z","iopub.status.idle":"2023-12-07T00:27:03.639126Z","shell.execute_reply.started":"2023-12-07T00:27:03.638951Z","shell.execute_reply":"2023-12-07T00:27:03.638969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}