{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%env CUDA_HOME=/usr/local/cuda-11.8/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:31:39.271365Z","iopub.execute_input":"2023-07-15T13:31:39.271723Z","iopub.status.idle":"2023-07-15T13:31:39.284538Z","shell.execute_reply.started":"2023-07-15T13:31:39.271692Z","shell.execute_reply":"2023-07-15T13:31:39.283255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/maskdino-sourcecode /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:31:39.288556Z","iopub.execute_input":"2023-07-15T13:31:39.289187Z","iopub.status.idle":"2023-07-15T13:31:40.719417Z","shell.execute_reply.started":"2023-07-15T13:31:39.289131Z","shell.execute_reply":"2023-07-15T13:31:40.718117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working/maskdino-sourcecode/MaskDINO/maskdino/modeling/pixel_decoder/ops","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:31:40.722074Z","iopub.execute_input":"2023-07-15T13:31:40.722461Z","iopub.status.idle":"2023-07-15T13:31:40.732293Z","shell.execute_reply.started":"2023-07-15T13:31:40.722424Z","shell.execute_reply":"2023-07-15T13:31:40.730931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!sh make.sh","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:31:40.734198Z","iopub.execute_input":"2023-07-15T13:31:40.735008Z","iopub.status.idle":"2023-07-15T13:32:33.121181Z","shell.execute_reply.started":"2023-07-15T13:31:40.734977Z","shell.execute_reply":"2023-07-15T13:32:33.120019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working/maskdino-sourcecode/MaskDINO/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:32:33.123948Z","iopub.execute_input":"2023-07-15T13:32:33.124252Z","iopub.status.idle":"2023-07-15T13:32:33.132383Z","shell.execute_reply.started":"2023-07-15T13:32:33.124225Z","shell.execute_reply":"2023-07-15T13:32:33.131451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/detectron2-wheel/detectron2/detectron2-0.6-cp310-cp310-linux_x86_64.whl --no-index --find-links=/kaggle/input/detectron2-wheel/detectron2\n!pip install /kaggle/input/yolov7-weights-and-wheels/yolo_wheel/yolov7-0.0.1-py37.py38.py39-none-any.whl --no-index --find-links=/kaggle/input/yolov7-weights-and-wheels/yolo_wheel","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:32:33.134264Z","iopub.execute_input":"2023-07-15T13:32:33.135412Z","iopub.status.idle":"2023-07-15T13:33:15.587939Z","shell.execute_reply.started":"2023-07-15T13:32:33.135379Z","shell.execute_reply":"2023-07-15T13:33:15.586737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install pycocotools package\nimport os\n!mkdir /kaggle/working/packages\n!cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\nos.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n!python setup.py install -q\n!pip install . --no-index --find-links /kaggle/working/packages/ -q\nos.chdir(\"/kaggle/working\")","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:33:15.58998Z","iopub.execute_input":"2023-07-15T13:33:15.590363Z","iopub.status.idle":"2023-07-15T13:33:52.707296Z","shell.execute_reply.started":"2023-07-15T13:33:15.590328Z","shell.execute_reply":"2023-07-15T13:33:52.705625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/addict-2.4.0-py3-none-any.whl\n!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/yapf-0.32.0-py2.py3-none-any.whl\n!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/terminal-0.4.0-py3-none-any.whl\n!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/terminaltables-3.1.10-py2.py3-none-any.whl\n!pip install /kaggle/input/mmdet3-wheels/mmcv_full-1.7.1-cp310-cp310-linux_x86_64.whl\n!cp -r /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdetection/ /kaggle/working/\n%cd /kaggle/working/mmdetection\n!pip install -e . --no-deps\n%cd /kaggle/working/\n\n!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdet-2.26.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:33:52.709682Z","iopub.execute_input":"2023-07-15T13:33:52.710119Z","iopub.status.idle":"2023-07-15T13:35:24.953343Z","shell.execute_reply.started":"2023-07-15T13:33:52.710083Z","shell.execute_reply":"2023-07-15T13:35:24.952116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/internimage-sourcecode/ops_dcnv3 /kaggle/working/\n%cd /kaggle/working/ops_dcnv3\n!sh ./make.sh\n!cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:35:24.95531Z","iopub.execute_input":"2023-07-15T13:35:24.955798Z","iopub.status.idle":"2023-07-15T13:37:00.275065Z","shell.execute_reply.started":"2023-07-15T13:35:24.955764Z","shell.execute_reply":"2023-07-15T13:37:00.273724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/mmdetection\n!rm -rf /kaggle/working/packages\n!rm -rf /kaggle/working/ops_dcnv3\n","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:00.277011Z","iopub.execute_input":"2023-07-15T13:37:00.277863Z","iopub.status.idle":"2023-07-15T13:37:03.154164Z","shell.execute_reply.started":"2023-07-15T13:37:00.277826Z","shell.execute_reply":"2023-07-15T13:37:03.152864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install /kaggle/input/detectron2-wheel/detectron2/detectron2-0.6-cp310-cp310-linux_x86_64.whl --no-index --find-links=/kaggle/input/detectron2-wheel/detectron2","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:03.159203Z","iopub.execute_input":"2023-07-15T13:37:03.159501Z","iopub.status.idle":"2023-07-15T13:37:03.164386Z","shell.execute_reply.started":"2023-07-15T13:37:03.159475Z","shell.execute_reply":"2023-07-15T13:37:03.16329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cp -r /kaggle/input/ensemble-boxes /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:03.165943Z","iopub.execute_input":"2023-07-15T13:37:03.166283Z","iopub.status.idle":"2023-07-15T13:37:03.175516Z","shell.execute_reply.started":"2023-07-15T13:37:03.166249Z","shell.execute_reply":"2023-07-15T13:37:03.174201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working/maskdino-sourcecode/MaskDINO/","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:03.178321Z","iopub.execute_input":"2023-07-15T13:37:03.179285Z","iopub.status.idle":"2023-07-15T13:37:03.188092Z","shell.execute_reply.started":"2023-07-15T13:37:03.179259Z","shell.execute_reply":"2023-07-15T13:37:03.187024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/yolov7-weights-and-wheels/yolov7 yolo","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:03.190094Z","iopub.execute_input":"2023-07-15T13:37:03.190502Z","iopub.status.idle":"2023-07-15T13:37:04.949743Z","shell.execute_reply.started":"2023-07-15T13:37:03.190469Z","shell.execute_reply":"2023-07-15T13:37:04.948419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yaml_text = \"\"\"\n# class names\nnames: \n 0: glomerulus\n 1: blood_vessel\n\"\"\"\nwith open('/kaggle/working/hubmap-coco.yaml', 'w') as text_file:\n text_file.write(yaml_text)","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:04.951391Z","iopub.execute_input":"2023-07-15T13:37:04.95179Z","iopub.status.idle":"2023-07-15T13:37:04.960178Z","shell.execute_reply.started":"2023-07-15T13:37:04.951756Z","shell.execute_reply":"2023-07-15T13:37:04.959059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yaml_text = \"\"\"\n# class names\nnames: \n 0: blood_vessel\n\"\"\"\nwith open('/kaggle/working/hubmap-coco_1class.yaml', 'w') as text_file:\n text_file.write(yaml_text)","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:04.96216Z","iopub.execute_input":"2023-07-15T13:37:04.96295Z","iopub.status.idle":"2023-07-15T13:37:04.968818Z","shell.execute_reply.started":"2023-07-15T13:37:04.962917Z","shell.execute_reply":"2023-07-15T13:37:04.967826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolo.seg.segment import predict","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:04.972119Z","iopub.execute_input":"2023-07-15T13:37:04.972405Z","iopub.status.idle":"2023-07-15T13:37:07.495819Z","shell.execute_reply.started":"2023-07-15T13:37:04.972374Z","shell.execute_reply":"2023-07-15T13:37:07.494845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import argparse\nimport os\nimport platform\nimport sys\nfrom pathlib import Path\nimport json\n\nimport torch\nimport torch.backends.cudnn as cudnn\n\n\n\nfrom models.common import DetectMultiBackend\nfrom utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadStreams\nfrom utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2,\n increment_path, non_max_suppression, print_args, scale_coords, strip_optimizer, xyxy2xywh)\nfrom utils.plots import Annotator, colors, save_one_box\nfrom utils.segment.general import process_mask, scale_masks\nfrom utils.segment.plots import plot_masks\nfrom utils.torch_utils import select_device, smart_inference_mode","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:07.49737Z","iopub.execute_input":"2023-07-15T13:37:07.497931Z","iopub.status.idle":"2023-07-15T13:37:07.506619Z","shell.execute_reply.started":"2023-07-15T13:37:07.497892Z","shell.execute_reply":"2023-07-15T13:37:07.504482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nCommon modules\n\"\"\"\n\nimport math\nimport warnings\nfrom collections import OrderedDict, namedtuple\nfrom copy import copy\nsys.path.insert(0,'.')\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport requests\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom torch.cuda import amp\n\nfrom utils.dataloaders import exif_transpose, letterbox\nfrom utils.general import (LOGGER, ROOT, Profile, check_requirements, check_suffix, check_version, colorstr,\n increment_path, make_divisible, non_max_suppression, scale_coords, xywh2xyxy, xyxy2xywh,\n yaml_load)\nfrom utils.plots import Annotator, colors, save_one_box\nfrom utils.torch_utils import copy_attr, smart_inference_mode\n\nclass DetectMultiBackend(nn.Module):\n # YOLOv5 MultiBackend class for python inference on various backends\n def __init__(self, weights='yolov5s.pt', device=torch.device('cpu'), dnn=False, data=None, fp16=False, fuse=True):\n # Usage:\n # PyTorch: weights = *.pt\n # TorchScript: *.torchscript\n # ONNX Runtime: *.onnx\n # ONNX OpenCV DNN: *.onnx with --dnn\n # OpenVINO: *.xml\n # CoreML: *.mlmodel\n # TensorRT: *.engine\n # TensorFlow SavedModel: *_saved_model\n # TensorFlow GraphDef: *.pb\n # TensorFlow Lite: *.tflite\n # TensorFlow Edge TPU: *_edgetpu.tflite\n from models.experimental import attempt_download, attempt_load # scoped to avoid circular import\n\n super().__init__()\n w = str(weights[0] if isinstance(weights, list) else weights)\n pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs = self._model_type(w) # get backend\n w = attempt_download(w) # download if not local\n fp16 &= pt or jit or onnx or engine # FP16\n stride = 32 # default stride\n\n if pt: # PyTorch\n model = attempt_load(weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse)\n stride = max(int(model.stride.max()), 32) # model stride\n names = model.module.names if hasattr(model, 'module') else model.names # get class names\n model.half() if fp16 else model.float()\n self.model = model # explicitly assign for to(), cpu(), cuda(), half()\n #segmentation_model = type(model.model[-1]).__name__ in ['Segment', 'ISegment', 'IRSegment']\n segmentation_model = True\n elif jit: # TorchScript\n LOGGER.info(f'Loading {w} for TorchScript inference...')\n extra_files = {'config.txt': ''} # model metadata\n model = torch.jit.load(w, _extra_files=extra_files)\n model.half() if fp16 else model.float()\n if extra_files['config.txt']: # load metadata dict\n d = json.loads(extra_files['config.txt'],\n object_hook=lambda d: {int(k) if k.isdigit() else k: v\n for k, v in d.items()})\n stride, names = int(d['stride']), d['names']\n elif dnn: # ONNX OpenCV DNN\n LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')\n check_requirements(('opencv-python>=4.5.4',))\n net = cv2.dnn.readNetFromONNX(w)\n elif onnx: # ONNX Runtime\n LOGGER.info(f'Loading {w} for ONNX Runtime inference...')\n cuda = torch.cuda.is_available() and device.type != 'cpu'\n check_requirements(('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))\n import onnxruntime\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']\n session = onnxruntime.InferenceSession(w, providers=providers)\n meta = session.get_modelmeta().custom_metadata_map # metadata\n if 'stride' in meta:\n stride, names = int(meta['stride']), eval(meta['names'])\n elif xml: # OpenVINO\n LOGGER.info(f'Loading {w} for OpenVINO inference...')\n check_requirements(('openvino',)) # requires openvino-dev: https://pypi.org/project/openvino-dev/\n from openvino.runtime import Core, Layout, get_batch\n ie = Core()\n if not Path(w).is_file(): # if not *.xml\n w = next(Path(w).glob('*.xml')) # get *.xml file from *_openvino_model dir\n network = ie.read_model(model=w, weights=Path(w).with_suffix('.bin'))\n if network.get_parameters()[0].get_layout().empty:\n network.get_parameters()[0].set_layout(Layout(\"NCHW\"))\n batch_dim = get_batch(network)\n if batch_dim.is_static:\n batch_size = batch_dim.get_length()\n executable_network = ie.compile_model(network, device_name=\"CPU\") # device_name=\"MYRIAD\" for Intel NCS2\n output_layer = next(iter(executable_network.outputs))\n meta = Path(w).with_suffix('.yaml')\n if meta.exists():\n stride, names = self._load_metadata(meta) # load metadata\n elif engine: # TensorRT\n LOGGER.info(f'Loading {w} for TensorRT inference...')\n import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download\n check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0\n if device.type == 'cpu':\n device = torch.device('cuda:0')\n Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr'))\n logger = trt.Logger(trt.Logger.INFO)\n with open(w, 'rb') as f, trt.Runtime(logger) as runtime:\n model = runtime.deserialize_cuda_engine(f.read())\n context = model.create_execution_context()\n bindings = OrderedDict()\n fp16 = False # default updated below\n dynamic = False\n for index in range(model.num_bindings):\n name = model.get_binding_name(index)\n dtype = trt.nptype(model.get_binding_dtype(index))\n if model.binding_is_input(index):\n if -1 in tuple(model.get_binding_shape(index)): # dynamic\n dynamic = True\n context.set_binding_shape(index, tuple(model.get_profile_shape(0, index)[2]))\n if dtype == np.float16:\n fp16 = True\n shape = tuple(context.get_binding_shape(index))\n im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device)\n bindings[name] = Binding(name, dtype, shape, im, int(im.data_ptr()))\n binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items())\n batch_size = bindings['images'].shape[0] # if dynamic, this is instead max batch size\n elif coreml: # CoreML\n LOGGER.info(f'Loading {w} for CoreML inference...')\n import coremltools as ct\n model = ct.models.MLModel(w)\n else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)\n if saved_model: # SavedModel\n LOGGER.info(f'Loading {w} for TensorFlow SavedModel inference...')\n import tensorflow as tf\n keras = False # assume TF1 saved_model\n model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w)\n elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt\n LOGGER.info(f'Loading {w} for TensorFlow GraphDef inference...')\n import tensorflow as tf\n\n def wrap_frozen_graph(gd, inputs, outputs):\n x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=\"\"), []) # wrapped\n ge = x.graph.as_graph_element\n return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs))\n\n gd = tf.Graph().as_graph_def() # graph_def\n with open(w, 'rb') as f:\n gd.ParseFromString(f.read())\n frozen_func = wrap_frozen_graph(gd, inputs=\"x:0\", outputs=\"Identity:0\")\n elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python\n try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu\n from tflite_runtime.interpreter import Interpreter, load_delegate\n except ImportError:\n import tensorflow as tf\n Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,\n if edgetpu: # Edge TPU https://coral.ai/software/#edgetpu-runtime\n LOGGER.info(f'Loading {w} for TensorFlow Lite Edge TPU inference...')\n delegate = {\n 'Linux': 'libedgetpu.so.1',\n 'Darwin': 'libedgetpu.1.dylib',\n 'Windows': 'edgetpu.dll'}[platform.system()]\n interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)])\n else: # Lite\n LOGGER.info(f'Loading {w} for TensorFlow Lite inference...')\n interpreter = Interpreter(model_path=w) # load TFLite model\n interpreter.allocate_tensors() # allocate\n input_details = interpreter.get_input_details() # inputs\n output_details = interpreter.get_output_details() # outputs\n elif tfjs:\n raise NotImplementedError('ERROR: YOLOv5 TF.js inference is not supported')\n else:\n raise NotImplementedError(f'ERROR: {w} is not a supported format')\n\n # class names\n if 'names' not in locals():\n names = yaml_load(data)['names'] if data else {i: f'class{i}' for i in range(999)}\n if names[0] == 'n01440764' and len(names) == 1000: # ImageNet\n names = yaml_load(ROOT / 'data/ImageNet.yaml')['names'] # human-readable names\n\n self.__dict__.update(locals()) # assign all variables to self\n\n def forward(self, im, augment=False, visualize=False, val=False):\n # YOLOv5 MultiBackend inference\n b, ch, h, w = im.shape # batch, channel, height, width\n if self.fp16 and im.dtype != torch.float16:\n im = im.half() # to FP16\n\n if self.pt: # PyTorch\n y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im)\n if isinstance(y, tuple) and not self.segmentation_model:\n y = y[0]\n elif self.jit: # TorchScript\n y = self.model(im)[0]\n elif self.dnn: # ONNX OpenCV DNN\n im = im.cpu().numpy() # torch to numpy\n self.net.setInput(im)\n y = self.net.forward()\n elif self.onnx: # ONNX Runtime\n im = im.cpu().numpy() # torch to numpy\n y = self.session.run([self.session.get_outputs()[0].name], {self.session.get_inputs()[0].name: im})[0]\n elif self.xml: # OpenVINO\n im = im.cpu().numpy() # FP32\n y = self.executable_network([im])[self.output_layer]\n elif self.engine: # TensorRT\n if self.dynamic and im.shape != self.bindings['images'].shape:\n i_in, i_out = (self.model.get_binding_index(x) for x in ('images', 'output'))\n self.context.set_binding_shape(i_in, im.shape) # reshape if dynamic\n self.bindings['images'] = self.bindings['images']._replace(shape=im.shape)\n self.bindings['output'].data.resize_(tuple(self.context.get_binding_shape(i_out)))\n s = self.bindings['images'].shape\n assert im.shape == s, f\"input size {im.shape} {'>' if self.dynamic else 'not equal to'} max model size {s}\"\n self.binding_addrs['images'] = int(im.data_ptr())\n self.context.execute_v2(list(self.binding_addrs.values()))\n y = self.bindings['output'].data\n elif self.coreml: # CoreML\n im = im.permute(0, 2, 3, 1).cpu().numpy() # torch BCHW to numpy BHWC shape(1,320,192,3)\n im = Image.fromarray((im[0] * 255).astype('uint8'))\n # im = im.resize((192, 320), Image.ANTIALIAS)\n y = self.model.predict({'image': im}) # coordinates are xywh normalized\n if 'confidence' in y:\n box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels\n conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float)\n y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1)\n else:\n k = 'var_' + str(sorted(int(k.replace('var_', '')) for k in y)[-1]) # output key\n y = y[k] # output\n else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)\n im = im.permute(0, 2, 3, 1).cpu().numpy() # torch BCHW to numpy BHWC shape(1,320,192,3)\n if self.saved_model: # SavedModel\n y = (self.model(im, training=False) if self.keras else self.model(im)).numpy()\n elif self.pb: # GraphDef\n y = self.frozen_func(x=self.tf.constant(im)).numpy()\n else: # Lite or Edge TPU\n input, output = self.input_details[0], self.output_details[0]\n int8 = input['dtype'] == np.uint8 # is TFLite quantized uint8 model\n if int8:\n scale, zero_point = input['quantization']\n im = (im / scale + zero_point).astype(np.uint8) # de-scale\n self.interpreter.set_tensor(input['index'], im)\n self.interpreter.invoke()\n y = self.interpreter.get_tensor(output['index'])\n if int8:\n scale, zero_point = output['quantization']\n y = (y.astype(np.float32) - zero_point) * scale # re-scale\n y[..., :4] *= [w, h, w, h] # xywh normalized to pixels\n\n if isinstance(y, np.ndarray):\n y = torch.tensor(y, device=self.device)\n return (y, []) if val else y\n\n def warmup(self, imgsz=(1, 3, 640, 640)):\n # Warmup model by running inference once\n warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb\n if any(warmup_types) and self.device.type != 'cpu':\n im = torch.empty(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input\n for _ in range(2 if self.jit else 1): #\n self.forward(im) # warmup\n\n @staticmethod\n def _model_type(p='path/to/model.pt'):\n # Return model type from model path, i.e. path='path/to/model.onnx' -> type=onnx\n from export import export_formats\n suffixes = list(export_formats().Suffix) + ['.xml'] # export suffixes\n check_suffix(p, suffixes) # checks\n p = Path(p).name # eliminate trailing separators\n pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, xml2 = (s in p for s in suffixes)\n xml |= xml2 # *_openvino_model or *.xml\n tflite &= not edgetpu # *.tflite\n return pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs\n\n @staticmethod\n def _load_metadata(f='path/to/meta.yaml'):\n # Load metadata from meta.yaml if it exists\n d = yaml_load(f)\n return d['stride'], d['names'] # assign stride, names","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:07.509297Z","iopub.execute_input":"2023-07-15T13:37:07.510442Z","iopub.status.idle":"2023-07-15T13:37:07.602207Z","shell.execute_reply.started":"2023-07-15T13:37:07.510409Z","shell.execute_reply":"2023-07-15T13:37:07.601037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIZE = 512\ndef get_glomerulus_mask(masks, classes, mask_shape= (SIZE, SIZE)) -> np.ndarray:\n \"\"\" Converts glomerulus labels into boolean mask \"\"\"\n glomerulus_mask = np.zeros(shape=mask_shape, dtype=np.float64)\n c = 0\n for i, mask in enumerate(masks):\n if int(classes[i]) != 0: continue \n mask = np.reshape(mask, mask_shape)\n glomerulus_mask += mask\n c += 1\n if c > 0:\n glomerulus_mask = np.where(glomerulus_mask>0.4, 1, 0).astype(np.bool)\n else:\n glomerulus_mask = glomerulus_mask.astype(np.bool)\n glomerulus_mask = ~glomerulus_mask\n return glomerulus_mask.astype(bool)","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:07.603909Z","iopub.execute_input":"2023-07-15T13:37:07.604623Z","iopub.status.idle":"2023-07-15T13:37:07.61689Z","shell.execute_reply.started":"2023-07-15T13:37:07.604589Z","shell.execute_reply":"2023-07-15T13:37:07.61235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls /kaggle/input/maskdino-finetune/ ","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:07.619324Z","iopub.execute_input":"2023-07-15T13:37:07.620528Z","iopub.status.idle":"2023-07-15T13:37:08.576609Z","shell.execute_reply.started":"2023-07-15T13:37:07.620495Z","shell.execute_reply":"2023-07-15T13:37:08.575281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/opt/conda/lib/python3.10/site-packages/MultiScaleDeformableAttention-1.0-py3.10-linux-x86_64.egg')\n# sys.path.append('/kaggle/working/ensemble-boxes')\nfrom detectron2.config import get_cfg\nfrom detectron2.engine.defaults import DefaultPredictor\nfrom detectron2.projects.deeplab import add_deeplab_config\nfrom maskdino import add_maskdino_config\n# from ensemble_boxes import *\nimport os\n\ndef setup_cfg(config_file):\n # load config from file and command-line arguments\n cfg = get_cfg()\n add_deeplab_config(cfg)\n add_maskdino_config(cfg)\n cfg.merge_from_file(config_file)\n# cfg.freeze()\n return cfg\n\nRESNET50_MODELS_SIZE = [1440]\nresnet50_config_files = [\n \"/kaggle/input/resnet50-1440/config.yaml\"\n ]\n\nRESNET50_MODELS = []\n\nresnet50_best_model = [\n \"/kaggle/input/resnet50-1440/model_final.pth\",\n ]\n\nfor (b_m, model_size, config_file) in zip(resnet50_best_model, RESNET50_MODELS_SIZE, resnet50_config_files):\n model_name=b_m\n cfg = setup_cfg(config_file)\n cfg.MODEL.WEIGHTS = model_name\n cfg.INPUT.IMAGE_SIZE = model_size\n RESNET50_MODELS.append(DefaultPredictor(cfg))","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:08.581042Z","iopub.execute_input":"2023-07-15T13:37:08.581885Z","iopub.status.idle":"2023-07-15T13:37:59.215863Z","shell.execute_reply.started":"2023-07-15T13:37:08.581828Z","shell.execute_reply":"2023-07-15T13:37:59.214798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nfrom detectron2.data import detection_utils as utils\n\ndef ensemble_resnet50_preds(file_name, path, models):\n img = utils.read_image(f'{path}/{file_name}', format=\"BGR\")\n img_resize_1024 = cv2.resize(img, (1024, 1024))\n img_resize_1440 = cv2.resize(img, (1440, 1440))\n \n outputs = []\n \n ensemble_classes = []\n ensemble_scores = []\n ensemble_bboxes = []\n ensemble_masks = []\n flip = [0, 0, 0]\n \n for i, model in enumerate(models):\n for index, image_size in enumerate([1024, 1440]):\n total_box = 0\n if image_size == 1024:\n output = model(img_resize_1024)\n elif image_size == 1440:\n output = model(img_resize_1440)\n elif image_size == 512:\n if flip[index] == 0:\n output = model(img)\n elif flip[index] == 1:\n output = model(img_flip)\n\n pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n pred_scores = output['instances'].scores.cpu().numpy().tolist()\n pred_boxes = output['instances'].pred_boxes.tensor.cpu().numpy()\n pred_masks = output['instances'].pred_masks.cpu().numpy()\n classes = []\n scores = []\n bboxes = []\n masks = []\n for j in range(len(pred_classes)):\n pred_class = 1\n score = pred_scores[j]\n if score < 0.1:\n continue\n if flip[index] == 0:\n mask = np.squeeze(cv2.resize(np.expand_dims(pred_masks[j], 2),dsize=(SIZE,SIZE)))\n bbox = pred_boxes[j]/ image_size\n elif flip[index] == 1:\n mask = np.squeeze(cv2.flip(cv2.resize(np.expand_dims(pred_masks[j], 2),dsize=(SIZE,SIZE)), 1))\n bbox = pred_boxes[j]\n x1_old = image_size - bbox[2]\n x2_old = image_size - bbox[0]\n bbox[0] = x1_old\n bbox[2] = x2_old\n bbox = bbox / image_size\n w_mask = int((bbox[2] - bbox[0])*SIZE)\n h_mask = int((bbox[3] - bbox[1])*SIZE)\n x1 = max(0, int(bbox[0]*SIZE - w_mask/6))\n x2 = min(SIZE, int(bbox[2]*SIZE + w_mask/6))\n y1 = max(0, int(bbox[1]*SIZE - h_mask/6))\n y2 = min(SIZE, int(bbox[3]*SIZE + h_mask/6))\n\n masks.append((mask[y1:y2, x1:x2], [x1, y1, x2, y2]))\n classes.append(pred_class)\n scores.append(score)\n bboxes.append([bbox[0], bbox[1], \n bbox[2], bbox[3]])\n total_box += 1\n print(\"Resnet50 model \", i, \" image size \", image_size, \" flip \", flip[index], \" predict total box=\", total_box)\n ensemble_classes.append(classes)\n ensemble_scores.append(scores)\n ensemble_bboxes.append(bboxes)\n ensemble_masks.append(masks)\n \n return ensemble_classes, ensemble_scores, ensemble_bboxes, ensemble_masks","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MASKDINO_MODELS_SIZE = [1440, 1440, 1408, 1408\n# 512, 512, 512\n ]\n\nmaskdino_config_files = [\n \"/kaggle/input/internimage-1440-deduplicate/only_d2_config.yaml\",\n \"/kaggle/input/internimage-1440-deduplicate/fold0_config.yaml\",\n \"/kaggle/input/maskdino-swin-data1/config.yaml\",\n \"/kaggle/input/internimage-1440-deduplicate/fold2_config.yaml\",\n# \"/kaggle/input/internimage-1440-deduplicate/fold3_config.yaml\"\n ]\n\nDATA_PATH = \"/kaggle/input/hubmap-hacking-the-human-vasculature\"\nMASKDINO_MODELS = []\nSUBM_PATH = f'{DATA_PATH}/test'\n\nmaskdino_best_model = [\n \"/kaggle/input/internimage-1440-deduplicate/only_d2_model_0009074.pth\",\n \"/kaggle/input/internimage-1440-deduplicate/fold0_model_0009074.pth\",\n \"/kaggle/input/maskdino-swin-data1/model_0008469.pth\",\n \"/kaggle/input/internimage-1440-deduplicate/fold2_model_0006654.pth\",\n# \"/kaggle/input/internimage-1440-deduplicate/fold3_model_0002419.pth\"\n ]\n\nfor (b_m, model_size, config_file) in zip(maskdino_best_model, MASKDINO_MODELS_SIZE, maskdino_config_files):\n model_name=b_m\n# cfg = get_cfg()\n cfg = setup_cfg(config_file)\n cfg.MODEL.WEIGHTS = model_name\n cfg.INPUT.IMAGE_SIZE = model_size\n# cfg.MODEL.DEVICE = \"cuda:1\"\n# print(cfg)\n MASKDINO_MODELS.append(DefaultPredictor(cfg))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#LB: 0.451, 0.409 , 0.462, 0.401, 0.446\nyolov7_imgsz = [1024, 1024, 1024, 1024]\nyolov7_nclasses = [1, 1, 1, 1]\ndatas = ['/kaggle/working/hubmap-coco_1class.yaml', \n '/kaggle/working/hubmap-coco_1class.yaml', '/kaggle/working/hubmap-coco_1class.yaml', \n '/kaggle/working/hubmap-coco_1class.yaml']\nyolov7_model_weights = ['/kaggle/input/yolov7-hubmap-finetune/fold0_best.pt', \n '/kaggle/input/yolov7-hubmap-finetune/fold1_best.pt', \"/kaggle/input/yolov7-hubmap-finetune/fold2_best.pt\",\n \"/kaggle/input/yolov7-hubmap-finetune/fold3_best.pt\"]\ndevice = select_device('0')\nyolov7_models = []\nfor (yolov7_model_weight, data) in zip(yolov7_model_weights, datas): \n yolov7_model = DetectMultiBackend(yolov7_model_weight, device=device, dnn=False, data=data, fp16=False)\n yolov7_models.append(yolov7_model)\n# for i in range(len(yolov7_models)):\n# yolov7_models[i].warmup(imgsz=(1, 3, yolov7_imgsz[i], yolov7_imgsz[i])) \n","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:37:59.217241Z","iopub.execute_input":"2023-07-15T13:37:59.217789Z","iopub.status.idle":"2023-07-15T13:38:09.22439Z","shell.execute_reply.started":"2023-07-15T13:37:59.217754Z","shell.execute_reply":"2023-07-15T13:38:09.223368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ensemble_yolov7_preds(\n file_name,\n path, models,\n device='cuda:0', # cuda device, i.e. 0 or 0,1,2,3 or cpu\n ):\n\n im0 = cv2.imread(f'{path}/{file_name}') # BGR\n im_512 = cv2.resize(im0, (512, 512))\n im_512 = im_512.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im_512 = np.ascontiguousarray(im_512)\n im_512 = torch.from_numpy(im_512).to(device)\n im_512 = im_512.float() # uint8 to fp16/32\n im_512 /= 255 # 0 - 255 to 0.0 - 1.0\n im_512 = im_512[None]\n\n im_1024 = cv2.resize(im0, (1024, 1024))\n im_1024 = im_1024.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im_1024 = np.ascontiguousarray(im_1024)\n im_1024 = torch.from_numpy(im_1024).to(device)\n im_1024 = im_1024.float() # uint8 to fp16/32\n im_1024 /= 255 # 0 - 255 to 0.0 - 1.0\n im_1024 = im_1024[None]\n\n im_1536 = cv2.resize(im0, (1536, 1536))\n im_1536 = im_1536.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im_1536 = np.ascontiguousarray(im_1536)\n im_1536 = torch.from_numpy(im_1536).to(device)\n im_1536 = im_1536.float() # uint8 to fp16/32\n im_1536 /= 255 # 0 - 255 to 0.0 - 1.0\n im_1536 = im_1536[None]\n\n ensemble_classes = []\n ensemble_scores = []\n ensemble_bboxes = []\n ensemble_masks = []\n for i, model in enumerate(models):\n for image_size in [512, 1024]:\n for flip in [\"no\"]:\n total_box = 0\n if image_size == 512:\n pred, out = model(im_512, augment=False, visualize=False)\n elif image_size == 1024:\n pred, out = model(im_1024, augment=False, visualize=False)\n elif image_size == 1536:\n pred, out = model(im_1536, augment=False, visualize=False)\n\n proto = out[1]\n if yolov7_nclasses[i] == 2:\n pred = non_max_suppression(pred, conf_thres=0.1, iou_thres=0.6, classes=1, agnostic=False, max_det=100, nm=32)\n elif yolov7_nclasses[i] == 1:\n pred = non_max_suppression(pred, conf_thres=0.1, iou_thres=0.6, classes=0, agnostic=False, max_det=100, nm=32)\n masks = []\n scores = []\n bboxes = []\n classes = []\n for j, det in enumerate(pred): # per image\n if len(det):\n pred_masks = process_mask(proto[j], det[:, 6:], det[:, :4], (image_size, image_size), upsample=True) # HWC\n bboxs = det[:, :4]\n confs = det[:, 4]\n clasf = det[:, 5]\n # mask_img = 0\n for mask, confidence, classification, bbox in zip(pred_masks, confs, clasf, bboxs):\n if flip == \"no\":\n bbox = bbox.cpu().numpy()\n bbox = bbox / image_size\n binary_mask = mask.cpu().numpy()\n binary_mask = np.squeeze(cv2.resize(np.expand_dims(binary_mask, 2),dsize=(SIZE,SIZE)))\n elif flip == \"yes\":\n bbox = bbox.cpu().numpy()\n x1_old = image_size - bbox[2]\n x2_old = image_size - bbox[0]\n bbox[0] = x1_old\n bbox[2] = x2_old\n bbox = bbox / image_size\n binary_mask = mask.cpu().numpy()\n binary_mask = np.squeeze(cv2.flip(cv2.resize(np.expand_dims(binary_mask, 2),dsize=(SIZE,SIZE)), 1))\n \n w_mask = int((bbox[2] - bbox[0])*SIZE)\n h_mask = int((bbox[3] - bbox[1])*SIZE)\n x1 = max(0, int(bbox[0]*SIZE - w_mask/6))\n x2 = min(SIZE, int(bbox[2]*SIZE + w_mask/6))\n y1 = max(0, int(bbox[1]*SIZE - h_mask/6))\n y2 = min(SIZE, int(bbox[3]*SIZE + h_mask/6))\n bboxes.append([bbox[0], bbox[1], bbox[2], bbox[3]])\n masks.append((binary_mask[y1:y2, x1:x2], [x1, y1, x2, y2]))\n scores.append(confidence.item())\n if yolov7_nclasses[i] == 2:\n classes.append(int(classification))\n elif yolov7_nclasses[i] == 1:\n classes.append(1)\n total_box += 1\n ensemble_classes.append(classes)\n ensemble_scores.append(scores)\n ensemble_bboxes.append(bboxes)\n ensemble_masks.append(masks)\n print(\"Yolov7 model \", i, \" image size \", image_size, \" hflip \", flip, \" predict total box=\", total_box)\n del im_512\n del im_1024\n del im_1536\n \n return ensemble_classes, ensemble_scores, ensemble_bboxes, ensemble_masks","metadata":{"execution":{"iopub.status.busy":"2023-07-16T08:29:34.42223Z","iopub.execute_input":"2023-07-16T08:29:34.422781Z","iopub.status.idle":"2023-07-16T08:29:34.458215Z","shell.execute_reply.started":"2023-07-16T08:29:34.42275Z","shell.execute_reply":"2023-07-16T08:29:34.456987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:38:09.258025Z","iopub.execute_input":"2023-07-15T13:38:09.258452Z","iopub.status.idle":"2023-07-15T13:38:11.911492Z","shell.execute_reply.started":"2023-07-15T13:38:09.258377Z","shell.execute_reply":"2023-07-15T13:38:11.910503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"/opt/conda/lib/python3.10/site-packages/DCNv3-1.0-py3.10-linux-x86_64.egg\")\nsys.path.append(\"/kaggle/input/internimage-sourcecode\")\n\nfrom mmdet.apis import init_detector, inference_detector,show_result_pyplot, set_random_seed\nimport mmcv_custom # noqa: F401,F403\nimport mmdet_custom # noqa: F401,F403\n\n# # #check file her\n\nfrom mmcv import Config\n\n# LB: \nintern_config_files = [\n '/kaggle/input/internimage-large-1536/hubmap_config_0.py',\n# '/kaggle/input/internimage-large-1536/hubmap_config_1.py',\n \"/kaggle/input/internimage-wsi2/wsi2_config_0.py\",\n '/kaggle/input/internimage-large-1536/hubmap_config_2.py',\n '/kaggle/input/internimage-large-1536/hubmap_config_3.py'\n ]\nintern_weight_files = [\n '/kaggle/input/internimage-large-1536/fold0_best_bbox_mAP_epoch_7.pth',\n# '/kaggle/input/internimage-large-1536/fold1_best_bbox_mAP_epoch_3.pth',\n \"/kaggle/input/internimage-wsi2/wsi2_best_bbox_mAP_epoch_6.pth\",\n '/kaggle/input/internimage-large-1536/fold2_best_bbox_mAP_epoch_7.pth',\n '/kaggle/input/internimage-large-1536/fold3_best_bbox_mAP_epoch_3.pth'\n]\nintern_image_sizes = [[(1440, 1440)],\n [(1440, 1440)],\n [(1440, 1440)],\n [(1440, 1440)]\n ]\ndevice = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\nintern_models = []\nfor weight_file, config_file, image_size in zip(intern_weight_files, intern_config_files, intern_image_sizes):\n cfg = Config.fromfile(config_file)\n cfg.data.test.pipeline[1].img_scale= image_size\n cfg.model.test_cfg.rcnn.nms.iou_threshold=0.6\n cfg.model.test_cfg.rcnn.score_thr=0.05\n cfg.model.test_cfg.rcnn.mask_thr_binary=0.5\n cfg.seed = 42\n set_random_seed(42, deterministic=True)\n model = init_detector(cfg, weight_file, device=device)\n intern_models.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:38:11.914106Z","iopub.execute_input":"2023-07-15T13:38:11.914739Z","iopub.status.idle":"2023-07-15T13:39:31.05661Z","shell.execute_reply.started":"2023-07-15T13:38:11.914704Z","shell.execute_reply":"2023-07-15T13:39:31.055351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Bbox: x1, x2, y1, y2. W, H = (512, 512)\n# Mask: True, False. w, h = (512, 512)\ndef ensemble_internimage_preds(file_name, path, models):\n outputs = []\n ensemble_classes = []\n ensemble_scores = []\n ensemble_bboxes = []\n ensemble_masks = []\n \n for i, model in enumerate(models):\n pred = inference_detector(model, f'{path}/{file_name}')\n pred_class = pred[0]\n pred_mask = pred[1]\n\n classes = []\n scores = []\n bboxes = []\n masks = []\n total_box = 0\n for j, classe in enumerate(pred_class):\n if(j==0):\n bbs = classe\n sgs = pred_mask[j]\n for bb, sg in zip(bbs,sgs):\n bbox = bb[:4]/SIZE\n cnf = bb[4]\n sg = sg.astype(np.uint8)\n pred_class = 1\n score = cnf\n \n mask = sg\n w_mask = int((bbox[2] - bbox[0])*SIZE)\n h_mask = int((bbox[3] - bbox[1])*SIZE)\n x1 = max(0, int(bbox[0]*SIZE - w_mask/6))\n x2 = min(SIZE, int(bbox[2]*SIZE + w_mask/6))\n y1 = max(0, int(bbox[1]*SIZE - h_mask/6))\n y2 = min(SIZE, int(bbox[3]*SIZE + h_mask/6))\n \n classes.append(pred_class)\n scores.append(score)\n bboxes.append([bbox[0], bbox[1], \n bbox[2], bbox[3]])\n masks.append((mask[y1:y2, x1:x2], [x1, y1, x2, y2]))\n total_box += 1\n ensemble_classes.append(classes)\n ensemble_scores.append(scores)\n ensemble_bboxes.append(bboxes)\n ensemble_masks.append(masks)\n print(\"Internimage model \", i, \" predict total box=\", total_box)\n \n return ensemble_classes, ensemble_scores, ensemble_bboxes, ensemble_masks","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:39:31.058927Z","iopub.execute_input":"2023-07-15T13:39:31.059353Z","iopub.status.idle":"2023-07-15T13:39:31.076936Z","shell.execute_reply.started":"2023-07-15T13:39:31.059312Z","shell.execute_reply":"2023-07-15T13:39:31.075441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nTHSS = [[.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1], [.5, .1]]\nMASKDINO_N_CLASSES = [1, 1, 1, 1, 1]\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nfrom detectron2.data import detection_utils as utils\n\n# MIN_PIXELS = [60, 60]\n# Bbox: x1, x2, y1, y2. W, H = (1024, 1024)\n# Mask: 1, 0. w, h = (1024, 1024)\ndef ensemble_maskdino_preds(file_name, path, models, ths):\n img = utils.read_image(f'{path}/{file_name}', format=\"BGR\")\n img_flip = cv2.flip(img, 1)\n img_resize_1024 = cv2.resize(img, (1024, 1024))\n img_resize_1408 = cv2.resize(img, (1408, 1408))\n img_resize_1440 = cv2.resize(img, (1440, 1440))\n# img = cv2.resize(img, (SIZE, SIZE), interpolation=cv2.INTER_CUBIC)\n \n outputs = []\n \n ensemble_classes = []\n ensemble_scores = []\n ensemble_bboxes = []\n ensemble_masks = []\n flip = [0, 0, 0]\n \n for i, model in enumerate(models):\n image_sizes = [[512, 1024, 1440], [512, 1024, 1440], [512, 1024, 1408], [512, 1024, 1408]]\n for index, image_size in enumerate(image_sizes[i]):\n# if image_size == 1440 and i > 2:\n# continue\n# if image_size == 512 and i <= 2:\n# continue\n total_box = 0\n if image_size == 1024:\n output = model(img_resize_1024)\n elif image_size == 1440:\n output = model(img_resize_1440)\n elif image_size == 1408:\n output = model(img_resize_1408)\n elif image_size == 512:\n if flip[index] == 0:\n output = model(img)\n elif flip[index] == 1:\n output = model(img_flip)\n\n pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n pred_scores = output['instances'].scores.cpu().numpy().tolist()\n pred_boxes = output['instances'].pred_boxes.tensor.cpu().numpy()\n pred_masks = output['instances'].pred_masks.cpu().numpy()\n classes = []\n scores = []\n bboxes = []\n masks = []\n for j in range(len(pred_classes)):\n if MASKDINO_N_CLASSES[i] == 2:\n pred_class = pred_classes[j]\n elif MASKDINO_N_CLASSES[i] == 1:\n pred_class = 1\n score = pred_scores[j]\n if i == 2 and image_size == 1408:\n if score < 0.01:\n continue\n else:\n if score < ths[i][pred_class]:\n continue\n if flip[index] == 0:\n mask = np.squeeze(cv2.resize(np.expand_dims(pred_masks[j], 2),dsize=(SIZE,SIZE)))\n bbox = pred_boxes[j]/ image_size\n elif flip[index] == 1:\n mask = np.squeeze(cv2.flip(cv2.resize(np.expand_dims(pred_masks[j], 2),dsize=(SIZE,SIZE)), 1))\n bbox = pred_boxes[j]\n x1_old = image_size - bbox[2]\n x2_old = image_size - bbox[0]\n bbox[0] = x1_old\n bbox[2] = x2_old\n bbox = bbox / image_size\n w_mask = int((bbox[2] - bbox[0])*SIZE)\n h_mask = int((bbox[3] - bbox[1])*SIZE)\n x1 = max(0, int(bbox[0]*SIZE - w_mask/6))\n x2 = min(SIZE, int(bbox[2]*SIZE + w_mask/6))\n y1 = max(0, int(bbox[1]*SIZE - h_mask/6))\n y2 = min(SIZE, int(bbox[3]*SIZE + h_mask/6))\n\n masks.append((mask[y1:y2, x1:x2], [x1, y1, x2, y2]))\n classes.append(pred_class)\n scores.append(score)\n bboxes.append([bbox[0], bbox[1], \n bbox[2], bbox[3]])\n total_box += 1\n print(\"Mask dino model \", i, \" image size \", image_size, \" flip \", flip[index], \" predict total box=\", total_box)\n ensemble_classes.append(classes)\n ensemble_scores.append(scores)\n ensemble_bboxes.append(bboxes)\n ensemble_masks.append(masks)\n \n return ensemble_classes, ensemble_scores, ensemble_bboxes, ensemble_masks","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:54:09.273368Z","iopub.execute_input":"2023-07-15T13:54:09.27378Z","iopub.status.idle":"2023-07-15T13:54:09.298903Z","shell.execute_reply.started":"2023-07-15T13:54:09.273749Z","shell.execute_reply":"2023-07-15T13:54:09.297921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile hubmap_worker.py\n\nSIZE = 512\nimport warnings\nimport numpy as np\nimport pycocotools\nimport pycocotools.mask\nimport time\nimport base64\nfrom pycocotools import _mask as coco_mask\nfrom typing import Text, Dict, Tuple\nimport zlib\nimport cv2\nimport pandas as pd\nimport typing as t\nimport matplotlib.pyplot as plt\nimport json\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n # check input mask --\n if mask.dtype != bool:\n raise ValueError(\n \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n mask.dtype)\n\n mask = np.squeeze(mask)\n if len(mask.shape) != 2:\n raise ValueError(\n \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n mask.shape)\n\n # convert input mask to expected COCO API input --\n mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n mask_to_encode = mask_to_encode.astype(np.uint8)\n mask_to_encode = np.asfortranarray(mask_to_encode)\n\n # RLE encode mask --\n encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n # compress and base64 encoding --\n binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n base64_str = base64.b64encode(binary_str)\n return base64_str\n\ndef coordinates_to_masks(coordinates, shape):\n masks = []\n for coord in coordinates:\n mask = np.zeros(shape, dtype=np.uint8)\n cv2.fillPoly(mask, [np.array(coord)], 1)\n masks.append(mask)\n return masks\n\ndef prefilter_boxes(boxes, scores, labels, weights, thr):\n # Create dict with boxes stored by its label\n new_boxes = dict()\n total_box = 0\n for t in range(len(boxes)):\n\n if len(boxes[t]) != len(scores[t]):\n print('Error. Length of boxes arrays not equal to length of scores array: {} != {}'.format(len(boxes[t]), len(scores[t])))\n exit()\n\n if len(boxes[t]) != len(labels[t]):\n print('Error. Length of boxes arrays not equal to length of labels array: {} != {}'.format(len(boxes[t]), len(labels[t])))\n exit()\n\n for j in range(len(boxes[t])):\n score = scores[t][j]\n if score < thr:\n continue\n label = int(labels[t][j])\n box_part = boxes[t][j]\n x1 = float(box_part[0])\n y1 = float(box_part[1])\n x2 = float(box_part[2])\n y2 = float(box_part[3])\n\n # Box data checks\n if x2 < x1:\n warnings.warn('X2 < X1 value in box. Swap them.')\n x1, x2 = x2, x1\n if y2 < y1:\n warnings.warn('Y2 < Y1 value in box. Swap them.')\n y1, y2 = y2, y1\n if x1 < 0:\n warnings.warn('X1 < 0 in box. Set it to 0.')\n x1 = 0\n if x1 > 1:\n warnings.warn('X1 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n x1 = 1\n if x2 < 0:\n warnings.warn('X2 < 0 in box. Set it to 0.')\n x2 = 0\n if x2 > 1:\n warnings.warn('X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n x2 = 1\n if y1 < 0:\n warnings.warn('Y1 < 0 in box. Set it to 0.')\n y1 = 0\n if y1 > 1:\n warnings.warn('Y1 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n y1 = 1\n if y2 < 0:\n warnings.warn('Y2 < 0 in box. Set it to 0.')\n y2 = 0\n if y2 > 1:\n warnings.warn('Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n y2 = 1\n if (x2 - x1) * (y2 - y1) == 0.0:\n warnings.warn(\"Zero area box skipped: {}.\".format(box_part))\n continue\n\n # [label, score, weight, model index, x1, y1, x2, y2, bbox index]\n b = np.zeros(8 + 1)\n b[0] = int(label)\n b[1] = float(score) * weights[t]\n b[2] = weights[t] \n b[3] = t\n b[4] = x1\n b[5] = y1\n b[6] = x2\n b[7] = y2\n b[8] = j\n \n if label not in new_boxes:\n new_boxes[label] = []\n new_boxes[label].append(b)\n total_box += 1\n\n # Sort each list in dict by score and transform it to numpy array\n for k in new_boxes:\n print(\"HEHE: \", len(new_boxes[k]))\n current_boxes = np.array(new_boxes[k])\n new_boxes[k] = current_boxes[current_boxes[:, 1].argsort()[::-1]]\n# for i in range(new_boxes[1].shape[0]):\n# print(new_boxes[1][i][3], new_boxes[1][i][8])\n print(\"After filter box, total box = \", total_box)\n return new_boxes\n\n\ndef get_weighted_box(boxes, conf_type='avg'):\n \"\"\"\n Create weighted box for set of boxes\n :param boxes: set of boxes to fuse\n :param conf_type: type of confidence one of 'avg' or 'max'\n :return: weighted box (label, score, weight, model index, x1, y1, x2, y2)\n \"\"\"\n\n box = np.zeros(8 + 1, dtype=np.float32)\n conf = 0\n conf_list = []\n w = 0\n for b in boxes:\n box[4:8] += (b[1] * b[4:8])\n conf += b[1]\n conf_list.append(b[1])\n w += b[2]\n box[0] = boxes[0][0]\n if conf_type in ('avg', 'box_and_model_avg', 'absent_model_aware_avg'):\n box[1] = conf / len(boxes)\n elif conf_type == 'max':\n box[1] = np.array(conf_list).max()\n box[2] = w\n box[3] = -1 # model index field is retained for consistency but is not used.\n box[4:8] /= conf\n box[8] = -1\n return box\n\n\ndef find_matching_box_fast(boxes_list, new_box, match_iou):\n \"\"\"\n Reimplementation of find_matching_box with numpy instead of loops. Gives significant speed up for larger arrays\n (~100x). This was previously the bottleneck since the function is called for every entry in the array.\n \"\"\"\n def bb_iou_array(boxes, new_box):\n # bb interesection over union\n xA = np.maximum(boxes[:, 0], new_box[0])\n yA = np.maximum(boxes[:, 1], new_box[1])\n xB = np.minimum(boxes[:, 2], new_box[2])\n yB = np.minimum(boxes[:, 3], new_box[3])\n\n interArea = np.maximum(xB - xA, 0) * np.maximum(yB - yA, 0)\n\n # compute the area of both the prediction and ground-truth rectangles\n boxAArea = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n boxBArea = (new_box[2] - new_box[0]) * (new_box[3] - new_box[1])\n\n iou = interArea / (boxAArea + boxBArea - interArea)\n\n return iou\n\n if boxes_list.shape[0] == 0:\n return -1, match_iou\n\n # boxes = np.array(boxes_list)\n boxes = boxes_list\n\n ious = bb_iou_array(boxes[:, 4:8], new_box[4:8])\n\n ious[boxes[:, 0] != new_box[0]] = -1\n\n best_idx = np.argmax(ious)\n best_iou = ious[best_idx]\n\n if best_iou <= match_iou:\n best_iou = match_iou\n best_idx = -1\n\n return best_idx, best_iou\n\ndef weighted_boxes_fusion(\n boxes_list,\n scores_list,\n labels_list,\n weights=None,\n iou_thr=0.55,\n skip_box_thr=0.0,\n conf_type='avg',\n allows_overflow=False\n):\n '''\n :param boxes_list: list of boxes predictions from each model, each box is 4 numbers.\n It has 3 dimensions (models_number, model_preds, 4)\n Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1]\n :param scores_list: list of scores for each model\n :param labels_list: list of labels for each model\n :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n :param iou_thr: IoU value for boxes to be a match\n :param skip_box_thr: exclude boxes with score lower than this variable\n :param conf_type: how to calculate confidence in weighted boxes.\n 'avg': average value,\n 'max': maximum value,\n 'box_and_model_avg': box and model wise hybrid weighted average,\n 'absent_model_aware_avg': weighted average that takes into account the absent model.\n :param allows_overflow: false if we want confidence score not exceed 1.0\n\n :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2).\n :return: scores: confidence scores\n :return: labels: boxes labels\n '''\n\n if weights is None:\n weights = np.ones(len(boxes_list))\n if len(weights) != len(boxes_list):\n print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights), len(boxes_list)))\n weights = np.ones(len(boxes_list))\n weights = np.array(weights)\n\n if conf_type not in ['avg', 'max', 'box_and_model_avg', 'absent_model_aware_avg']:\n print('Unknown conf_type: {}. Must be \"avg\", \"max\" or \"box_and_model_avg\", or \"absent_model_aware_avg\"'.format(conf_type))\n exit()\n# t1_s = time.time()\n filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr)\n# t1_e = time.time()\n# print(\"Prefilter: \", t1_e - t1_s)\n if len(filtered_boxes) == 0:\n return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,)), []\n\n overall_boxes = []\n trace_boxes = []\n for label in filtered_boxes:\n boxes = filtered_boxes[label]\n new_boxes = []\n weighted_boxes = np.empty((0, 8 + 1))\n\n # Clusterize boxes\n for j in range(0, len(boxes)):\n# t2_s = time.time()\n index, best_iou = find_matching_box_fast(weighted_boxes, boxes[j], iou_thr)\n# t2_e = time.time()\n# print(\"find_matching_box_fast: \", t2_e - t2_s)\n# t3_s = time.time()\n if index != -1:\n# print(\"Append box: \", boxes[j][3], boxes[j][8], \" to \", index)\n new_boxes[index].append(boxes[j])\n weighted_boxes[index] = get_weighted_box(new_boxes[index], conf_type)\n else:\n# print(\"Create new box: \", len(new_boxes), boxes[j][3], boxes[j][8])\n new_boxes.append([boxes[j].copy()])\n weighted_boxes = np.vstack((weighted_boxes, boxes[j].copy()))\n# t3_e = time.time()\n# print(\"weighted_boxes: \", t3_e - t3_s)\n\n # Rescale confidence based on number of models and boxes\n \n for i in range(len(new_boxes)):\n clustered_boxes = new_boxes[i]\n trace_box = []\n for j in range(len(clustered_boxes)):\n trace_box.append((clustered_boxes[j][3], clustered_boxes[j][8], clustered_boxes[j][1]))\n if conf_type == 'box_and_model_avg':\n clustered_boxes = np.array(clustered_boxes)\n # weighted average for boxes\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / weighted_boxes[i, 2]\n # identify unique model index by model index column\n _, idx = np.unique(clustered_boxes[:, 3], return_index=True)\n # rescale by unique model weights\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * clustered_boxes[idx, 2].sum() / weights.sum()\n elif conf_type == 'absent_model_aware_avg':\n clustered_boxes = np.array(clustered_boxes)\n # get unique model index in the cluster\n models = np.unique(clustered_boxes[:, 3]).astype(int)\n # create a mask to get unused model weights\n mask = np.ones(len(weights), dtype=bool)\n mask[models] = False\n # absent model aware weighted average\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / (weighted_boxes[i, 2] + weights[mask].sum())\n elif conf_type == 'max':\n weighted_boxes[i, 1] = weighted_boxes[i, 1] / weights.max()\n elif not allows_overflow:\n if len(trace_boxes) == 1 and weighted_boxes[i, 1] >= 0.3:\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * 1 * min(len(weights), len(clustered_boxes)) / weights.sum()\n else:\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * min(len(weights), len(clustered_boxes)) / weights.sum()\n# print(\"Weight box \", i , \" - trace box \", len(trace_boxes))\n weighted_boxes[i, 8] = len(trace_boxes)\n else:\n weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / weights.sum()\n \n# print(\"Box \", i, \" has: \", trace_box)\n trace_boxes.append(trace_box)\n overall_boxes.append(weighted_boxes)\n overall_boxes = np.concatenate(overall_boxes, axis=0)\n overall_boxes = overall_boxes[overall_boxes[:, 1].argsort()[::-1]]\n masks = []\n total_box = 0\n for i in range(overall_boxes.shape[0]):\n masks.append(trace_boxes[int(overall_boxes[i][8].item())])\n print(i, masks[i], \"Score=\", overall_boxes[i, 1], \", SIZE=\", (overall_boxes[i][6]-overall_boxes[i][4])*(overall_boxes[i][7]-overall_boxes[i][5])*SIZE*SIZE)\n total_box += len(masks[i])\n boxes = overall_boxes[:, 4:8]\n scores = overall_boxes[:, 1]\n labels = overall_boxes[:, 0]\n print(\"After NMS, total box = \", total_box)\n return boxes, scores, labels, masks\n\ndef find_matching_mask_fast(rle_masks, rle_new_mask, match_iou):\n if len(rle_masks) == 0:\n return -1, match_iou\n rle_masks = [rle[0] for rle in rle_masks]\n rle_new_mask = [rle_new_mask[0]]\n \n ious = pycocotools.mask.iou(rle_new_mask, rle_masks, [0] * len(rle_masks))[0]\n\n# ious[masks_list[:, 0] != new_mask[0]] = -1\n\n best_idx = np.argmax(ious)\n best_iou = ious[best_idx]\n\n if best_iou <= match_iou:\n# best_iou = match_iou\n best_idx = -1\n print(best_idx, best_iou)\n\n return best_idx, best_iou\n\ndef nms_masks(\n rle_list,\n iou_thr=0.6,\n skip_box_thr=0.0,\n):\n new_masks = []\n total_mask = 0\n for j in range(0, len(rle_list)):\n index, best_iou = find_matching_mask_fast(new_masks, rle_list[j], iou_thr)\n \n if index != -1:\n pass\n else:\n new_masks.append(rle_list[j])\n total_mask += 1\n\n print(\"After NMS mask, total mask\", total_mask)\n return new_masks\n\ndef nms_predictions(ensemble_classes, ensemble_scores, ensemble_bboxes, ensemble_masks, weights,\n iou_th=.5):\n nms_bboxes, nms_scores, nms_classes, nms_mask_idx = weighted_boxes_fusion(\n ensemble_bboxes, \n ensemble_scores, \n ensemble_classes, \n weights=weights,\n iou_thr=iou_th,\n skip_box_thr=0.0001,\n )\n\n return nms_bboxes, nms_classes, nms_scores, nms_mask_idx\n\n\ndef nms(input_q, output_q):\n while True:\n inp = input_q.get()\n if inp is None:\n output_q.put(None)\n break\n classes, scores, bboxes, masks, weights, test_name, iou_th = inp\n bboxes, classes, scores, nms_mask_idx = nms_predictions(\n classes, \n scores, \n bboxes, \n masks, \n weights,\n iou_th=iou_th\n )\n new_masks = []\n for i in range(len(classes)):\n x_min = 10000\n y_min = 10000\n x_max = 0\n y_max = 0\n for j in range(len(nms_mask_idx[i])):\n model_idx = int(nms_mask_idx[i][j][0].item())\n mask_idx = int(nms_mask_idx[i][j][1].item())\n x1, y1, x2, y2 = masks[model_idx][mask_idx][1]\n if x1 < x_min:\n x_min = x1\n if x2 > x_max:\n x_max = x2\n if y1 < y_min:\n y_min = y1\n if y2 > y_max:\n y_max = y2\n ensemble_mask = np.zeros((y_max-y_min, x_max-x_min))\n for j in range(len(nms_mask_idx[i])):\n model_idx = int(nms_mask_idx[i][j][0].item())\n mask_idx = int(nms_mask_idx[i][j][1].item())\n x1, y1, x2, y2 = masks[model_idx][mask_idx][1]\n mask = np.zeros((y_max-y_min, x_max-x_min))\n mask[y1-y_min: y2-y_min, x1-x_min: x2-x_min] = masks[model_idx][mask_idx][0]\n ensemble_mask = ensemble_mask + mask\n \n ensemble_mask = ensemble_mask / len(nms_mask_idx[i])\n result_mask = np.zeros((SIZE, SIZE))\n result_mask[y_min:y_max, x_min:x_max] = ensemble_mask\n \n dataset1_models = [6, 7, 8, 23]\n if len(nms_mask_idx[i]) == 1 and (int(nms_mask_idx[i][j][0].item()) in dataset1_models):\n new_masks.append((result_mask, \"not_dialate\"))\n else:\n new_masks.append((result_mask, \"dialate\"))\n del masks\n while not output_q.empty():\n time.sleep(0.2)\n output_q.put((classes, scores, bboxes, new_masks, test_name))\n\ndef mask_to_rle(mask: np.ndarray):\n # check input mask --\n if mask.dtype != bool:\n raise ValueError(\n \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n mask.dtype)\n\n mask = np.squeeze(mask)\n if len(mask.shape) != 2:\n raise ValueError(\n \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n mask.shape)\n\n # convert input mask to expected COCO API input --\n mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n mask_to_encode = mask_to_encode.astype(np.uint8)\n mask_to_encode = np.asfortranarray(mask_to_encode)\n\n # RLE encode mask --\n rle = coco_mask.encode(mask_to_encode)[0]\n return rle\n\ndef post(input_q):\n jsonl_file_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\"\n glomerulus_masks = {}\n with open(jsonl_file_path, \"r\") as file:\n for line in file:\n item = json.loads(line)\n image_id = item[\"id\"]\n glomerulus_mask = np.zeros((512, 512))\n for annotation in item[\"annotations\"]:\n if annotation[\"type\"] != 'glomerulus':\n continue\n segmentation = annotation[\"coordinates\"]\n mask_img = coordinates_to_masks(segmentation, (512, 512))[0]\n glomerulus_mask += mask_img\n glomerulus_masks[image_id] = np.array(glomerulus_mask, dtype=bool)\n \n ids = []\n heights = []\n widths = []\n prediction_strings = []\n \n while True:\n inp = input_q.get()\n if inp is None:\n break\n classes, scores, bboxes, masks, test_name = inp\n img = cv2.imread(f'/kaggle/input/hubmap-hacking-the-human-vasculature/test/{test_name}')\n test_image_id = test_name.replace(\".tif\", \"\")\n pred_string = \"\"\n rles = []\n for i, mask_tuple in enumerate(masks):\n if int(classes[i]) != 1: continue\n if test_image_id in glomerulus_masks.keys():\n glomerulus_mask = glomerulus_masks[test_image_id]\n else:\n glomerulus_mask = np.zeros((512, 512)).astype(np.bool)\n \n mask = np.reshape(mask_tuple[0], (SIZE, SIZE))\n mask = np.where(mask>0.6, 1, 0).astype(np.bool)\n mask = np.array(mask, dtype=np.uint8)\n bbox = bboxes[i]\n score = scores[i]\n\n area = int(np.sum(mask))\n\n if area <= 0:\n continue\n if area < 60*SIZE/512:\n continue\n if area < 1000:\n kernel = np.ones(shape=(3, 3), dtype=np.uint8)\n elif area < 2000:\n kernel = np.ones(shape=(3, 3), dtype=np.uint8)\n elif area < 4000:\n kernel = np.ones(shape=(5, 5), dtype=np.uint8)\n elif area < 8000:\n kernel = np.ones(shape=(6, 6), dtype=np.uint8)\n elif area < 16000:\n kernel = np.ones(shape=(7, 7), dtype=np.uint8)\n elif area < 32000:\n kernel = np.ones(shape=(8, 8), dtype=np.uint8)\n else:\n kernel = np.ones(shape=(9, 9), dtype=np.uint8)\n #if mask_tuple[1] == \"dialate\":\n #mask = cv2.dilate(mask, kernel, iterations=2)\n \n mask = np.array(mask, dtype=bool)\n intersect_mask = mask & glomerulus_mask\n intersect_mask = np.array(intersect_mask, dtype=np.uint8)\n intersect_area = int(np.sum(intersect_mask))\n if intersect_area/area > 0.5:\n continue\n \n rle = mask_to_rle(mask)\n rles.append((rle, score, bbox, i))\n \n def myFunc(e):\n return e[1]\n rles.sort(reverse=True, key=myFunc)\n nms_rles = rles\n# nms_rles = nms_masks(rles)\n# mask_img = 0\n t = 0\n\n for i, rle in enumerate(nms_rles):\n encoded_mask = rle[0][\"counts\"]\n bbox = rle[2]\n# mask = masks[rle[3]]\n# mask_img += mask\n# img = cv2.rectangle(img, [int(bbox[0]*SIZE), int(bbox[1]*SIZE)], [int(bbox[2]*SIZE), int(bbox[3]*SIZE)], (255, 0, 0), 1)\n # compress and base64 encoding --\n binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n base64_str = base64.b64encode(binary_str)\n if t == 0:\n pred_string += f\"0 {rle[1]} {base64_str.decode('utf-8')}\"\n else:\n pred_string += f\" 0 {rle[1]} {base64_str.decode('utf-8')}\"\n t += 1\n\n ids.append(test_name.split('.')[0])\n heights.append(512)\n widths.append(512)\n prediction_strings.append(pred_string)\n# plt.imshow(mask_img)\n# plt.show()\n# plt.savefig('/kaggle/working/mask.png')\n print(\"Total mask: \", t)\n \n submission = pd.DataFrame()\n submission['id'] = ids\n submission['height'] = heights\n submission['width'] = widths\n submission['prediction_string'] = prediction_strings\n submission = submission.set_index('id')\n submission.to_csv(\"/kaggle/working/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:47:11.100366Z","iopub.execute_input":"2023-07-15T13:47:11.100767Z","iopub.status.idle":"2023-07-15T13:47:11.123465Z","shell.execute_reply.started":"2023-07-15T13:47:11.100734Z","shell.execute_reply":"2023-07-15T13:47:11.122404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nIOU_THRES = 0.6\nimport multiprocessing as mp\nimport time\nfrom hubmap_worker import nms, post\nif __name__ == '__main__':\n DATA_PATH = \"/kaggle/input/hubmap-hacking-the-human-vasculature\"\n SUBM_PATH = f'{DATA_PATH}/test'\n mp.set_start_method('spawn', force=True)\n# test_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train\"\n sample_submission = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv')\n\n test_names = os.listdir(SUBM_PATH)\n\n nms_in_q = mp.Queue()\n nms_out_q = mp.Queue()\n p_nms = mp.Process(target=nms, args=(nms_in_q,nms_out_q))\n p_nms.start()\n\n p_post = mp.Process(target=post, args=(nms_out_q,))\n p_post.start()\n\n for test_name in test_names:\n # img = cv2.imread(f'{SUBM_PATH}/{test_name}')\n h, w = SIZE, SIZE\n t1_s = time.time()\n maskdino_classes, maskdino_scores, maskdino_bboxes, maskdino_masks = ensemble_maskdino_preds(\n file_name=test_name, \n path=SUBM_PATH, \n models=MASKDINO_MODELS, \n ths=THSS\n )\n\n t1_e = time.time()\n print(\"Maskdino infer time: \", t1_e - t1_s)\n t2_s = time.time()\n yolov7_classes, yolov7_scores, yolov7_bboxes, yolov7_masks = ensemble_yolov7_preds(file_name=test_name, \n path=SUBM_PATH, models=yolov7_models)\n\n t2_e = time.time()\n print(\"Yolov7 infer time: \", t2_e - t2_s)\n t3_s = time.time()\n resnet50_classes, resnet50_scores, resnet50_bboxes, resnet50_masks = ensemble_resnet50_preds(file_name=test_name, \n path=SUBM_PATH, models=RESNET50_MODELS)\n t3_e = time.time()\n\n print(\"Restnet50 infer time: \", t3_e - t3_s)\n \n t4_s = time.time()\n internimage_classes, internimage_scores, internimage_bboxes, internimage_masks = ensemble_internimage_preds(file_name=test_name, \n path=SUBM_PATH, models=intern_models)\n t4_e = time.time()\n\n print(\"Internimage infer time: \", t4_e - t4_s)\n\n classes = []\n scores = []\n bboxes = []\n masks = []\n for j in range(len(maskdino_classes)):\n classes.append(maskdino_classes[j])\n scores.append(maskdino_scores[j])\n bboxes.append(maskdino_bboxes[j])\n masks.append(maskdino_masks[j])\n\n for j in range(len(yolov7_classes)):\n classes.append(yolov7_classes[j])\n scores.append(yolov7_scores[j])\n bboxes.append(yolov7_bboxes[j])\n masks.append(yolov7_masks[j])\n\n for j in range(len(resnet50_classes)):\n classes.append(resnet50_classes[j])\n scores.append(resnet50_scores[j])\n bboxes.append(resnet50_bboxes[j])\n masks.append(resnet50_masks[j])\n \n for j in range(len(internimage_classes)):\n classes.append(internimage_classes[j])\n scores.append(internimage_scores[j])\n bboxes.append(internimage_bboxes[j])\n masks.append(internimage_masks[j])\n while not nms_in_q.empty():\n time.sleep(0.2)\n# print(bboxes, classes, scores, masks, test_name)\n nms_in_q.put((classes, scores, bboxes, masks, None, test_name, IOU_THRES))\n\n nms_in_q.put(None) \n p_nms.join()\n p_post.join()\n # cd /kaggle/working/maskdino-sourcecode/MaskDINO","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:54:49.359832Z","iopub.execute_input":"2023-07-15T13:54:49.360431Z","iopub.status.idle":"2023-07-15T13:55:10.969904Z","shell.execute_reply.started":"2023-07-15T13:54:49.360396Z","shell.execute_reply":"2023-07-15T13:55:10.96854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working/\n","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.820591Z","iopub.status.idle":"2023-07-15T13:43:35.821091Z","shell.execute_reply.started":"2023-07-15T13:43:35.820839Z","shell.execute_reply":"2023-07-15T13:43:35.820862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/__pycache__","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.822673Z","iopub.status.idle":"2023-07-15T13:43:35.823578Z","shell.execute_reply.started":"2023-07-15T13:43:35.823325Z","shell.execute_reply":"2023-07-15T13:43:35.823353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/ensemble-boxes","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.825141Z","iopub.status.idle":"2023-07-15T13:43:35.825609Z","shell.execute_reply.started":"2023-07-15T13:43:35.825367Z","shell.execute_reply":"2023-07-15T13:43:35.825391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/maskdino-sourcecode","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.827273Z","iopub.status.idle":"2023-07-15T13:43:35.827743Z","shell.execute_reply.started":"2023-07-15T13:43:35.827497Z","shell.execute_reply":"2023-07-15T13:43:35.827519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls .","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.829173Z","iopub.status.idle":"2023-07-15T13:43:35.829965Z","shell.execute_reply.started":"2023-07-15T13:43:35.829691Z","shell.execute_reply":"2023-07-15T13:43:35.829715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm -rf mask*.png","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.8315Z","iopub.status.idle":"2023-07-15T13:43:35.8322Z","shell.execute_reply.started":"2023-07-15T13:43:35.831957Z","shell.execute_reply":"2023-07-15T13:43:35.831981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %matplotlib inline\n# import os\n\n# for filename in os.listdir(\"/kaggle/working/\"):\n# if \".png\" in filename:\n# print(filename)\n# file_path = os.path.join(\"/kaggle/working/\", filename)\n# image = cv2.imread(file_path)\n# plt.imshow(image)\n# plt.show()\n ","metadata":{"execution":{"iopub.status.busy":"2023-07-15T13:43:35.833772Z","iopub.status.idle":"2023-07-15T13:43:35.834664Z","shell.execute_reply.started":"2023-07-15T13:43:35.834411Z","shell.execute_reply":"2023-07-15T13:43:35.834439Z"},"trusted":true},"execution_count":null,"outputs":[]}]}