{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":160082444,"sourceType":"kernelVersion"},{"sourceId":153879438,"sourceType":"kernelVersion"},{"sourceId":6568,"sourceType":"modelInstanceVersion","modelInstanceId":5146}],"dockerImageVersionId":30627,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install ultralytics library and copy previously finetuned yolov8 model","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install --no-index -f /kaggle/input/download-ultralytics /kaggle/input/download-ultralytics/ultralytics-8.1.5-py3-none-any.whl\n\n!mkdir trained_model\n!cp /kaggle/input/yolo-10-epochs-size-1928-all-except-kidney_3_dense/pytorch/run_tm53kdwd_model/1/best.pt /kaggle/working/trained_model","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-25T10:33:15.140776Z","iopub.execute_input":"2024-01-25T10:33:15.141213Z","iopub.status.idle":"2024-01-25T10:33:34.115470Z","shell.execute_reply.started":"2024-01-25T10:33:15.141181Z","shell.execute_reply":"2024-01-25T10:33:34.113726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\nfrom ultralytics.models.sam import Predictor as SAMPredictor\n\nimport numpy as np\nimport pandas as pd\n\nfrom ultralytics.engine.results import Results","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:33:34.118651Z","iopub.execute_input":"2024-01-25T10:33:34.119119Z","iopub.status.idle":"2024-01-25T10:33:44.884483Z","shell.execute_reply.started":"2024-01-25T10:33:34.119076Z","shell.execute_reply":"2024-01-25T10:33:44.883296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility functions for prediction conversions","metadata":{}},{"cell_type":"code","source":"def add_masks(masks):\n    result = 255*(np.sum(masks, axis=0))\n    result = result.clip(0, 255).astype(\"uint8\")\n    return result","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:33:44.886107Z","iopub.execute_input":"2024-01-25T10:33:44.886654Z","iopub.status.idle":"2024-01-25T10:33:44.892385Z","shell.execute_reply.started":"2024-01-25T10:33:44.886620Z","shell.execute_reply":"2024-01-25T10:33:44.891022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(mask):\n    pixel = mask.flatten()\n    pixel = np.concatenate([[0], pixel, [0]])\n    run = np.where(pixel[1:] != pixel[:-1])[0] + 1\n    run[1::2] -= run[::2]\n    rle = ' '.join(str(r) for r in run)\n    if rle == '':\n        rle = '1 0'\n    return rle","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:33:44.894027Z","iopub.execute_input":"2024-01-25T10:33:44.894371Z","iopub.status.idle":"2024-01-25T10:33:44.911230Z","shell.execute_reply.started":"2024-01-25T10:33:44.894342Z","shell.execute_reply":"2024-01-25T10:33:44.909737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_id_from_result(result: Results):\n    dataset_name = result.path.split('/')[-3]\n    file_name = result.path.split('/')[-1].split('.')[0]\n    return f'{dataset_name}_{file_name}'","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:33:44.917075Z","iopub.execute_input":"2024-01-25T10:33:44.917531Z","iopub.status.idle":"2024-01-25T10:33:44.924602Z","shell.execute_reply.started":"2024-01-25T10:33:44.917459Z","shell.execute_reply":"2024-01-25T10:33:44.923397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_rle_from_result(result: Results):\n    if not result.masks:\n        return '1 0'\n    else:\n        masks_array = result.masks.data.cpu().numpy()\n        combined_mask = add_masks(masks_array)\n        return rle_encode(combined_mask)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:33:44.926049Z","iopub.execute_input":"2024-01-25T10:33:44.926452Z","iopub.status.idle":"2024-01-25T10:33:44.937901Z","shell.execute_reply.started":"2024-01-25T10:33:44.926422Z","shell.execute_reply":"2024-01-25T10:33:44.936647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\n## Init previously fine-tuned yolov8 and off-the-shelf SAM model","metadata":{}},{"cell_type":"code","source":"yolo_confidence_threshold = 0.1\nsam_confidence_threshold = 0.8","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:26:05.358244Z","iopub.execute_input":"2024-01-25T11:26:05.358716Z","iopub.status.idle":"2024-01-25T11:26:05.365978Z","shell.execute_reply.started":"2024-01-25T11:26:05.358682Z","shell.execute_reply":"2024-01-25T11:26:05.364476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nis_submission = len(glob(\"/kaggle/input/blood-vessel-segmentation/test/kidney_5/images/*.tif\")) > 3\n\nmodel = YOLO('/kaggle/working/trained_model/best.pt')\n\ndevice = [0, 1] if is_submission else None\noverrides = dict(conf=sam_confidence_threshold, \n                 task='segment', \n                 mode='predict', \n                 model='/kaggle/input/download-ultralytics/sam_b.pt',\n                 device=device,\n                )\npredictor = SAMPredictor(overrides=overrides)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:52:59.799075Z","iopub.execute_input":"2024-01-25T10:52:59.799582Z","iopub.status.idle":"2024-01-25T10:52:59.921726Z","shell.execute_reply.started":"2024-01-25T10:52:59.799535Z","shell.execute_reply":"2024-01-25T10:52:59.920391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make predictions with yolov8\nWe will pass the bounding boxes predicted with yolov8 as prompts for SAM.","metadata":{}},{"cell_type":"code","source":"if is_submission:\n    source = '/kaggle/input/blood-vessel-segmentation/test/**/*.tif'\n    results = model.predict(source, \n                            # otherwise we get OOM\n                            stream=True, \n                            # only to use T4 * 2\n                            device=[0,1], \n                            # confidence threshold\n                            conf=yolo_confidence_threshold)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:53:03.321940Z","iopub.execute_input":"2024-01-25T10:53:03.322392Z","iopub.status.idle":"2024-01-25T10:53:03.330197Z","shell.execute_reply.started":"2024-01-25T10:53:03.322360Z","shell.execute_reply":"2024-01-25T10:53:03.328552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_submission:\n    submission_list = []\n    for yolo_result in results:\n        img_path = yolo_result.path\n        if not yolo_result.masks:\n            rle = '0 1'\n        else:\n            predictor.set_image(yolo_result.path)\n            sam_results = predictor(bboxes=yolo_result.boxes.xyxy, labels=[1])\n            rle = get_rle_from_result(sam_result[0])\n            predictor.reset_image()\n        img_id = extract_id_from_result(result)\n        submission_list.append({\n            'id': img_id,\n            'rle': rle\n        })\n\n    df = pd.DataFrame(submission_list, columns=['id', 'rle'])\n\n    df.to_csv('submission.csv', index=False)    \n\n    ","metadata":{"execution":{"iopub.status.busy":"2024-01-25T10:53:04.052313Z","iopub.execute_input":"2024-01-25T10:53:04.053005Z","iopub.status.idle":"2024-01-25T10:53:04.062291Z","shell.execute_reply.started":"2024-01-25T10:53:04.052958Z","shell.execute_reply":"2024-01-25T10:53:04.060783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize on an individual image ","metadata":{}},{"cell_type":"code","source":"model = YOLO('/kaggle/working/trained_model/best.pt')\n\noverrides = dict(conf=sam_confidence_threshold, \n                 task='segment', \n                 mode='predict', \n                 model='/kaggle/input/download-ultralytics/sam_b.pt',\n                 device=device,\n                 retina_masks=True\n                )\npredictor = SAMPredictor(overrides=overrides)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:24:34.953656Z","iopub.execute_input":"2024-01-25T11:24:34.954249Z","iopub.status.idle":"2024-01-25T11:24:35.031904Z","shell.execute_reply.started":"2024-01-25T11:24:34.954212Z","shell.execute_reply":"2024-01-25T11:24:35.030538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source = '/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/images/1005.tif'\n# source = '/kaggle/input/blood-vessel-segmentation/test/**/*.tif'\nresults = model.predict(source,  \n                        # confidence threshold\n                        conf=yolo_confidence_threshold)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:24:35.034560Z","iopub.execute_input":"2024-01-25T11:24:35.034964Z","iopub.status.idle":"2024-01-25T11:24:39.302277Z","shell.execute_reply.started":"2024-01-25T11:24:35.034932Z","shell.execute_reply":"2024-01-25T11:24:39.300901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolo_res = results[0]\npredictor.set_image(yolo_res.path)\nsam_results = predictor(bboxes=yolo_res.boxes.xyxy, labels=[1])\n","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:24:39.304126Z","iopub.execute_input":"2024-01-25T11:24:39.304539Z","iopub.status.idle":"2024-01-25T11:25:25.232311Z","shell.execute_reply.started":"2024-01-25T11:24:39.304504Z","shell.execute_reply":"2024-01-25T11:25:25.230723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot results\n\n### Masks from SAM with yolo bounding boxes as prompt","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\n\nplt.figure(figsize=(10, 15))\nsam_masks = add_masks(sam_results[0].masks.data.cpu().numpy())\nplt.imshow(sam_masks);","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:25.234697Z","iopub.execute_input":"2024-01-25T11:25:25.235179Z","iopub.status.idle":"2024-01-25T11:25:25.951178Z","shell.execute_reply.started":"2024-01-25T11:25:25.235136Z","shell.execute_reply":"2024-01-25T11:25:25.949512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(10, 15))\n# plt.imshow(sam_results[0].plot())","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:25.955418Z","iopub.execute_input":"2024-01-25T11:25:25.955908Z","iopub.status.idle":"2024-01-25T11:25:25.961027Z","shell.execute_reply.started":"2024-01-25T11:25:25.955873Z","shell.execute_reply":"2024-01-25T11:25:25.959798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Masks from Yolo  ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 15))\nyolo_masks = add_masks(yolo_res.masks.data.cpu().numpy())\nplt.imshow(yolo_masks);","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:25.963088Z","iopub.execute_input":"2024-01-25T11:25:25.963488Z","iopub.status.idle":"2024-01-25T11:25:26.885962Z","shell.execute_reply.started":"2024-01-25T11:25:25.963456Z","shell.execute_reply":"2024-01-25T11:25:26.884341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ground truth","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 15))\ngroundtruth_mask = cv2.imread(yolo_res.path.replace('images', 'labels'))\nplt.imshow(groundtruth_mask)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:26.887459Z","iopub.execute_input":"2024-01-25T11:25:26.887869Z","iopub.status.idle":"2024-01-25T11:25:27.611004Z","shell.execute_reply.started":"2024-01-25T11:25:26.887837Z","shell.execute_reply":"2024-01-25T11:25:27.609377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nall_masks = [sam_masks, groundtruth_mask, yolo_masks]\nfor n in range(3):\n    ax = plt.subplot(1, 3, n + 1)\n    ax.imshow(all_masks[n])\n","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:27.612466Z","iopub.execute_input":"2024-01-25T11:25:27.612883Z","iopub.status.idle":"2024-01-25T11:25:29.064186Z","shell.execute_reply.started":"2024-01-25T11:25:27.612841Z","shell.execute_reply":"2024-01-25T11:25:29.062844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sam_masks.shape, groundtruth_mask.shape, yolo_masks.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:29.066010Z","iopub.execute_input":"2024-01-25T11:25:29.067275Z","iopub.status.idle":"2024-01-25T11:25:29.079069Z","shell.execute_reply.started":"2024-01-25T11:25:29.067225Z","shell.execute_reply":"2024-01-25T11:25:29.077219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv2.imread(yolo_res.path.replace('images', 'labels')).shape, cv2.imread(yolo_res.path).shape","metadata":{"execution":{"iopub.status.busy":"2024-01-25T11:25:29.081048Z","iopub.execute_input":"2024-01-25T11:25:29.082816Z","iopub.status.idle":"2024-01-25T11:25:29.106519Z","shell.execute_reply.started":"2024-01-25T11:25:29.082760Z","shell.execute_reply":"2024-01-25T11:25:29.105050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}