{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, glob\nimport sys\nimport json\nfrom PIL import Image\nfrom collections import Counter\n\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport tifffile as tiff\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport torch\nimport cv2\n\nimport pandas as pd\n\nfrom sklearn.model_selection import KFold\n\nsys.path.append(\"/kaggle/input/detection-wheel\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-31T11:04:42.924678Z","iopub.execute_input":"2023-07-31T11:04:42.925074Z","iopub.status.idle":"2023-07-31T11:04:47.142484Z","shell.execute_reply.started":"2023-07-31T11:04:42.925020Z","shell.execute_reply":"2023-07-31T11:04:47.141430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_valid_all_fold1.json',\"r\") as f:\n    t = json.loads(f.read())\n    \nimgs = []\nfor img in t[\"images\"]:\n    img[\"height\"] = 512\n    img[\"width\"] = 512\n    imgs.append(img)\nt[\"images\"] = imgs\n\nprint(t[\"categories\"])\nt[\"categories\"]=[t[\"categories\"][0]]\n\nanns = []\nfor ann in t[\"annotations\"]:\n    if ann[\"category_id\"]==1:\n        anns.append(ann)\nt[\"annotations\"]=anns\n\nwith open(\"coco_annotations_valid_all.json\",\"w\") as f:\n    json.dump(t,f)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:04:47.144620Z","iopub.execute_input":"2023-07-31T11:04:47.145363Z","iopub.status.idle":"2023-07-31T11:04:47.918242Z","shell.execute_reply.started":"2023-07-31T11:04:47.145323Z","shell.execute_reply":"2023-07-31T11:04:47.914903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_train_all_fold1.json\",\"r\") as f:\n    t = json.loads(f.read())\n    \nimgs = []\nfor img in t[\"images\"]:\n    img[\"height\"] = 512\n    img[\"width\"] = 512\n    imgs.append(img)\nt[\"images\"] = imgs\n\nprint(t[\"categories\"])\nt[\"categories\"]=[t[\"categories\"][0]]\n\nanns = []\nfor ann in t[\"annotations\"]:\n    if ann[\"category_id\"]==1:\n        anns.append(ann)\nt[\"annotations\"]=anns\n\nwith open(\"coco_annotations_train_all.json\",\"w\") as f:\n    json.dump(t,f)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:04:47.919789Z","iopub.execute_input":"2023-07-31T11:04:47.920513Z","iopub.status.idle":"2023-07-31T11:04:49.869616Z","shell.execute_reply.started":"2023-07-31T11:04:47.920457Z","shell.execute_reply":"2023-07-31T11:04:49.868583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.__version__  #torch 2.0\n#!nvidia-smi   #CUDA Version: 11.4\n! ls /usr/local","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:04:49.872315Z","iopub.execute_input":"2023-07-31T11:04:49.873188Z","iopub.status.idle":"2023-07-31T11:04:50.834540Z","shell.execute_reply.started":"2023-07-31T11:04:49.873148Z","shell.execute_reply":"2023-07-31T11:04:50.833323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qqq /kaggle/input/mmdet3-wheels-ando/addict-2.4.0-py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdet3-wheels-ando/mmengine-0.7.3-py3-none-any.whl\n!pip install -qqq /kaggle/input/pycocotools-206/wheels/pycocotools-2.0.6-cp310-cp310-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:04:50.836254Z","iopub.execute_input":"2023-07-31T11:04:50.836962Z","iopub.status.idle":"2023-07-31T11:05:16.930851Z","shell.execute_reply.started":"2023-07-31T11:04:50.836924Z","shell.execute_reply":"2023-07-31T11:05:16.929324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qqq /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/yapf-0.32.0-py2.py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/terminal-0.4.0-py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/terminaltables-3.1.10-py2.py3-none-any.whl\n\n!cp -r /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdetection/ /kaggle/working/\n%cd /kaggle/working/mmdetection\n!pip install -e . --no-deps\n%cd /kaggle/working/\n\n!pip install /kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdet-2.26.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:05:16.933575Z","iopub.execute_input":"2023-07-31T11:05:16.933957Z","iopub.status.idle":"2023-07-31T11:06:56.232613Z","shell.execute_reply.started":"2023-07-31T11:05:16.933918Z","shell.execute_reply":"2023-07-31T11:06:56.231386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/k/atom1231/mmdet3-wheels/mmcv_full-1.7.1-cp310-cp310-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:06:56.235298Z","iopub.execute_input":"2023-07-31T11:06:56.235937Z","iopub.status.idle":"2023-07-31T11:07:08.906745Z","shell.execute_reply.started":"2023-07-31T11:06:56.235897Z","shell.execute_reply":"2023-07-31T11:07:08.905358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:08.909019Z","iopub.execute_input":"2023-07-31T11:07:08.909441Z","iopub.status.idle":"2023-07-31T11:07:08.921558Z","shell.execute_reply.started":"2023-07-31T11:07:08.909402Z","shell.execute_reply":"2023-07-31T11:07:08.920621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(\n        \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n        mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n        \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n        mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:08.923440Z","iopub.execute_input":"2023-07-31T11:07:08.923836Z","iopub.status.idle":"2023-07-31T11:07:08.934417Z","shell.execute_reply.started":"2023-07-31T11:07:08.923800Z","shell.execute_reply":"2023-07-31T11:07:08.933367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%mkdir /kaggle/working/configs/\n%mkdir work_dir","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:08.939492Z","iopub.execute_input":"2023-07-31T11:07:08.939901Z","iopub.status.idle":"2023-07-31T11:07:10.830659Z","shell.execute_reply.started":"2023-07-31T11:07:08.939876Z","shell.execute_reply":"2023-07-31T11:07:10.829305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mmdet, mmcv\nprint(mmdet.__version__)\nprint(mmcv.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:10.832613Z","iopub.execute_input":"2023-07-31T11:07:10.832942Z","iopub.status.idle":"2023-07-31T11:07:12.740604Z","shell.execute_reply.started":"2023-07-31T11:07:10.832913Z","shell.execute_reply":"2023-07-31T11:07:12.739599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mmengine.config import Config\n","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.742165Z","iopub.execute_input":"2023-07-31T11:07:12.742765Z","iopub.status.idle":"2023-07-31T11:07:12.853946Z","shell.execute_reply.started":"2023-07-31T11:07:12.742728Z","shell.execute_reply":"2023-07-31T11:07:12.852845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### cascade_mask_rcnn_x101_64x4d_fpn_20e_coco CONF","metadata":{}},{"cell_type":"code","source":"# cfg = Config.fromfile(\"/kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdetection/configs/cascade_rcnn/cascade_mask_rcnn_x101_64x4d_fpn_20e_coco.py\")\n\n# # from mmengine.config import Config\n# # from mmengine.runner import Runner\n\n# # from mmdet.utils import register_all_modules\n\n# # cfg = Config.fromfile(\"/kaggle/working/configs/custom_config.py\")\n# cfg.data.samples_per_gpu=4\n# cfg.seed=0\n# cfg.gpu_ids = range(0, 1)\n# cfg.fp16 = dict(loss_scale=512.0)\n# cfg.classes=('blood_vessel',)\n# cfg.work_dir = \"/kaggle/working/work_dir\"\n# cfg.device = 'cuda' #torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n# for head in cfg.model.roi_head.bbox_head:\n#     head.num_classes = 1\n    \n# cfg.model.roi_head.mask_head.num_classes=1\n# cfg.data.train.classes=cfg.classes\n# cfg.data.val.classes=cfg.classes\n# cfg.data.test.classes=cfg.classes\n\n# cfg.train_pipeline[2].img_scale=(1024,1024)\n# # del cfg.train_pipeline[5]\n# cfg.test_pipeline[1].img_scale=(1024,1024)\n# del cfg.test_pipeline[1].transforms[3]\n\n# cfg.data.train.ann_file='/kaggle/input/hubmap-2023-coco-data-prep/coco_annotations_train_all.json'#\n# cfg.data.train.ann_file='/kaggle/working/coco_annotations_train_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_train_all_fold1.json'\n# cfg.data.train.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\n# cfg.data.val.ann_file='/kaggle/input/hubmap-2023-coco-data-prep/coco_annotations_valid_all.json'#\n# cfg.data.val.ann_file='/kaggle/working/coco_annotations_valid_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_valid_all_fold1.json'\n# cfg.data.val.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\n# cfg.data.test.ann_file='/kaggle/input/hubmap-2023-coco-data-prep/coco_annotations_valid_all.json'#\n# cfg.data.test.ann_file='/kaggle/working/coco_annotations_valid_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_valid_all_fold1.json'\n# cfg.data.test.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\n# cfg.data.train.pipeline = cfg.train_pipeline\n# cfg.data.val.pipeline = cfg.test_pipeline\n# cfg.data.test.pipeline = cfg.test_pipeline\n\n# cfg.load_from = '../input/cascade-mask-rcnn-mmdet/cascade_mask_rcnn_x101_64x4d_fpn_20e_coco_20200512_161033-bdb5126a.pth'\n# cfg.runner.max_epochs=15\n","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.855761Z","iopub.execute_input":"2023-07-31T11:07:12.856173Z","iopub.status.idle":"2023-07-31T11:07:12.863290Z","shell.execute_reply.started":"2023-07-31T11:07:12.856134Z","shell.execute_reply":"2023-07-31T11:07:12.862239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### htc_x101_64x4d_fpn_16x1_20e_coco CONF","metadata":{}},{"cell_type":"code","source":"cfg = Config.fromfile(\"/kaggle/input/mmdetection-2-26-0/mmdetection-2-26-0/mmdetection/configs/htc/htc_x101_64x4d_fpn_16x1_20e_coco.py\")\n# from mmengine.config import Config\n# from mmengine.runner import Runner\n\n# from mmdet.utils import register_all_modules\n\n# cfg = Config.fromfile(\"/kaggle/working/configs/custom_config.py\")\ncfg.data.samples_per_gpu=4\ncfg.seed=0\ncfg.gpu_ids = range(0, 1)\ncfg.fp16 = dict(loss_scale=512.0)\ncfg.classes=('blood_vessel',)\ncfg.work_dir = \"/kaggle/working/work_dir\"\ncfg.device = 'cuda' #torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nfor head in cfg.model.roi_head.bbox_head:\n    head.num_classes = 1\n\nfor head in cfg.model.roi_head.mask_head:\n    head.num_classes=1\n    \ncfg.data.train.classes=cfg.classes\ncfg.data.val.classes=cfg.classes\ncfg.data.test.classes=cfg.classes\n\ncfg.train_pipeline[2].img_scale=(1024,1024)\ndel cfg.train_pipeline[5]\ndel cfg.train_pipeline[5]\ncfg.test_pipeline[1].img_scale=(1024,1024)\ndel cfg.test_pipeline[1].transforms[3]\ncfg.train_pipeline[1].with_seg=False\ncfg.train_pipeline[6][\"keys\"]=[\"img\", \"gt_bboxes\", \"gt_labels\", \"gt_masks\"]\ncfg.data.train.ann_file='/kaggle/working/coco_annotations_train_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_train_all_fold1.json'\ncfg.data.train.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\ncfg.data.val.ann_file='/kaggle/working/coco_annotations_valid_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_valid_all_fold1.json'\ncfg.data.val.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\ncfg.data.test.ann_file='/kaggle/working/coco_annotations_valid_all.json'#'/kaggle/input/hubmap-human-vasculature-mmdet-models/coco_annotations_valid_all_fold1.json'\ncfg.data.test.img_prefix='/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n\ncfg.data.train.pipeline = cfg.train_pipeline\ncfg.data.val.pipeline = cfg.test_pipeline\ncfg.data.test.pipeline = cfg.test_pipeline\n\n\ncfg.runner.max_epochs=30\ndel cfg.model.roi_head.semantic_roi_extractor\ndel cfg.model.roi_head.semantic_head\n# cfg.train_pipeline[1].with_seg=False\n# del cfg.data.train.seg_prefix","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.864866Z","iopub.execute_input":"2023-07-31T11:07:12.865516Z","iopub.status.idle":"2023-07-31T11:07:12.949863Z","shell.execute_reply.started":"2023-07-31T11:07:12.865468Z","shell.execute_reply":"2023-07-31T11:07:12.948865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del cfg.data.train.seg_prefix","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.951367Z","iopub.execute_input":"2023-07-31T11:07:12.951738Z","iopub.status.idle":"2023-07-31T11:07:12.956126Z","shell.execute_reply.started":"2023-07-31T11:07:12.951699Z","shell.execute_reply":"2023-07-31T11:07:12.955187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg.optimizer.lr=0.01\ncfg.evaluation.save_best='segm_mAP'\ncfg.evaluation.interval=1\ncfg.lr_config.step=[8,14,19,26]\ncfg.checkpoint_config = None","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.957598Z","iopub.execute_input":"2023-07-31T11:07:12.958192Z","iopub.status.idle":"2023-07-31T11:07:12.967195Z","shell.execute_reply.started":"2023-07-31T11:07:12.958152Z","shell.execute_reply":"2023-07-31T11:07:12.966155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mmdet.datasets import build_dataset\nfrom mmdet.models import build_detector\nfrom mmdet.apis import train_detector\nfrom mmdet.apis import inference_detector, init_detector, show_result_pyplot, set_random_seed","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:12.968908Z","iopub.execute_input":"2023-07-31T11:07:12.969259Z","iopub.status.idle":"2023-07-31T11:07:14.327574Z","shell.execute_reply.started":"2023-07-31T11:07:12.969225Z","shell.execute_reply":"2023-07-31T11:07:14.326446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datasets = [build_dataset(cfg.data.train)]\nmodel = build_detector(cfg.model, train_cfg=cfg.get('train_cfg'), test_cfg=cfg.get('test_cfg'))\nmodel.CLASSES = datasets[0].CLASSES\n\nmmcv.mkdir_or_exist(os.path.abspath(cfg.work_dir))\nmeta = dict()\nmeta['config'] = cfg.pretty_text\ntrain_detector(model, datasets, cfg, distributed=False, validate=True, meta=meta)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T11:07:14.329130Z","iopub.execute_input":"2023-07-31T11:07:14.329498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg.dump(\"config.json\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/mmdetection","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}