{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **[HuBMAP 2023] MMDetection 3.1 Training**","metadata":{}},{"cell_type":"markdown","source":"##### [HuBMAP 2023] K-fold CV COCO Dataset Generator [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-k-fold-cv-coco-dataset-generator)\n##### -------------------------------------------------------------------\n##### [HuBMAP 2023] MMDetectron 3.1 Wheel [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-mmdetectron-3-1-wheel)\n##### [HuBMAP 2023] MMDetection 3.1 Inference [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-mmdetection-3-1-inference)\n##### -------------------------------------------------------------------\n##### [HuBMAP 2023] Detectron2-Training [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-detectron2-training)\n##### [HuBMAP 2023] Detectron2-Inference [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-detectron2-inference)\n##### -------------------------------------------------------------------\n##### [HuBMAP 2023] Torch Mask R-CNN [here](https://www.kaggle.com/code/ammarnassanalhajali/hubmap-2023-torch-mask-r-cnn)","metadata":{}},{"cell_type":"markdown","source":"### Install MMdetectron 3.1 offline","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/pycocotools-2.0.6-cp310-cp310-linux_x86_64.whl\n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/torch-1.12.1+cu116-cp310-cp310-linux_x86_64.whl\n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/torchvision-0.13.1+cu116-cp310-cp310-linux_x86_64.whl\n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/mmcv-2.0.1-cp310-cp310-manylinux1_x86_64.whl \n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/openmim-0.3.9-py2.py3-none-any.whl\n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/mmengine-0.7.4-py3-none-any.whl\n!pip install --no-index --no-deps /kaggle/input/mmdetectron-31-wheel/addict-2.4.0-py3-none-any.whl","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T09:37:24.144892Z","iopub.execute_input":"2023-07-03T09:37:24.146126Z","iopub.status.idle":"2023-07-03T09:37:55.852449Z","shell.execute_reply.started":"2023-07-03T09:37:24.146082Z","shell.execute_reply":"2023-07-03T09:37:55.851263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Libraries","metadata":{}},{"cell_type":"code","source":"from itertools import groupby\nfrom pycocotools import mask as mutils\nfrom pycocotools.coco import COCO\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport wandb\nfrom PIL import Image\nimport gc\n\nfrom glob import glob\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:37:55.854910Z","iopub.execute_input":"2023-07-03T09:37:55.855612Z","iopub.status.idle":"2023-07-03T09:37:55.862603Z","shell.execute_reply.started":"2023-07-03T09:37:55.855567Z","shell.execute_reply":"2023-07-03T09:37:55.861432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install MMdetection 3.1","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/open-mmlab/mmdetection.git\n%cd /kaggle/working/mmdetection\n!pip install -v -e .","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T09:37:55.864292Z","iopub.execute_input":"2023-07-03T09:37:55.865406Z","iopub.status.idle":"2023-07-03T09:38:21.348165Z","shell.execute_reply.started":"2023-07-03T09:37:55.865373Z","shell.execute_reply":"2023-07-03T09:38:21.346984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check versions","metadata":{}},{"cell_type":"code","source":"import torch, torchvision,mmdet\nprint(\"torch=\",torch.__version__,torch.cuda.is_available())\nprint(\"mmdet=\",mmdet.__version__)\nfrom mmcv.ops import get_compiling_cuda_version, get_compiler_version\nprint(get_compiling_cuda_version())\nprint(get_compiler_version())\n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:21.351296Z","iopub.execute_input":"2023-07-03T09:38:21.351996Z","iopub.status.idle":"2023-07-03T09:38:21.359187Z","shell.execute_reply.started":"2023-07-03T09:38:21.351949Z","shell.execute_reply":"2023-07-03T09:38:21.358185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CFG","metadata":{}},{"cell_type":"code","source":"from mmengine.config import Config\nbase=\"mask-rcnn_r50_fpn_1x_coco\"\nbase1=\"mask-rcnn_x101-64x4d_fpn_2x_coco\"\ncfg = Config.fromfile(f'/kaggle/working/mmdetection/configs/mask_rcnn/{base}.py')\n#----------------------------------------------------\nwidth=1024\nheight=1024\n\nmax_epochs=20\n\nbatch_size=2\nnum_classes=2\n\ndataset_type = 'CocoDataset' \nclasses = ('blood_vessel','glomerulus') \ndata_root = '/kaggle/input/hubmap-hacking-the-human-vasculature/'\n#-----------------------------------------------------\ncfg.model.roi_head.bbox_head.num_classes = num_classes\ncfg.model.roi_head.mask_head.num_classes = num_classes\n#-----------------------------------------------------\ncfg.train_pipeline[2]['scale']=(width,height)\ncfg.test_pipeline [1]['scale']=(width,height)\n#-----------------------------------------------------\ncfg.train_dataloader.dataset.type=dataset_type\ncfg.train_dataloader.dataset.metainfo=dict(classes=classes)\ncfg.train_dataloader.dataset.data_root=data_root\ncfg.train_dataloader.dataset.ann_file='/kaggle/input/coco-dataset-hubmap-2023/coco_annotations_train_all_fold1.json'\ncfg.train_dataloader.dataset.data_prefix=dict(img='train/')\ncfg.train_dataloader.batch_size=batch_size\ncfg.train_dataloader.dataset.pipeline[2]['scale']=(width,height)\n#-----------------------------------------------------\ncfg.val_dataloader.dataset.type=dataset_type\ncfg.val_dataloader.dataset.metainfo=dict(classes=classes)\ncfg.val_dataloader.dataset.data_root=data_root\ncfg.val_dataloader.dataset.ann_file='/kaggle/input/coco-dataset-hubmap-2023/coco_annotations_valid_all_fold1.json'\ncfg.val_dataloader.dataset.data_prefix=dict(img='train/')\ncfg.val_dataloader.dataset.pipeline[1]['scale']=(width,height)\n#-----------------------------------------------------\ncfg.test_dataloader.dataset.type=dataset_type\ncfg.test_dataloader.dataset.metainfo=dict(classes=classes)\ncfg.test_dataloader.dataset.data_root=data_root\ncfg.test_dataloader.dataset.ann_file='/kaggle/input/coco-dataset-hubmap-2023/coco_annotations_valid_all_fold1.json'\ncfg.test_dataloader.dataset.data_prefix=dict(img='train/')\ncfg.test_dataloader.dataset.pipeline[1]['scale']=(width,height)\n#------------------------------------------------------\n#------------------------------------------------------\ncfg.val_evaluator.type='CocoMetric'\ncfg.val_evaluator.ann_file='/kaggle/input/coco-dataset-hubmap-2023/coco_annotations_valid_all_fold1.json'\ncfg.val_evaluator.metric=['segm']\n\ncfg.test_evaluator.type='CocoMetric'\ncfg.test_evaluator.ann_file='/kaggle/input/coco-dataset-hubmap-2023/coco_annotations_valid_all_fold1.json'\ncfg.test_evaluator.metric=['segm']\n#------------------------------------------------------\ncfg.train_cfg.max_epochs=max_epochs\ncfg.optim_wrapper.type='OptimWrapper'\ncfg.optim_wrapper.optimizer=dict(type='AdamW',lr=0.001,weight_decay=0.05,eps=1e-8,betas=(0.9, 0.999))\ncfg.default_hooks = dict(logger=dict(type='LoggerHook', interval=200),\n                         checkpoint=dict(type='CheckpointHook', interval=1, save_best='coco/segm_mAP'))\n#------------------------------------------------------\n!mkdir -p configs/HuBMAP\nconfig=f'configs/HuBMAP/custom_config_{base}_{width}_{height}.py'\nwith open(config, 'w') as f:\n    f.write(cfg.pretty_text)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:21.361566Z","iopub.execute_input":"2023-07-03T09:38:21.361906Z","iopub.status.idle":"2023-07-03T09:38:23.605440Z","shell.execute_reply.started":"2023-07-03T09:38:21.361877Z","shell.execute_reply":"2023-07-03T09:38:23.604315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Trainig","metadata":{}},{"cell_type":"code","source":"!python tools/train.py {config}","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T09:38:23.606989Z","iopub.execute_input":"2023-07-03T09:38:23.607583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove files","metadata":{}},{"cell_type":"code","source":"from glob import glob\nimport shutil\nfrom shutil import copyfile\nfrom pathlib import Path\n\n!mkdir -p /kaggle/working/MMdet\nbest_model = glob(f\"/kaggle/working/mmdetection/work_dirs/*/best_coco_segm_mAP*.pth\")[0]\ncustom_config = glob(f\"/kaggle/working/mmdetection/work_dirs/*/custom_config*.py\")[0]\ndst_dir = Path(\"/kaggle/working/MMdet/\")\n\nsrc_best_model = Path(best_model)\nsrc_custom_config = Path(custom_config)\ncopyfile(src_best_model, dst_dir/src_best_model.name)\ncopyfile(src_custom_config, dst_dir/src_custom_config.name)\n\n%cd /kaggle/working\n!rm -r /kaggle/working/mmdetection","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}