{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-26T06:56:49.975761Z","iopub.execute_input":"2022-01-26T06:56:49.976095Z","iopub.status.idle":"2022-01-26T06:57:00.933902Z","shell.execute_reply.started":"2022-01-26T06:56:49.976003Z","shell.execute_reply":"2022-01-26T06:57:00.932968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport random\nimport json, os\nfrom pycocotools.coco import COCO\nfrom skimage import io\nfrom matplotlib import pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:00.937719Z","iopub.execute_input":"2022-01-26T06:57:00.937989Z","iopub.status.idle":"2022-01-26T06:57:01.059225Z","shell.execute_reply.started":"2022-01-26T06:57:00.937959Z","shell.execute_reply":"2022-01-26T06:57:01.058526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nfrom pathlib import Path\nimport random, cv2, os\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pycocotools.mask as mask_util\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:01.060517Z","iopub.execute_input":"2022-01-26T06:57:01.060798Z","iopub.status.idle":"2022-01-26T06:57:01.802165Z","shell.execute_reply.started":"2022-01-26T06:57:01.060757Z","shell.execute_reply":"2022-01-26T06:57:01.801367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize and train/valid Split","metadata":{}},{"cell_type":"code","source":"train_dataset_dir=\"../input/cow-seg/train_dataset/200/\"\ntrain_label=\"../input/cow-seg/train_dataset/200/data.json\"\n\n#copy from https://blog.csdn.net/wtandyn/article/details/109751015\n\ndef visualization_bbox_seg(num_image, json_path, img_path, *str):# 需要画图的是第num副图片， 对应的json路径和图片路径\n\n    coco = COCO(json_path)\n\n    if len(str) == 0:\n        catIds = []\n    else:\n        catIds = coco.getCatIds(catNms = [str[0]])  # 获取给定类别对应的id 的dict（单个内嵌字典的类别[{}]）\n        catIds = coco.loadCats(catIds)[0]['id'] # 获取给定类别对应的id 的dict中的具体id\n\n    list_imgIds = coco.getImgIds(catIds=catIds ) # 获取含有该给定类别的所有图片的id\n    img = coco.loadImgs(list_imgIds[num_image-1])[0]  # 获取满足上述要求，并给定显示第num幅image对应的dict\n    image = io.imread(img_path + img['file_name'])  # 读取图像\n    image_name =  img['file_name'].split('/')[-1] # 读取图像名字\n    image_id = img['id'] # 读取图像id\n\n    img_annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None) # 读取这张图片的所有seg_id\n    img_anns = coco.loadAnns(img_annIds)\n\n    for i in range(len(img_annIds)):\n        x, y, w, h = img_anns[i-1]['bbox']  # 读取边框\n        image = cv2.rectangle(image, (int(x), int(y)), (int(x + w), int(y + h)), (0, 255, 255), 2)\n\n    plt.rcParams['figure.figsize'] = (20.0, 20.0)\n    plt.imshow(image)\n    coco.showAnns(img_anns)\n    plt.show()\n\nvisualization_bbox_seg(1, train_label, train_dataset_dir,'cow') # 最后一个参数不写就是画出一张图中的所有类别\nwith open(train_label,'r') as f:\n    data=json.load(f)\n#\ntrain_dict={\n    'images':[],\n    'categories':[],\n    'annotations':[]\n}\nval_dict={\n    'images':[],\n    'categories':[],\n    'annotations':[]\n}\ntrain_dict['categories']=data['categories']\nval_dict['categories']=data['categories']\nimages=data['images']\nannos=data['annotations']\n#20张用于验证\ntrain_img_lst=[]\ntrain_ann_lst=[]\nval_img_lst=[]\nval_ann_lst=[]\nfor i in range(len(images)):\n    img_info=images[i]\n    img_id=img_info['id']\n    #遍历annos，找到对应image_id的所有标注\n    tmp=[]\n    for ann in annos:\n            if ann['image_id']==img_id:\n                tmp.append(ann)\n    #\n    if i%10==0:\n        val_img_lst.append(img_info)\n        val_ann_lst+=tmp\n    else:\n        train_img_lst.append(img_info)\n        train_ann_lst+=tmp\n#\ntrain_dict['images']=train_img_lst\nval_dict['images']=val_img_lst\ntrain_dict['annotations']=train_ann_lst\nval_dict['annotations']=val_ann_lst\nprint(\"train: {},valid: {}\".format(len(train_img_lst),len(val_img_lst)))\n#\nwith open('val.json','w') as f:\n    tmp=json.dumps(val_dict)\n    f.write(tmp)\nwith open('train.json','w') as f:\n    tmp=json.dumps(train_dict)\n    f.write(tmp)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:01.804025Z","iopub.execute_input":"2022-01-26T06:57:01.804297Z","iopub.status.idle":"2022-01-26T06:57:03.38508Z","shell.execute_reply.started":"2022-01-26T06:57:01.804259Z","shell.execute_reply":"2022-01-26T06:57:03.384259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataDir=Path('../input/cow-seg/train_dataset/200/')\ncfg = get_cfg()\ncfg.INPUT.MASK_FORMAT='bitmask'\nregister_coco_instances('cow_train',{}, 'train.json', dataDir)\nregister_coco_instances('cow_val',{},'val.json', dataDir)\nmetadata = MetadataCatalog.get('cow_train')\ntrain_ds = DatasetCatalog.get('cow_train')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:05.002227Z","iopub.execute_input":"2022-01-26T06:57:05.002787Z","iopub.status.idle":"2022-01-26T06:57:05.092371Z","shell.execute_reply.started":"2022-01-26T06:57:05.002745Z","shell.execute_reply":"2022-01-26T06:57:05.091556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = train_ds[42]\nimg = cv2.imread(d[\"file_name\"])\nvisualizer = Visualizer(img[:, :, ::-1], metadata=metadata)\nout = visualizer.draw_dataset_dict(d)\nplt.figure(figsize = (20,15))\nplt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:12.965942Z","iopub.execute_input":"2022-01-26T06:57:12.966662Z","iopub.status.idle":"2022-01-26T06:57:14.581165Z","shell.execute_reply.started":"2022-01-26T06:57:12.966622Z","shell.execute_reply":"2022-01-26T06:57:14.58037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport logging\nimport os\nfrom collections import OrderedDict\nimport torch\n\nimport detectron2.utils.comm as comm\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.config import get_cfg\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, hooks, launch\nfrom detectron2.evaluation import (\n    CityscapesInstanceEvaluator,\n    CityscapesSemSegEvaluator,\n    COCOEvaluator,\n    COCOPanopticEvaluator,\n    DatasetEvaluators,\n    LVISEvaluator,\n    PascalVOCDetectionEvaluator,\n    SemSegEvaluator,\n    verify_results,\n)\nfrom detectron2.modeling import GeneralizedRCNNWithTTA\n\n\ndef build_evaluator(cfg, dataset_name, output_folder=None):\n    \"\"\"\n    Create evaluator(s) for a given dataset.\n    This uses the special metadata \"evaluator_type\" associated with each builtin dataset.\n    For your own dataset, you can simply create an evaluator manually in your\n    script and do not have to worry about the hacky if-else logic here.\n    \"\"\"\n    if output_folder is None:\n        output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n    evaluator_list = []\n    evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type\n    if evaluator_type in [\"sem_seg\", \"coco_panoptic_seg\"]:\n        evaluator_list.append(\n            SemSegEvaluator(\n                dataset_name,\n                distributed=True,\n                output_dir=output_folder,\n            )\n        )\n    if evaluator_type in [\"coco\", \"coco_panoptic_seg\"]:\n        evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder))\n    if evaluator_type == \"coco_panoptic_seg\":\n        evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder))\n    if evaluator_type == \"cityscapes_instance\":\n        return CityscapesInstanceEvaluator(dataset_name)\n    if evaluator_type == \"cityscapes_sem_seg\":\n        return CityscapesSemSegEvaluator(dataset_name)\n    elif evaluator_type == \"pascal_voc\":\n        return PascalVOCDetectionEvaluator(dataset_name)\n    elif evaluator_type == \"lvis\":\n        return LVISEvaluator(dataset_name, output_dir=output_folder)\n    if len(evaluator_list) == 0:\n        raise NotImplementedError(\n            \"no Evaluator for the dataset {} with the type {}\".format(dataset_name, evaluator_type)\n        )\n    elif len(evaluator_list) == 1:\n        return evaluator_list[0]\n    return DatasetEvaluators(evaluator_list)\n\n\nclass Trainer(DefaultTrainer):\n    \"\"\"\n    We use the \"DefaultTrainer\" which contains pre-defined default logic for\n    standard training workflow. They may not work for you, especially if you\n    are working on a new research project. In that case you can write your\n    own training loop. You can use \"tools/plain_train_net.py\" as an example.\n    \"\"\"\n\n    @classmethod\n    def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n        return build_evaluator(cfg, dataset_name, output_folder)\n\n    @classmethod\n    def test_with_TTA(cls, cfg, model):\n        logger = logging.getLogger(\"detectron2.trainer\")\n        # In the end of training, run an evaluation with TTA\n        # Only support some R-CNN models.\n        logger.info(\"Running inference with test-time augmentation ...\")\n        model = GeneralizedRCNNWithTTA(cfg, model)\n        evaluators = [\n            cls.build_evaluator(\n                cfg, name, output_folder=os.path.join(cfg.OUTPUT_DIR, \"inference_TTA\")\n            )\n            for name in cfg.DATASETS.TEST\n        ]\n        res = cls.test(cfg, model, evaluators)\n        res = OrderedDict({k + \"_TTA\": v for k, v in res.items()})\n        return res\n","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:57:36.704236Z","iopub.execute_input":"2022-01-26T06:57:36.704543Z","iopub.status.idle":"2022-01-26T06:57:36.720412Z","shell.execute_reply.started":"2022-01-26T06:57:36.704501Z","shell.execute_reply":"2022-01-26T06:57:36.719424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training","metadata":{}},{"cell_type":"code","source":"cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"cow_train\",)\ncfg.DATASETS.TEST = (\"cow_val\",)\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\ncfg.SOLVER.IMS_PER_BATCH = 4\ncfg.SOLVER.BASE_LR = 0.0005 \ncfg.SOLVER.MAX_ITER = 2000    \ncfg.SOLVER.STEPS = []        \n#cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   \ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  \n#cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = .5\ncfg.TEST.EVAL_PERIOD = 4*len(DatasetCatalog.get('cow_train')) // cfg.SOLVER.IMS_PER_BATCH  # Once per epoch\n","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-01-26T06:58:25.99277Z","iopub.execute_input":"2022-01-26T06:58:25.993584Z","iopub.status.idle":"2022-01-26T06:58:26.099036Z","shell.execute_reply.started":"2022-01-26T06:58:25.993538Z","shell.execute_reply":"2022-01-26T06:58:26.098263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = Trainer(cfg)\ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T06:59:08.513672Z","iopub.execute_input":"2022-01-26T06:59:08.514454Z","iopub.status.idle":"2022-01-26T07:37:54.881565Z","shell.execute_reply.started":"2022-01-26T06:59:08.514413Z","shell.execute_reply":"2022-01-26T07:37:54.880648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Infer Visualize","metadata":{}},{"cell_type":"code","source":"cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")  # path to the model we just trained\n#cfg.MODEL.WEIGHTS = '../input/cow-seg-with-detectron-training/output/model_final.pth'\npredictor = DefaultPredictor(cfg)\ndataset_dicts = DatasetCatalog.get('cow_val')\nouts = []\nfor d in random.sample(dataset_dicts, 3):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n    v = Visualizer(im[:, :, ::-1],\n                   metadata = MetadataCatalog.get('cow_train'), \n                    \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels. This option is only available for segmentation models\n    )\n    out_pred = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    visualizer = Visualizer(im[:, :, ::-1], metadata=MetadataCatalog.get('cow_train'))\n    out_target = visualizer.draw_dataset_dict(d)\n    outs.append(out_pred)\n    outs.append(out_target)\n_,axs = plt.subplots(len(outs)//2,2,figsize=(40,45))\nfor ax, out in zip(axs.reshape(-1), outs):\n    ax.imshow(out.get_image()[:, :, ::-1])\n","metadata":{"execution":{"iopub.status.busy":"2022-01-26T07:38:27.035236Z","iopub.execute_input":"2022-01-26T07:38:27.035567Z","iopub.status.idle":"2022-01-26T07:38:38.386425Z","shell.execute_reply.started":"2022-01-26T07:38:27.035523Z","shell.execute_reply":"2022-01-26T07:38:38.385454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## get the submit.json","metadata":{}},{"cell_type":"code","source":"\n#\nfrom tqdm import tqdm\nimport pycocotools.mask as mask_util\npre_sub_json=[]\ndataDir='../input/cow-seg/test_dataset_A/images'\nids, masks=[],[]\nid_inx=0\nimage_id_inx=0\ntest_names = os.listdir(dataDir)\nfor name in tqdm(test_names):\n    fn=os.path.join(dataDir,name)\n    im = cv2.imread(fn)\n    pred = predictor(im)\n    image_id_inx+=1\n    take = pred['instances'].scores >= 0.05\n    instances=pred['instances']\n    pred_masks_rle = [mask_util.encode(np.asfortranarray(mask)) for mask in instances.pred_masks[take].cpu()]\n    pred_masks = pred_masks_rle\n    pred_bboxs = instances.pred_boxes.tensor[take]\n    pred_scores = instances.scores[take]\n    pred_bboxs = pred_bboxs.cpu().numpy()\n    pred_scores = pred_scores.cpu().numpy()\n    for i in range(len(pred_masks)):\n        id_inx+=1\n        pre_sub={\n              'image_id': '',\n              'category_id': 1,\n              'segmentation': {},\n              'score': 1.\n        }\n        pre_sub['image_id']='images/'+name\n        mask=pred_masks[i]\n        counts=mask['counts']\n        #mask['counts']=mask['counts'].decode('utf-8')\n        counts=str(counts, encoding='utf-8')\n        mask['counts']=counts\n        bbox=pred_bboxs[i]\n        score=pred_scores[i]\n        pre_sub['segmentation']=mask\n        pre_sub['score']=float(score)\n        #x1=bbox[0]\n        #y1=bbox[1]\n        #w=bbox[2]-bbox[0]\n        #h=bbox[3]-bbox[1]\n        #pre_sub['bbox']=[int(bbox[0]),int(bbox[1]),int(w),int(h)]\n        ##deal with segmentation\n        #img=mask.astype(np.uint8)\n        #img[img==1]=255\n        #edge=cv2.Canny(img,30,100)\n        #xs,ys=np.where(edge==255)\n        #polygon=[]\n        #for j in range(0,len(xs),10):\n        #    polygon.append(int(xs[j]))\n        #    polygon.append(int(ys[j]))\n        #pre_sub['segmentation']=[polygon]\n        #pre_sub['area']=int(w*h)\n        #print(len(polygon))\n        pre_sub_json.append(pre_sub)\n    #break\nlen(pre_sub_json)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T07:38:56.822171Z","iopub.execute_input":"2022-01-26T07:38:56.822439Z","iopub.status.idle":"2022-01-26T07:39:32.27601Z","shell.execute_reply.started":"2022-01-26T07:38:56.822408Z","shell.execute_reply":"2022-01-26T07:39:32.275123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('submission.json','w') as f:\n    data=json.dumps(pre_sub_json)\n    f.write(data)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T07:39:40.501517Z","iopub.execute_input":"2022-01-26T07:39:40.502276Z","iopub.status.idle":"2022-01-26T07:39:40.522559Z","shell.execute_reply.started":"2022-01-26T07:39:40.502234Z","shell.execute_reply":"2022-01-26T07:39:40.52186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}