{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is based on Slawek Biel's notebook (https://www.kaggle.com/slawekbiel/positive-score-with-detectron-2-3-training)","metadata":{}},{"cell_type":"markdown","source":"Please UPVOTE !!","metadata":{}},{"cell_type":"markdown","source":"## Transfer learning\nThese notebooks ([train](https://www.kaggle.com/markunys/sartorius-transfer-learning-train), [inference](https://www.kaggle.com/markunys/sartorius-transfer-learning-inference)) show how to do transfer learning with LIVECell dataset.","metadata":{}},{"cell_type":"code","source":"!pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-04T05:39:55.94987Z","iopub.execute_input":"2021-11-04T05:39:55.950645Z","iopub.status.idle":"2021-11-04T05:43:00.669741Z","shell.execute_reply.started":"2021-11-04T05:39:55.95055Z","shell.execute_reply":"2021-11-04T05:43:00.668838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nfrom pathlib import Path\nimport random, cv2, os\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pycocotools.mask as mask_util\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.structures import polygons_to_bitmask\nfrom detectron2.evaluation import inference_on_dataset, print_csv_format\nfrom detectron2.utils import comm\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:43:00.672044Z","iopub.execute_input":"2021-11-04T05:43:00.672332Z","iopub.status.idle":"2021-11-04T05:43:01.999625Z","shell.execute_reply.started":"2021-11-04T05:43:00.67228Z","shell.execute_reply":"2021-11-04T05:43:01.998939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the LIVECell data","metadata":{}},{"cell_type":"code","source":"dataDir=Path('../input/livecell-dataset/LIVECell_dataset_2021/images')\ncfg = get_cfg()\nregister_coco_instances('sartorius_train',{}, '../input/livecell-dataset/livecell_annotations_train.json', dataDir)\nregister_coco_instances('sartorius_val',{},'../input/livecell-dataset/livecell_annotations_val.json', dataDir)\nregister_coco_instances('sartorius_test',{}, '../input/livecell-dataset/livecell_annotations_test.json', dataDir)\nmetadata = MetadataCatalog.get('sartorius_train')\ntrain_ds = DatasetCatalog.get('sartorius_train')","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:43:02.000863Z","iopub.execute_input":"2021-11-04T05:43:02.00215Z","iopub.status.idle":"2021-11-04T05:43:05.633793Z","shell.execute_reply.started":"2021-11-04T05:43:02.002106Z","shell.execute_reply":"2021-11-04T05:43:05.633052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define evaluator \nGenerates lines like this in the training output:\n`[10/27 18:31:26 d2.evaluation.testing]: copypaste: MaP IoU=0.2192638391201311` \n\nSee here for definition: https://www.kaggle.com/c/sartorius-cell-instance-segmentation/overview/evaluation\n\nI've made some modifications, because LIVECell coco data is not bitmask but polygon.","metadata":{}},{"cell_type":"code","source":"def polygon_to_rle(polygon, shape=(520, 704)):\n    #print(polygon)\n    mask = polygons_to_bitmask([np.asarray(polygon) + 0.25], shape[0], shape[1])\n\n    rle = mask_util.encode(np.asfortranarray(mask))\n    return rle\n\n# Taken from https://www.kaggle.com/theoviel/competition-metric-map-iou\ndef precision_at(threshold, iou):\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) == 1  # Correct objects\n    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n    return np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)\n\ndef score(pred, targ):\n    pred_masks = pred['instances'].pred_masks.cpu().numpy()\n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    enc_targs = list(map(lambda x:x['segmentation'], targ))\n    enc_targs = [polygon_to_rle(enc_targ[0]) for enc_targ in enc_targs]\n    ious = mask_util.iou(enc_preds, enc_targs, [0]*len(enc_targs))\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tp, fp, fn = precision_at(t, ious)\n        p = tp / (tp + fp + fn)\n        prec.append(p)\n    return np.mean(prec)\n\nclass MAPIOUEvaluator(DatasetEvaluator):\n    def __init__(self, dataset_name):\n        dataset_dicts = DatasetCatalog.get(dataset_name)\n        self.annotations_cache = {item['image_id']:item['annotations'] for item in dataset_dicts}\n            \n    def reset(self):\n        self.scores = []\n\n    def process(self, inputs, outputs):\n        for inp, out in zip(inputs, outputs):\n            if len(out['instances']) == 0:\n                self.scores.append(0)    \n            else:\n                targ = self.annotations_cache[inp['image_id']]\n                self.scores.append(score(out, targ))\n\n    def evaluate(self):\n        return {\"MaP IoU\": np.mean(self.scores)}\n\nclass Trainer(DefaultTrainer):\n    @classmethod\n    def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n        return MAPIOUEvaluator(dataset_name)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:43:06.875274Z","iopub.execute_input":"2021-11-04T05:43:06.875503Z","iopub.status.idle":"2021-11-04T05:43:06.893155Z","shell.execute_reply.started":"2021-11-04T05:43:06.875472Z","shell.execute_reply":"2021-11-04T05:43:06.892489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train\n\nTraining for 100 iterations here for demonstration.\n\nI trained the model for about 100000 iterations and selected best model.","metadata":{}},{"cell_type":"code","source":"cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"sartorius_train\", \"sartorius_test\")\ncfg.DATASETS.TEST = (\"sartorius_val\",)\ncfg.DATALOADER.NUM_WORKERS = 8\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\ncfg.SOLVER.IMS_PER_BATCH = 2\ncfg.SOLVER.BASE_LR = 0.0005 \ncfg.SOLVER.MAX_ITER = 100\ncfg.SOLVER.STEPS = []       \ncfg.SOLVER.CHECKPOINT_PERIOD = (len(DatasetCatalog.get('sartorius_train')) + len(DatasetCatalog.get('sartorius_test'))) // cfg.SOLVER.IMS_PER_BATCH  # Once per epoch\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   \ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 8\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = .5\ncfg.TEST.EVAL_PERIOD = (len(DatasetCatalog.get('sartorius_train')) + len(DatasetCatalog.get('sartorius_test'))) // cfg.SOLVER.IMS_PER_BATCH  # Once per epoch\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\nprint(cfg.OUTPUT_DIR)\ntrainer = Trainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2021-11-04T05:43:06.894652Z","iopub.execute_input":"2021-11-04T05:43:06.89514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ./output/model_final.pth","metadata":{"execution":{"iopub.status.busy":"2021-10-27T18:46:40.221665Z","iopub.status.idle":"2021-10-27T18:46:40.222097Z","shell.execute_reply.started":"2021-10-27T18:46:40.221855Z","shell.execute_reply":"2021-10-27T18:46:40.221876Z"},"trusted":true},"execution_count":null,"outputs":[]}]}