{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Library to silence Tensorflow Logs\n!pip install -q silence-tensorflow\nimport silence_tensorflow.auto","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:03.736903Z","iopub.execute_input":"2022-03-10T09:15:03.737328Z","iopub.status.idle":"2022-03-10T09:15:22.796095Z","shell.execute_reply.started":"2022-03-10T09:15:03.737227Z","shell.execute_reply":"2022-03-10T09:15:22.795081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add Folders to path, required to import them\nimport sys\nsys.path.append('../input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/efficientnetv2')\nsys.path.append('../input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/Mask_RCNN')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:22.798504Z","iopub.execute_input":"2022-03-10T09:15:22.799194Z","iopub.status.idle":"2022-03-10T09:15:22.805522Z","shell.execute_reply.started":"2022-03-10T09:15:22.799152Z","shell.execute_reply":"2022-03-10T09:15:22.804485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install LZ4 Compression/Decompression Library\n!pip install -q ../input/maskrcnn-tf-2-efficientnetv2-caching/lz4-3.1.3-cp37-cp37m-manylinux1_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:22.807309Z","iopub.execute_input":"2022-03-10T09:15:22.807654Z","iopub.status.idle":"2022-03-10T09:15:31.360901Z","shell.execute_reply.started":"2022-03-10T09:15:22.807613Z","shell.execute_reply":"2022-03-10T09:15:31.359851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport mrcnn.utils as utils\nimport mrcnn.model as modellib\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nimport os\nimport sys\nimport json\nimport time\nimport skimage\nimport imageio\nimport glob\nimport imgaug\nimport multiprocessing\n\nfrom PIL import Image, ImageDraw\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import KFold\nfrom PIL import Image, ImageEnhance\nfrom mrcnn.config import Config\nfrom mrcnn import visualize\n\n# ignore warnings to make outputs clearer\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(f'Python Version: {sys.version}')\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Tensorflow Keras Version: {tf.keras.__version__}')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:31.367634Z","iopub.execute_input":"2022-03-10T09:15:31.368253Z","iopub.status.idle":"2022-03-10T09:15:33.797418Z","shell.execute_reply.started":"2022-03-10T09:15:31.368205Z","shell.execute_reply":"2022-03-10T09:15:33.796392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:33.799034Z","iopub.execute_input":"2022-03-10T09:15:33.799933Z","iopub.status.idle":"2022-03-10T09:15:33.976288Z","shell.execute_reply.started":"2022-03-10T09:15:33.799893Z","shell.execute_reply":"2022-03-10T09:15:33.975214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\n\n# Unique Image IDs\nid_unique = train['id'].unique()\n\n# Original Image File Path\ndef get_file_path(image_id):\n    return f'/kaggle/input/sartorius-cell-instance-segmentation/train/{image_id}.png'\n\ntrain['file_path'] = train['id'].apply(get_file_path)\n\n# Unique Cell Names\nCELL_NAMES = np.sort(train['cell_type'].unique())\nprint(f'CELL_NAMES: {CELL_NAMES}')\n\n# Cell Type to Label Dictionary\nCELL_NAMES_DICT = dict([(v, k) for k, v in enumerate(CELL_NAMES)])\nprint(f'CELL_NAMES_DICT: {CELL_NAMES_DICT}')\n\n# Add Cell Type Label to train, \" + 1\" becaue label 0 is reserved for background\ntrain['cell_type_label'] = train['cell_type'].apply(CELL_NAMES_DICT.get) + 1\n\n# Image Id to Cell Type Label Dictionary\nID2CELL_LABEL = dict(\n    [(k, v) for k, v in train[['id', 'cell_type_label']].itertuples(name=None, index=False)]\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:33.977485Z","iopub.execute_input":"2022-03-10T09:15:33.978014Z","iopub.status.idle":"2022-03-10T09:15:34.693465Z","shell.execute_reply.started":"2022-03-10T09:15:33.977968Z","shell.execute_reply":"2022-03-10T09:15:34.69237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# path to COCO-dataset weights\nCOCO_MODEL_PATH = '../input/maskrcnn-tf-2-efficientnetv2-caching/mask_rcnn_coco.h5'\n\n# Download COCO trained weights from Releases if needed\nif not os.path.exists(COCO_MODEL_PATH):\n    utils.download_trained_weights(COCO_MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.695416Z","iopub.execute_input":"2022-03-10T09:15:34.696096Z","iopub.status.idle":"2022-03-10T09:15:34.704085Z","shell.execute_reply.started":"2022-03-10T09:15:34.696054Z","shell.execute_reply":"2022-03-10T09:15:34.70301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Configuration","metadata":{}},{"cell_type":"code","source":"# Original Image Dimensions\nHEIGHT = 520\nWIDTH = 704\nSHAPE = (HEIGHT, WIDTH)\n\n# Target Image Dimensions which are divisable by 64 as required by the MASK-RCNN model\nHEIGHT_TARGET = 576\nWIDTH_TARGET = 704\nSHAPE_TARGET = (HEIGHT_TARGET, WIDTH_TARGET)\n\nBATCH_SIZE = 1\nN_SAMPLES = train['id'].nunique()\n\n# Debug mode for fast experementing with 50 samples\nDEBUG = False\nDEBUG_SIZE = 50","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.706277Z","iopub.execute_input":"2022-03-10T09:15:34.707022Z","iopub.status.idle":"2022-03-10T09:15:34.729924Z","shell.execute_reply.started":"2022-03-10T09:15:34.706977Z","shell.execute_reply":"2022-03-10T09:15:34.728936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS_ALL = 5 if DEBUG else 30","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.731898Z","iopub.execute_input":"2022-03-10T09:15:34.732561Z","iopub.status.idle":"2022-03-10T09:15:34.738438Z","shell.execute_reply.started":"2022-03-10T09:15:34.73252Z","shell.execute_reply":"2022-03-10T09:15:34.737211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Configuration\n\n1. Class classification size is reduced from 1024 to 128 to reduce model size, which is done to reduce overfitting. The COCO dataset contains 80 object classes, whereas this training method only contains a single \"cell\" class, requiring less parameters.\n2. Disable multithreading for data generators by setting workers to 0. Surprisingly, multithreading is slower than running the data generator on a single core.","metadata":{}},{"cell_type":"code","source":"class CellConfig(Config):\n    \"\"\"Configuration for training on the cigarette butts dataset.\n    Derives from the base Config class and overrides values specific\n    to the cigarette butts dataset.\n    \"\"\"\n    \n    NAME = \"cell\"\n\n    # Set batch size to 1.\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = BATCH_SIZE\n    STEPS_PER_EPOCH = int(DEBUG_SIZE / BATCH_SIZE)  if DEBUG else int(N_SAMPLES / BATCH_SIZE)\n    \n    # Number of Classes\n    NUM_CLASSES = 1 + len(CELL_NAMES)\n\n    # Image Dimensions\n    IMAGE_MIN_DIM = HEIGHT_TARGET\n    IMAGE_MAX_DIM = WIDTH_TARGET\n    IMAGE_SHAPE = [HEIGHT_TARGET, WIDTH_TARGET, 3]\n    IMAGE_RESIZE_MODE = 'none'\n    \n    BACKBONE = 'efficientnetv2-b1'\n\n    # Training Structure\n    FPN_CLASSIF_FC_LAYERS_SIZE = 1024\n    RPN_ANCHOR_SCALES = (32, 64, 128, 256, 512)\n    # Regions of Interest\n    PRE_NMS_LIMIT = 6000\n    # Non Max Supression\n    POST_NMS_ROIS_TRAINING = 2000\n    POST_NMS_ROIS_INFERENCE = 2000\n    # Instances\n    MAX_GT_INSTANCES = 790\n    TRAIN_ROIS_PER_IMAGE = 200\n    DETECTION_MAX_INSTANCES = 200\n    \n    # Thresholds\n    RPN_NMS_THRESHOLD = 0.70        # IoU Threshold for RPN proposals and GT\n    DETECTION_MIN_CONFIDENCE = 0.50 # Non-Background Confidence Threshold\n    DETECTION_NMS_THRESHOLD = 0.30  # IoU Threshold for ROI and GT\n    ROI_POSITIVE_RATIO = 0.33\n    \n    # Prediction Mask Shape\n    MASK_SHAPE = (56, 56)\n    # Size of mask groundtruth\n    USE_MINI_MASK = True\n    MINI_MASK_SHAPE = (112, 112)\n    \n    # DO NOT train Batch Normalization because of small batch size\n    # There are too few samples to correctly train the normalization\n    TRAIN_BN = False\n    \n    # Learning Rate\n    LEARNING_RATE = 0.004\n    WEIGHT_DECAY = 0.0\n    N_WARMUP_STEPS = 2\n    LR_SCHEDULE = True\n    \n    # Dataloader Queue Size (was set to 100 but resulted in OOM error)\n    MAX_QUEUE_SIZE = 10\n    \n    # Cache Items\n    CACHE = True\n    \n    # Debug mode will disable model checkpoints\n    DEBUG = False\n    \n    # Do not use multithreading as this slows down the dataloader!\n    WORKERS = 0\n    \n    # Losses\n    LOSS_WEIGHTS = {\n        'rpn_class_loss': 1.0,    # is the class of the bbox correct? / RPN anchor classifier loss (Forground/Background)\n        'rpn_bbox_loss': 1.0,     # is the size of the bbox correct? / RPN bounding box loss graph (bbox of generic object)\n        'mrcnn_class_loss': 1.0,  # loss for the classifier head of Mask R-CNN (Background / specific class)\n        'mrcnn_bbox_loss': 1.0,   # is the size of the bounding box correct or not? / loss for Mask R-CNN bounding box refinement\n        'mrcnn_mask_loss': 1.0,   # is the class correct? is the pixel correctly assign to the class? / mask binary cross-entropy loss for the masks head\n    }\n    \nconfig = CellConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.744232Z","iopub.execute_input":"2022-03-10T09:15:34.744503Z","iopub.status.idle":"2022-03-10T09:15:34.775452Z","shell.execute_reply.started":"2022-03-10T09:15:34.744475Z","shell.execute_reply":"2022-03-10T09:15:34.773877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RLE Decode","metadata":{}},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode_by_image_id(image_id):\n    rows = train.loc[train['id'] == image_id]\n    \n    # Image Shape\n    mask = np.full(shape=[len(rows), np.prod(SHAPE)], fill_value=0, dtype=np.uint8)\n    \n    for idx, (_, row) in enumerate(rows.iterrows()):\n        s = row['annotation'].split()\n        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n        for lo, hi in zip(starts, ends):\n            mask[idx, lo:hi] = True\n    \n    mask = mask.reshape([len(rows), *SHAPE])\n    mask = np.moveaxis(mask, 0, 2)\n    \n    return mask","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.779207Z","iopub.execute_input":"2022-03-10T09:15:34.779407Z","iopub.status.idle":"2022-03-10T09:15:34.789694Z","shell.execute_reply.started":"2022-03-10T09:15:34.779383Z","shell.execute_reply":"2022-03-10T09:15:34.788526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Training Dataset","metadata":{}},{"cell_type":"code","source":"# Function to pad images and masks\ndef pad_image(image, constant_values):\n    pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n    pad_w = (WIDTH_TARGET - WIDTH) // 2\n    \n    if len(image.shape) == 3:\n        return np.pad(image, ((pad_h, pad_h), (pad_w, pad_w), (0,0)), constant_values=constant_values)\n    else:\n        return np.pad(image, ((pad_h, pad_h), (pad_w, pad_w)), constant_values=constant_values)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.791609Z","iopub.execute_input":"2022-03-10T09:15:34.792003Z","iopub.status.idle":"2022-03-10T09:15:34.801946Z","shell.execute_reply.started":"2022-03-10T09:15:34.791965Z","shell.execute_reply":"2022-03-10T09:15:34.800924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./train ./test\n!mkdir ./train ./test","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:34.803906Z","iopub.execute_input":"2022-03-10T09:15:34.804335Z","iopub.status.idle":"2022-03-10T09:15:36.361994Z","shell.execute_reply.started":"2022-03-10T09:15:34.804285Z","shell.execute_reply":"2022-03-10T09:15:36.360757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_unique = train['id'].unique()\nif DEBUG:\n    id_unique = id_unique[:DEBUG_SIZE]\n\nimage_id2file_path = train.groupby('id')[['id', 'file_path']].head(1)\nimage_id2file_path = image_id2file_path.set_index('id').squeeze().to_dict()\n\n# Create padded training samples with enhanced contrast\nfor image_id in tqdm(id_unique):\n    # Read Original Image\n    image = imageio.imread(image_id2file_path[image_id])\n    # Pad Image\n    image = pad_image(image, 128)\n    \n    # Save image in working directory, required for Mask-RCNN\n    imageio.imwrite(f'./train/{image_id}.png', image)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:15:36.365613Z","iopub.execute_input":"2022-03-10T09:15:36.366457Z","iopub.status.idle":"2022-03-10T09:16:33.919453Z","shell.execute_reply.started":"2022-03-10T09:15:36.36641Z","shell.execute_reply":"2022-03-10T09:16:33.918436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class CellDataset(utils.Dataset):\n\n    def load_data(self, image_ids, form, image_group):\n   \n        for i, name in enumerate(CELL_NAMES):\n            self.add_class('cell', 1 + i, name)\n       \n        # Add the image using the base method from utils.Dataset\n        for vertical_flip in [True, False]:\n            for horizontal_flip in [True, False]:\n                for image in tqdm(image_ids):\n                    self.add_image('cell', \n                           image_id=image, \n                           path=(f'./{image_group}/{image}.png'), \n                           label = ID2CELL_LABEL[image],\n                           height=512, width=512,\n                          vertical_flip=vertical_flip, horizontal_flip=horizontal_flip,\n                      )\n            \n            \n    def load_mask(self, image_id):\n        \"\"\" Load instance masks for the given image.\n        MaskRCNN expects masks in the form of a bitmap [height, width, instances].\n        Args:\n            image_id: The id of the image to load masks for\n        Returns:\n            masks: A bool array of shape [height, width, instance count] with\n                one mask per instance.\n            class_ids: a 1D array of class IDs of the instance masks.\n        \"\"\"\n    \n        info = self.image_info[image_id]\n        image_id = info['id']\n    \n        # Get masks by image_id\n        masks = rle_decode_by_image_id(image_id)\n        masks = pad_image(masks, 0)\n\n        # Get label\n        _, _, size = masks.shape\n        label = info['label']\n        class_ids = np.full(size, label, dtype=np.int32)\n        \n        return masks, class_ids","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:16:33.920979Z","iopub.execute_input":"2022-03-10T09:16:33.92206Z","iopub.status.idle":"2022-03-10T09:16:33.934953Z","shell.execute_reply.started":"2022-03-10T09:16:33.922014Z","shell.execute_reply":"2022-03-10T09:16:33.933998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Training Dataset\ndataset_train = CellDataset()\ndataset_train.load_data(id_unique, 'png', 'train')\ndataset_train.prepare()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:16:33.936745Z","iopub.execute_input":"2022-03-10T09:16:33.93717Z","iopub.status.idle":"2022-03-10T09:16:34.145004Z","shell.execute_reply.started":"2022-03-10T09:16:33.937099Z","shell.execute_reply":"2022-03-10T09:16:34.143847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Training Samples with Target Masks, note the instance classes!\ndataset = dataset_train\nimage_ids = np.random.choice(dataset.image_ids, 5)\nfor image_id in image_ids:\n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:16:34.147096Z","iopub.execute_input":"2022-03-10T09:16:34.147465Z","iopub.status.idle":"2022-03-10T09:16:40.481157Z","shell.execute_reply.started":"2022-03-10T09:16:34.147423Z","shell.execute_reply":"2022-03-10T09:16:40.480098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# Create model in training mode\n!mkdir 'model_checkpoints'\nmodel = modellib.MaskRCNN(mode=\"training\", config=config, model_dir='model_checkpoints')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:16:40.482539Z","iopub.execute_input":"2022-03-10T09:16:40.483406Z","iopub.status.idle":"2022-03-10T09:17:00.704554Z","shell.execute_reply.started":"2022-03-10T09:16:40.483356Z","shell.execute_reply":"2022-03-10T09:17:00.703567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"init_with = \"coco\"\nexclude = [\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \"mrcnn_bbox\", \"mrcnn_mask\"]\nif 'efficientnetv2-b' in config.BACKBONE:\n    exclude += [\n        \"fpn_c5p5\", \"fpn_c4p4\", \"fpn_c3p3\", \"fpn_c2p2\",\n    ]\n    \n# Exclude FC layer if it is not the original size\nif config.FPN_CLASSIF_FC_LAYERS_SIZE != 1024:\n    print(f'Excluding FC layer')\n    exclude += [\n        \"mrcnn_class_conv1\", \"mrcnn_class_bn1\", \"mrcnn_class_conv2\", \"mrcnn_class_bn2\",\n    ]\n    \n# using coco weights\nmodel.load_weights(\n    COCO_MODEL_PATH,\n    by_name=True,\n    exclude=exclude,\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:00.706543Z","iopub.execute_input":"2022-03-10T09:17:00.706947Z","iopub.status.idle":"2022-03-10T09:17:08.471667Z","shell.execute_reply.started":"2022-03-10T09:17:00.706889Z","shell.execute_reply":"2022-03-10T09:17:08.47066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load EfficientNetV2 Weights Pretrained on Imagenet21K \nmodel.keras_model.layers[1].load_weights('/kaggle/input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/efficientnetv2_model_checkpoints/efficientnetv2-b1-imagenet21k.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:08.473575Z","iopub.execute_input":"2022-03-10T09:17:08.473941Z","iopub.status.idle":"2022-03-10T09:17:11.259028Z","shell.execute_reply.started":"2022-03-10T09:17:08.473881Z","shell.execute_reply":"2022-03-10T09:17:11.258016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Summary","metadata":{}},{"cell_type":"code","source":"# Added functionality, plot model summary\nmodel.show_summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:11.26094Z","iopub.execute_input":"2022-03-10T09:17:11.261505Z","iopub.status.idle":"2022-03-10T09:17:13.660111Z","shell.execute_reply.started":"2022-03-10T09:17:11.261461Z","shell.execute_reply":"2022-03-10T09:17:13.658531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Mask-RCNN Architecture\nplt.figure(figsize=(25, 10))\nplt.title('Mask-RCNN Model Architecture')\nplt.imshow(imageio.imread('./model.png'))\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:13.662752Z","iopub.execute_input":"2022-03-10T09:17:13.663154Z","iopub.status.idle":"2022-03-10T09:17:17.659073Z","shell.execute_reply.started":"2022-03-10T09:17:13.663104Z","shell.execute_reply":"2022-03-10T09:17:17.658117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"# Added functionality, show learning rate schedule\nmodel.plot_lr_schedule(EPOCHS_ALL)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:17.660681Z","iopub.execute_input":"2022-03-10T09:17:17.661189Z","iopub.status.idle":"2022-03-10T09:17:18.400823Z","shell.execute_reply.started":"2022-03-10T09:17:17.661146Z","shell.execute_reply":"2022-03-10T09:17:18.399825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Whole Model","metadata":{}},{"cell_type":"markdown","source":"here the actual training happens. Training will be increasingly fast as the training samples are cached and cache hits increase.","metadata":{}},{"cell_type":"code","source":"start_train = time.time()\ncallback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\nhistory = model.train(\n    dataset_train, None, \n    learning_rate=config.LEARNING_RATE,\n    epochs=EPOCHS_ALL, \n    layers=\"all\",\n    augmentation=None,\n    custom_callbacks=[callback],\n)\n\nend_train = time.time()\nminutes = round((end_train - start_train) / 60, 2)\nprint(f'Training took {minutes} minutes')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:17:18.402657Z","iopub.execute_input":"2022-03-10T09:17:18.403317Z","iopub.status.idle":"2022-03-10T09:23:12.674983Z","shell.execute_reply.started":"2022-03-10T09:17:18.403264Z","shell.execute_reply":"2022-03-10T09:23:12.672886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"# Plots a metric\ndef plot_history_metric(metric, f_best=np.argmax):\n    values = history.history[metric]\n    plt.figure(figsize=(15, 8))\n    N_EPOCHS = len(values)\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n    x_ticks = np.arange(1, N_EPOCHS+1)\n        \n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label='train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label='train_best')\n    \n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n    plt.xlabel('epoch', fontsize=20, labelpad=10)\n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    plt.legend(prop={'size': 18})\n    plt.grid()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.67612Z","iopub.status.idle":"2022-03-10T09:23:12.676524Z","shell.execute_reply.started":"2022-03-10T09:23:12.676308Z","shell.execute_reply":"2022-03-10T09:23:12.676333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mean loss\nplot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.677873Z","iopub.status.idle":"2022-03-10T09:23:12.678275Z","shell.execute_reply.started":"2022-03-10T09:23:12.678046Z","shell.execute_reply":"2022-03-10T09:23:12.678073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Region Proposal Network Foreground / Background Classifier\nplot_history_metric('rpn_class_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.680101Z","iopub.status.idle":"2022-03-10T09:23:12.680508Z","shell.execute_reply.started":"2022-03-10T09:23:12.680299Z","shell.execute_reply":"2022-03-10T09:23:12.680325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Region Proposal Network Bounding Box Loss\nplot_history_metric('rpn_bbox_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.682073Z","iopub.status.idle":"2022-03-10T09:23:12.682592Z","shell.execute_reply.started":"2022-03-10T09:23:12.682355Z","shell.execute_reply":"2022-03-10T09:23:12.682397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mask RCNN Head Class Classifier Background / specific class\nplot_history_metric('mrcnn_class_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.685307Z","iopub.status.idle":"2022-03-10T09:23:12.685899Z","shell.execute_reply.started":"2022-03-10T09:23:12.685546Z","shell.execute_reply":"2022-03-10T09:23:12.685588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Mask RCNN Head Bounding Box Loss\nplot_history_metric('mrcnn_bbox_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.687108Z","iopub.status.idle":"2022-03-10T09:23:12.688157Z","shell.execute_reply.started":"2022-03-10T09:23:12.687833Z","shell.execute_reply":"2022-03-10T09:23:12.687865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mask RCNN Head Object Mask Binary Cross Entropy Loss\nplot_history_metric('mrcnn_mask_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.690045Z","iopub.status.idle":"2022-03-10T09:23:12.690819Z","shell.execute_reply.started":"2022-03-10T09:23:12.690467Z","shell.execute_reply":"2022-03-10T09:23:12.690494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"class InferenceConfig(CellConfig):\n    IMAGES_PER_GPU = 1\n    DETECTION_MAX_INSTANCES = 200\n    DETECTION_MIN_CONFIDENCE = 0.70\n    USE_MINI_MASK = False\n    \n\ninference_config = InferenceConfig()\ninference_config.display()","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.692477Z","iopub.status.idle":"2022-03-10T09:23:12.693355Z","shell.execute_reply.started":"2022-03-10T09:23:12.693031Z","shell.execute_reply":"2022-03-10T09:23:12.693059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode=\"inference\", config=inference_config, model_dir='model_checkpoints')","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.695476Z","iopub.status.idle":"2022-03-10T09:23:12.696052Z","shell.execute_reply.started":"2022-03-10T09:23:12.695728Z","shell.execute_reply":"2022-03-10T09:23:12.695757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set EfficientNetV2 head untrainable\nif 'efficientnetv2-b' in  inference_config.BACKBONE:\n    model.keras_model.layers[1].layers[-1].trainable = False","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.697913Z","iopub.status.idle":"2022-03-10T09:23:12.698682Z","shell.execute_reply.started":"2022-03-10T09:23:12.698356Z","shell.execute_reply":"2022-03-10T09:23:12.698386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find last epoch\nmodel_path = model.find_last()\n\n# Load trained weights (fill in path to trained weights here)\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.700473Z","iopub.status.idle":"2022-03-10T09:23:12.701059Z","shell.execute_reply.started":"2022-03-10T09:23:12.700756Z","shell.execute_reply":"2022-03-10T09:23:12.700783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Train Predictions","metadata":{}},{"cell_type":"code","source":"for file_path in glob.glob('./train/*.png')[:25]:\n    img = skimage.io.imread(file_path)\n    img = np.expand_dims(img, axis=2)\n    img = np.concatenate((img, img, img), axis=2)\n    results = model.detect([img], verbose=1)\n    r = results[0]\n    \n    # Image Id\n    image_id = file_path.split('/')[-1].split('.')[0]\n    print(f'image_id: {image_id}')\n    \n    mask = rle_decode_by_image_id(image_id)\n    mask = np.sum(mask, axis=2)\n    plt.figure(figsize=(16,16))\n    plt.imshow(mask)\n    plt.show()\n    \n    visualize.display_instances(\n        img,\n        r['rois'],\n        r['masks'],\n        r['class_ids'], \n        ['BG'] + CELL_NAMES.tolist(),\n        r['scores'],\n        figsize=(16,16)\n    )","metadata":{"execution":{"iopub.status.busy":"2022-03-10T09:23:12.70277Z","iopub.status.idle":"2022-03-10T09:23:12.703937Z","shell.execute_reply.started":"2022-03-10T09:23:12.703592Z","shell.execute_reply":"2022-03-10T09:23:12.703624Z"},"trusted":true},"execution_count":null,"outputs":[]}]}