{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello Fellow Kagglers,\n\nThis notebook demonstrates the pretraining process on 4000 LIVECell samples. The LIVECell dataset is retrieved from [this](https://github.com/google/automl/tree/master/efficientnetv2) GitHub page. The dataset containing preprocessed samples with horizontal and vertical flips, resulting in 4000x4=16000 training samples is created in [this](https://www.kaggle.com/markwijkhuizen/sartorius-livecell-preprocessing) notebook.\n\nThis pretraining process should make the training process easier by reducing the distance between the starting and optimal weights. COCO weights are trained on a dataset which is conceptually far from the Sartorius dataset, pretraining on LIVECell should therefore result in better performance.\n\nA custom Mask-RCNN library is used which supports preprocessed LZ4 compressed training samples, allowing for lightning fast training.","metadata":{}},{"cell_type":"code","source":"# Shut up Tensorflow\n!pip install -q silence-tensorflow\nimport silence_tensorflow.auto","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:31.181084Z","iopub.execute_input":"2021-12-17T18:30:31.181886Z","iopub.status.idle":"2021-12-17T18:30:45.910410Z","shell.execute_reply.started":"2021-12-17T18:30:31.181776Z","shell.execute_reply":"2021-12-17T18:30:45.909611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/sartorius-coco-models/Instance_Segmentation/efficientnetv2')\nsys.path.append('../input/sartorius-coco-models/Instance_Segmentation/Mask_RCNN')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:45.912545Z","iopub.execute_input":"2021-12-17T18:30:45.912847Z","iopub.status.idle":"2021-12-17T18:30:45.919082Z","shell.execute_reply.started":"2021-12-17T18:30:45.912811Z","shell.execute_reply":"2021-12-17T18:30:45.916825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install LZ4 Compression/Decompression Library\n!pip install -q lz4","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:45.920565Z","iopub.execute_input":"2021-12-17T18:30:45.921186Z","iopub.status.idle":"2021-12-17T18:30:53.930908Z","shell.execute_reply.started":"2021-12-17T18:30:45.921127Z","shell.execute_reply":"2021-12-17T18:30:53.930027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport mrcnn.utils as utils\nimport mrcnn.model as modellib\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nimport os\nimport sys\nimport json\nimport time\nimport skimage\nimport imageio\nimport glob\nimport imgaug\nimport multiprocessing\nimport effnetv2_model\n\nfrom PIL import Image, ImageDraw\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import KFold\nfrom PIL import Image, ImageEnhance\nfrom mrcnn.config import Config\nfrom mrcnn import visualize\n\n# ignore warnings to make outputs clearer\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(f'Python Version: {sys.version}')\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Tensorflow Keras Version: {tf.keras.__version__}')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:53.933098Z","iopub.execute_input":"2021-12-17T18:30:53.933378Z","iopub.status.idle":"2021-12-17T18:30:55.944357Z","shell.execute_reply.started":"2021-12-17T18:30:53.933342Z","shell.execute_reply":"2021-12-17T18:30:55.943400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Num GPUs Available: ', len(tf.config.experimental.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:55.949410Z","iopub.execute_input":"2021-12-17T18:30:55.949765Z","iopub.status.idle":"2021-12-17T18:30:56.121968Z","shell.execute_reply.started":"2021-12-17T18:30:55.949729Z","shell.execute_reply":"2021-12-17T18:30:56.120616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_feather('/kaggle/input/sartorius-livecell-preprocessing-dataset/df_processed.feather')\n\n# Unique Image IDs\nid_unique = train['image_id'].unique()\n\n# Original Image File Path\ndef get_file_path(image_id):\n    return f'/kaggle/input/sartorius-livecell-preprocessing-dataset/train/{image_id}/{image_id}.png'\n\ntrain['file_path'] = train['image_id'].apply(get_file_path)\n\n# Unique Cell Names\nCELL_TYPES = np.sort(train['cell_type'].unique())\nprint(f'CELL_TYPES: {CELL_TYPES}')\n\n# Cell Type to Label Dictionary\nCELL_NAMES_DICT = dict([(v, k) for k, v in enumerate(CELL_TYPES)])\n\n# Image Id to Cell Type Label Dictionary\nID2CELL_LABEL = dict(\n    [(k, v) for k, v in train[['image_id', 'label']].itertuples(name=None, index=False)]\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:56.126031Z","iopub.execute_input":"2021-12-17T18:30:56.127394Z","iopub.status.idle":"2021-12-17T18:30:57.590617Z","shell.execute_reply.started":"2021-12-17T18:30:56.127353Z","shell.execute_reply":"2021-12-17T18:30:57.589882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head())","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.592203Z","iopub.execute_input":"2021-12-17T18:30:57.592456Z","iopub.status.idle":"2021-12-17T18:30:57.617394Z","shell.execute_reply.started":"2021-12-17T18:30:57.592423Z","shell.execute_reply":"2021-12-17T18:30:57.616651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.info())","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.618719Z","iopub.execute_input":"2021-12-17T18:30:57.618979Z","iopub.status.idle":"2021-12-17T18:30:57.770643Z","shell.execute_reply.started":"2021-12-17T18:30:57.618945Z","shell.execute_reply":"2021-12-17T18:30:57.769957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# path to COCO-dataset weights\nCOCO_MODEL_PATH = '../input/sartorius-coco-models/mask_rcnn_coco.h5'\n\n# Download COCO trained weights from Releases if needed\nif not os.path.exists(COCO_MODEL_PATH):\n    utils.download_trained_weights(COCO_MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.772038Z","iopub.execute_input":"2021-12-17T18:30:57.772467Z","iopub.status.idle":"2021-12-17T18:30:57.778657Z","shell.execute_reply.started":"2021-12-17T18:30:57.772432Z","shell.execute_reply":"2021-12-17T18:30:57.777885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Configuration","metadata":{}},{"cell_type":"code","source":"HEIGHT = 520\nWIDTH = 704\nSHAPE = (HEIGHT, WIDTH)\n\nHEIGHT_TARGET = 576\nWIDTH_TARGET = 704\nSHAPE_TARGET = (HEIGHT_TARGET, WIDTH_TARGET)\n\nBATCH_SIZE = 1\nN_SAMPLES = train['image_id'].nunique()\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\nDEBUG = False\nDEBUG_SIZE = 100","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.781461Z","iopub.execute_input":"2021-12-17T18:30:57.782317Z","iopub.status.idle":"2021-12-17T18:30:57.811990Z","shell.execute_reply.started":"2021-12-17T18:30:57.782277Z","shell.execute_reply":"2021-12-17T18:30:57.811192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS_ALL = 2 if DEBUG else (10 * BATCH_SIZE)\nprint(f'EPOCHS_ALL: {EPOCHS_ALL}')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.813239Z","iopub.execute_input":"2021-12-17T18:30:57.813600Z","iopub.status.idle":"2021-12-17T18:30:57.819017Z","shell.execute_reply.started":"2021-12-17T18:30:57.813563Z","shell.execute_reply":"2021-12-17T18:30:57.818010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mask-RCNN Config","metadata":{}},{"cell_type":"code","source":"class CellConfig(Config):\n    \"\"\"Configuration for training on the cigarette butts dataset.\n    Derives from the base Config class and overrides values specific\n    to the cigarette butts dataset.\n    \"\"\"\n    \n    NAME = \"cell\"\n\n    # Set batch size to 1.\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = BATCH_SIZE\n    STEPS_PER_EPOCH = int(DEBUG_SIZE / BATCH_SIZE)  if DEBUG else int(N_SAMPLES / BATCH_SIZE)\n    \n    # Number of Classes\n    NUM_CLASSES = 1 + len(CELL_TYPES)\n\n    # Image Dimensions\n    IMAGE_MIN_DIM = HEIGHT_TARGET\n    IMAGE_MAX_DIM = WIDTH_TARGET\n    IMAGE_SHAPE = [HEIGHT_TARGET, WIDTH_TARGET, 3]\n    IMAGE_RESIZE_MODE = 'none'\n    \n    BACKBONE = 'efficientnetv2-b3'\n\n    # Training Structure\n    FPN_CLASSIF_FC_LAYERS_SIZE = 1024\n    TOP_DOWN_PYRAMID_SIZE = 256\n    RPN_ANCHOR_SCALES = (32, 64, 128, 256, 512)\n    # Regions of Interest\n    PRE_NMS_LIMIT = 6000\n    # Non Max Supression\n    POST_NMS_ROIS_TRAINING = 2000\n    POST_NMS_ROIS_INFERENCE = 2000\n    # Instances\n    MAX_GT_INSTANCES = 1000\n    TRAIN_ROIS_PER_IMAGE = 500\n    DETECTION_MAX_INSTANCES = 500\n    \n    # Thresholds\n    RPN_NMS_THRESHOLD = 0.70        # IoU Threshold for RPN proposals and GT\n    DETECTION_MIN_CONFIDENCE = 0.50 # Non-Background Confidence Threshold\n    DETECTION_NMS_THRESHOLD = 0.30  # IoU Threshold for ROI and GT\n    ROI_POSITIVE_RATIO = 0.33\n    \n    # Mini Mask\n    USE_MINI_MASK = True\n    MINI_MASK_SHAPE = (112, 112)\n    MASK_SHAPE = (56, 56)\n    \n    # DO NOT train Batch Normalization because of small batch size\n    # There are too few samples to correctly train the normalization\n    TRAIN_BN = False\n    \n    # Learning Rate\n    LEARNING_RATE = 0.002\n    WEIGHT_DECAY = 0.0\n    N_WARMUP_STEPS = 2\n    LR_SCHEDULE = True\n    \n    # Dataloader Queue Size (was set to 100 but resulted in OOM error)\n    MAX_QUEUE_SIZE = 10\n    \n    # Cache Items\n    CACHE = True\n    \n    DEBUG = DEBUG\n    \n    WORKERS = 0\n    \n    # Losses\n    LOSS_WEIGHTS = {\n        'rpn_class_loss': 1.0,    # is the class of the bbox correct? / RPN anchor classifier loss (Forground/Background)\n        'rpn_bbox_loss': 1.0,     # is the size of the bbox correct? / RPN bounding box loss graph (bbox of generic object)\n        'mrcnn_class_loss': 1.0,  # loss for the classifier head of Mask R-CNN (Background / specific class)\n        'mrcnn_bbox_loss': 1.0,   # is the size of the bounding box correct or not? / loss for Mask R-CNN bounding box refinement\n        'mrcnn_mask_loss': 1.0,   # is the class correct? is the pixel correctly assign to the class? / mask binary cross-entropy loss for the masks head\n    }\n    \n    ENABLE_XLA = True\n    \nconfig = CellConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:36:04.797384Z","iopub.execute_input":"2021-12-17T18:36:04.797658Z","iopub.status.idle":"2021-12-17T18:36:04.820975Z","shell.execute_reply.started":"2021-12-17T18:36:04.797630Z","shell.execute_reply":"2021-12-17T18:36:04.820122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def pad_image(image, constant_values):\n    pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n    pad_w = (WIDTH_TARGET - WIDTH) // 2\n    \n    if len(image.shape) == 3:\n        return np.pad(image, ((pad_h, pad_h), (pad_w, pad_w), (0,0)), constant_values=constant_values)\n    else:\n        return np.pad(image, ((pad_h, pad_h), (pad_w, pad_w)), constant_values=constant_values)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.847645Z","iopub.execute_input":"2021-12-17T18:30:57.848150Z","iopub.status.idle":"2021-12-17T18:30:57.855385Z","shell.execute_reply.started":"2021-12-17T18:30:57.848112Z","shell.execute_reply":"2021-12-17T18:30:57.854468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy Train Dataset to Working Directory\n!cp -r '/kaggle/input/sartorius-livecell-preprocessing-dataset/train' './'\n\n# Copy Preprocessed LZ4 Files\n!cp -r '/kaggle/input/sartorius-livecell-preprocessing-dataset/temp' './'","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:30:57.860596Z","iopub.execute_input":"2021-12-17T18:30:57.861418Z","iopub.status.idle":"2021-12-17T18:34:10.627437Z","shell.execute_reply.started":"2021-12-17T18:30:57.861378Z","shell.execute_reply":"2021-12-17T18:34:10.626396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CellDataset(utils.Dataset):\n\n    def load_data(self, image_ids, form, image_group):\n        self.image_group = image_group\n   \n        for i, name in enumerate(CELL_TYPES):\n            self.add_class('cell', 1 + i, name)\n       \n        # Add the image using the base method from utils.Dataset\n        for vf in [True, False]:\n            for hf in [True, False]:\n                for image in tqdm(image_ids):\n                    self.add_image('cell', \n                           image_id=image,\n                           path=(f'./{image_group}/{image}/{image}.png'),\n                           label = ID2CELL_LABEL[image],\n                           height=512, width=512,\n                           vertical_flip=vf, horizontal_flip=hf,\n                      )\n            \n            \n    def load_mask(self, image_id):\n        \"\"\" Load instance masks for the given image.\n        MaskRCNN expects masks in the form of a bitmap [height, width, instances].\n        Args:\n            image_id: The id of the image to load masks for\n        Returns:\n            masks: A bool array of shape [height, width, instance count] with\n                one mask per instance.\n            class_ids: a 1D array of class IDs of the instance masks.\n        \"\"\"\n    \n        info = self.image_info[image_id]\n        image_id = info['id']\n    \n        masks = np.load(f'./{self.image_group}/{image_id}/{image_id}_masks.npz')['v']\n\n        class_ids = np.load(f'./{self.image_group}/{image_id}/{image_id}_class_ids.npz')['v']\n        \n        return masks, class_ids","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:10.629288Z","iopub.execute_input":"2021-12-17T18:34:10.629805Z","iopub.status.idle":"2021-12-17T18:34:10.639405Z","shell.execute_reply.started":"2021-12-17T18:34:10.629766Z","shell.execute_reply":"2021-12-17T18:34:10.638775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_train = CellDataset()\ndataset_train.load_data(id_unique[:DEBUG_SIZE] if DEBUG else id_unique, 'png', 'train')\ndataset_train.prepare()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:10.640524Z","iopub.execute_input":"2021-12-17T18:34:10.641266Z","iopub.status.idle":"2021-12-17T18:34:11.025109Z","shell.execute_reply.started":"2021-12-17T18:34:10.641232Z","shell.execute_reply":"2021-12-17T18:34:11.024480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = dataset_train\nimage_ids = np.random.choice(dataset.image_ids, 10)\nfor image_id in tqdm(image_ids):\n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:11.028840Z","iopub.execute_input":"2021-12-17T18:34:11.030779Z","iopub.status.idle":"2021-12-17T18:34:33.717677Z","shell.execute_reply.started":"2021-12-17T18:34:11.030737Z","shell.execute_reply":"2021-12-17T18:34:33.716997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# Create model in training mode\n!mkdir 'model_checkpoints'\nmodel = modellib.MaskRCNN(mode='training', config=config, model_dir='model_checkpoints')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:33.718727Z","iopub.execute_input":"2021-12-17T18:34:33.719843Z","iopub.status.idle":"2021-12-17T18:34:50.806512Z","shell.execute_reply.started":"2021-12-17T18:34:33.719805Z","shell.execute_reply":"2021-12-17T18:34:50.805779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"init_with = \"coco\"\nexclude = [\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \"mrcnn_bbox\", \"mrcnn_mask\"]\nif 'efficientnetv2-' in config.BACKBONE:\n    exclude += [\n        \"fpn_c5p5\", \"fpn_c4p4\", \"fpn_c3p3\", \"fpn_c2p2\",\n    ]\n    \n# Modified Fully Connected Layer\nif config.FPN_CLASSIF_FC_LAYERS_SIZE != 1024:\n    print(f'Excluding FC layer')\n    exclude += [\n        \"mrcnn_class_conv1\", \"mrcnn_class_bn1\", \"mrcnn_class_conv2\", \"mrcnn_class_bn2\",\n    ]\n    \n# Modified Top Down Pyramid Size\nif config.TOP_DOWN_PYRAMID_SIZE != 256:\n    print(f'Excluding Top Down Pyramid Layers')\n    exclude += [\n        \"fpn_p2\", \"fpn_p3\", \"fpn_p4\", \"fpn_p5\",\n        \"rpn_model\",\n        \"mrcnn_class_conv1\",\n        \"mrcnn_mask_conv1\", \"mrcnn_mask_conv2\", \"mrcnn_mask_conv3\", \"mrcnn_mask_conv4\",\n        \"mrcnn_mask_bn1\", \"mrcnn_mask_bn2\", \"mrcnn_mask_bn3\", \"mrcnn_mask_bn4\",\n        \"mrcnn_mask_deconv\",\n    ]\n    \n# using coco weights\nmodel.load_weights(\n    COCO_MODEL_PATH,\n    by_name=True,\n    exclude=exclude,\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:50.808145Z","iopub.execute_input":"2021-12-17T18:34:50.808407Z","iopub.status.idle":"2021-12-17T18:34:59.663661Z","shell.execute_reply.started":"2021-12-17T18:34:50.808373Z","shell.execute_reply":"2021-12-17T18:34:59.662937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load EfficientNetV2 Weights Pretrained on Imagenet21K and Fine Tuned on ImageNet1K\nmodel.keras_model.layers[1].load_weights(f'/kaggle/input/sartorius-coco-models/Instance_Segmentation/efficientnetv2_model_checkpoints/{config.BACKBONE}-imagenet21k-ft1k.h5')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:34:59.665127Z","iopub.execute_input":"2021-12-17T18:34:59.665391Z","iopub.status.idle":"2021-12-17T18:35:03.061800Z","shell.execute_reply.started":"2021-12-17T18:34:59.665357Z","shell.execute_reply":"2021-12-17T18:35:03.061034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Summary","metadata":{}},{"cell_type":"code","source":"model.show_summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:03.063289Z","iopub.execute_input":"2021-12-17T18:35:03.063533Z","iopub.status.idle":"2021-12-17T18:35:05.287347Z","shell.execute_reply.started":"2021-12-17T18:35:03.063500Z","shell.execute_reply":"2021-12-17T18:35:05.286384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"model.plot_lr_schedule(EPOCHS_ALL)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:05.289361Z","iopub.execute_input":"2021-12-17T18:35:05.289736Z","iopub.status.idle":"2021-12-17T18:35:05.620510Z","shell.execute_reply.started":"2021-12-17T18:35:05.289665Z","shell.execute_reply":"2021-12-17T18:35:05.619803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Whole Model","metadata":{}},{"cell_type":"code","source":"start_train = time.time()\nhistory = model.train(\n    dataset_train, None, \n    learning_rate=config.LEARNING_RATE,\n    epochs=EPOCHS_ALL, \n    layers=\"all\",\n    augmentation=None,\n)\n\nend_train = time.time()\nminutes = round((end_train - start_train) / 60, 2)\nprint(f'Training took {minutes} minutes')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:05.621858Z","iopub.execute_input":"2021-12-17T18:35:05.622245Z","iopub.status.idle":"2021-12-17T18:35:56.963013Z","shell.execute_reply.started":"2021-12-17T18:35:05.622206Z","shell.execute_reply":"2021-12-17T18:35:56.961753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax):\n    values = history.history[metric]\n    plt.figure(figsize=(15, 8))\n    N_EPOCHS = len(values)\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n    x_ticks = np.arange(1, N_EPOCHS+1)\n        \n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label='train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label='train_best')\n    \n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n    plt.xlabel('epoch', fontsize=20, labelpad=10)\n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    plt.legend(prop={'size': 18})\n    plt.grid()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.964336Z","iopub.status.idle":"2021-12-17T18:35:56.965227Z","shell.execute_reply.started":"2021-12-17T18:35:56.964968Z","shell.execute_reply":"2021-12-17T18:35:56.964997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.966940Z","iopub.status.idle":"2021-12-17T18:35:56.967374Z","shell.execute_reply.started":"2021-12-17T18:35:56.967137Z","shell.execute_reply":"2021-12-17T18:35:56.967159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Region Proposal Network Foreground / Background Classifier\nplot_history_metric('rpn_class_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.968617Z","iopub.status.idle":"2021-12-17T18:35:56.969359Z","shell.execute_reply.started":"2021-12-17T18:35:56.969106Z","shell.execute_reply":"2021-12-17T18:35:56.969131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Region Proposal Network Bounding Box Loss\nplot_history_metric('rpn_bbox_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.971443Z","iopub.status.idle":"2021-12-17T18:35:56.972182Z","shell.execute_reply.started":"2021-12-17T18:35:56.971941Z","shell.execute_reply":"2021-12-17T18:35:56.971964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mask RCNN Head Class Classifier Background / specific class\nplot_history_metric('mrcnn_class_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.973479Z","iopub.status.idle":"2021-12-17T18:35:56.974342Z","shell.execute_reply.started":"2021-12-17T18:35:56.974110Z","shell.execute_reply":"2021-12-17T18:35:56.974132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Mask RCNN Head Bounding Box Loss\nplot_history_metric('mrcnn_bbox_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.975487Z","iopub.status.idle":"2021-12-17T18:35:56.976301Z","shell.execute_reply.started":"2021-12-17T18:35:56.976044Z","shell.execute_reply":"2021-12-17T18:35:56.976068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mask RCNN Head Object Mask Binary Cross Entropy Loss\nplot_history_metric('mrcnn_mask_loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.977524Z","iopub.status.idle":"2021-12-17T18:35:56.977950Z","shell.execute_reply.started":"2021-12-17T18:35:56.977728Z","shell.execute_reply":"2021-12-17T18:35:56.977748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"class InferenceConfig(CellConfig):\n    IMAGES_PER_GPU = 1\n    DETECTION_MAX_INSTANCES = 500\n    DETECTION_MIN_CONFIDENCE = 0.70    \n\ninference_config = InferenceConfig()\ninference_config.display()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.979154Z","iopub.status.idle":"2021-12-17T18:35:56.979895Z","shell.execute_reply.started":"2021-12-17T18:35:56.979651Z","shell.execute_reply":"2021-12-17T18:35:56.979675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode=\"inference\", config=inference_config, model_dir='model_checkpoints')","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.981161Z","iopub.status.idle":"2021-12-17T18:35:56.981764Z","shell.execute_reply.started":"2021-12-17T18:35:56.981513Z","shell.execute_reply":"2021-12-17T18:35:56.981535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'efficientnetv2-' in  inference_config.BACKBONE:\n    model.keras_model.layers[1].layers[-1].trainable = False","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.983007Z","iopub.status.idle":"2021-12-17T18:35:56.983565Z","shell.execute_reply.started":"2021-12-17T18:35:56.983325Z","shell.execute_reply":"2021-12-17T18:35:56.983348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = model.find_last()\n\n# Load trained weights (fill in path to trained weights here)\nprint('Loading weights from', model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.984598Z","iopub.status.idle":"2021-12-17T18:35:56.985256Z","shell.execute_reply.started":"2021-12-17T18:35:56.985000Z","shell.execute_reply":"2021-12-17T18:35:56.985023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Train Predictions","metadata":{}},{"cell_type":"code","source":"for file_path in glob.glob('./train/*/*.png')[:25]:\n    img = skimage.io.imread(file_path)\n    img = np.stack((img,img, img), axis=2)\n    results = model.detect([img], verbose=1)\n    r = results[0]\n    \n    # Image Id\n    image_id = file_path.split('/')[-1].split('.')[0]\n    print(f'image_id: {image_id}')\n    \n    mask = np.load(f'./train/{image_id}/{image_id}_masks.npz')['v']\n    mask = mask.sum(axis=2)\n    \n    plt.figure(figsize=(16,16))\n    plt.imshow(mask)\n    plt.show()\n    \n    visualize.display_instances(\n        img,\n        r['rois'],\n        r['masks'],\n        r['class_ids'], \n        ['BG'] + CELL_TYPES.tolist(),\n        r['scores'],\n        figsize=(16,16)\n    )","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.986776Z","iopub.status.idle":"2021-12-17T18:35:56.987182Z","shell.execute_reply.started":"2021-12-17T18:35:56.986963Z","shell.execute_reply":"2021-12-17T18:35:56.986984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clear All Cached Training Samples\nmodel.clear_cache()\n# Clear all train images in working directory\n!rm -rf train","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:35:56.988341Z","iopub.status.idle":"2021-12-17T18:35:56.988814Z","shell.execute_reply.started":"2021-12-17T18:35:56.988565Z","shell.execute_reply":"2021-12-17T18:35:56.988587Z"},"trusted":true},"execution_count":null,"outputs":[]}]}