{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h2><font color='green'>This is Tensorflow(2.4) implementation of Mask RCNN which is used for Instance Segmentation</h2>\n\n**<h3><font color='red'>If You Find It Useful, Please Don't Forget to Upvote</h3>**\n    \n**<h2>Inference Notebook:</h2>**\nhttps://www.kaggle.com/susnato/mask-rcnn-inference-in-tensorflow","metadata":{}},{"cell_type":"markdown","source":"**<h3><font color='green'>First Install Requirements and Download the coco-weights</h3>**","metadata":{}},{"cell_type":"code","source":"!pip install keras==2.4.0\n!pip install tensorflow==2.4.0\n!pip install -U scikit-image==0.16.2\n!pip install ../input/leekunhee-mask-rcnn/Mask_RCNN-master\n\n###coco-weights\n!wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-02T12:07:48.018888Z","iopub.execute_input":"2021-11-02T12:07:48.019890Z","iopub.status.idle":"2021-11-02T12:11:34.300929Z","shell.execute_reply.started":"2021-11-02T12:07:48.019692Z","shell.execute_reply":"2021-11-02T12:11:34.300115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='green'>Imports</h3>**","metadata":{}},{"cell_type":"code","source":"import os\nimport keras\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom mrcnn import config, utils\nimport matplotlib.pyplot as plt \nfrom numpy import zeros, asarray\nfrom mrcnn import model as modellib\n\n%matplotlib inline\n\nseed=42\nnp.random.seed(seed)\ntf.random.set_seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\ntrain_fol = '../input/sartorius-cell-instance-segmentation/train'\ntest_fol = '../input/sartorius-cell-instance-segmentation/test'\ntrain_csv = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\ntrain_csv = train_csv.drop(['cell_type', 'plate_time', 'sample_date', 'sample_id', 'elapsed_timedelta', 'height', 'width'], axis = 1)\nsub_csv = pd.read_csv('../input/sartorius-cell-instance-segmentation/sample_submission.csv')\n\nprint(tf.__version__, keras.__version__, tf.keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:11:34.303813Z","iopub.execute_input":"2021-11-02T12:11:34.304308Z","iopub.status.idle":"2021-11-02T12:11:37.869946Z","shell.execute_reply.started":"2021-11-02T12:11:34.304268Z","shell.execute_reply":"2021-11-02T12:11:37.869153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='gray'>SPLIT THE DATASET (80-20)</h3>**","metadata":{}},{"cell_type":"code","source":"os.makedirs('./dataset', exist_ok=True)\nos.makedirs('./dataset/train', exist_ok=True)\nos.makedirs('./dataset/val', exist_ok=True)\n\nt = len(os.listdir(train_fol))#\nids = np.array(os.listdir(train_fol))\n#80-20\nix = np.arange(t)\nnp.random.shuffle(ix)\ntrix = ids[ix[:484]]#80%\nvlix = ids[ix[484:]]#20%\n[shutil.copyfile(os.path.join(train_fol, etrix), f'./dataset/train/{etrix}') for etrix in trix]\n[shutil.copyfile(os.path.join(train_fol, evlix), f'./dataset/val/{evlix}') for evlix in vlix]\n\nprint(f\"no of files in train: {len(os.listdir('./dataset/train/'))}, val: {len(os.listdir('./dataset/val/'))}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:11:37.871159Z","iopub.execute_input":"2021-11-02T12:11:37.871428Z","iopub.status.idle":"2021-11-02T12:11:41.110115Z","shell.execute_reply.started":"2021-11-02T12:11:37.871392Z","shell.execute_reply":"2021-11-02T12:11:41.108406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='purple'>Change the config as required</h3>**","metadata":{}},{"cell_type":"code","source":"class KaggleSartoriusConfig(config.Config):\n    NAME = \"kaggle_Sartorius_cfg\"\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    NUM_CLASSES = 2\n    STEPS_PER_EPOCH = 484\n    VALIDATION_STEPS = 122\n    USE_MINI_MASK = False\n#Display The Config\nkaggle_sartorius_model_config = KaggleSartoriusConfig()\nkaggle_sartorius_model_config.display()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:23:48.632267Z","iopub.execute_input":"2021-11-02T12:23:48.632547Z","iopub.status.idle":"2021-11-02T12:23:48.647863Z","shell.execute_reply.started":"2021-11-02T12:23:48.632514Z","shell.execute_reply":"2021-11-02T12:23:48.646907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='orange'>Define the Custom Dataset for with load_mask</h3>**","metadata":{}},{"cell_type":"code","source":"class KaggleSartoriusDataset(utils.Dataset):\n    def load_dataset(self, dataset_dir, is_train=True):\n        # Adds information (image ID, image path, and annotation file path) about each image in a dictionary.\n        self.add_class(\"dataset\", 1, \"cell\")\n        if is_train:\n            images_dir = dataset_dir+'/train/' \n        else:\n            images_dir = dataset_dir+'/val/'\n\n        for filename in os.listdir(images_dir):\n            image_id = filename[:-4]\n            img_path = images_dir + filename\n            self.add_image('dataset', image_id=image_id, path=img_path, annotation=None)\n\n    # Loads the binary masks for an image.\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        \n        train_annots = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\n        annot = train_annots[train_annots.id==info['id']].annotation.values\n\n        class_ids = list()\n        masks = np.zeros([520*704, len(annot)], dtype=np.uint8)\n        for i, rle in enumerate(annot):\n            rle = np.array(rle.split(' ')).reshape(-1, 2)\n            for r in rle:\n                masks[int(r[0]):int(r[0])+int(r[1]), i] = 1\n            class_ids.append(self.class_names.index('cell'))\n        masks = masks.reshape(520, 704, len(annot))\n        return masks, asarray(class_ids, dtype='int32')\n\n#mAP Callback\nclass MeanAveragePrecisionCallback(tf.keras.callbacks.Callback):\n    def __init__(self, train_model: modellib.MaskRCNN, inference_model: modellib.MaskRCNN, dataset: utils.Dataset,\n                 dataset_limit: int = None, verbose: int = 1):\n        super().__init__()\n        self.train_model = train_model\n        self.inference_model = inference_model\n        self.dataset = dataset\n        #self.calculate_at_every_X_epoch = calculate_at_every_X_epoch\n        self.dataset_limit = len(self.dataset.image_ids)\n        if dataset_limit is not None:\n            self.dataset_limit = dataset_limit\n        self.dataset_image_ids = self.dataset.image_ids.copy()\n\n        if inference_model.config.BATCH_SIZE != 1:\n            raise ValueError(\"This callback only works with the bacth size of 1\")\n\n        self._verbose_print = print if verbose > 0 else lambda *a, **k: None\n\n    def on_epoch_end(self, epoch, logs=None):\n        #if epoch > 0 and epoch % self.calculate_at_every_X_epoch == 0:\n        self._verbose_print(\"Calculating mAP...\")\n        self._load_weights_for_model()\n\n        mAPs = self._calculate_mean_average_precision()\n        mAP = np.mean(mAPs)\n\n        if logs is not None:\n            logs[\"val_mean_average_precision\"] = mAP\n\n        self._verbose_print(\"mAP at epoch {0} is: {1}\".format(epoch+1, mAP))\n\n        super().on_epoch_end(epoch, logs)\n\n    def _load_weights_for_model(self):\n        last_weights_path = self.train_model.find_last()\n        self._verbose_print(\"Loaded weights for the inference model (last checkpoint of the train model): {0}\".format(\n            last_weights_path))\n        self.inference_model.load_weights(last_weights_path,\n                                          by_name=True)\n\n    def _calculate_mean_average_precision(self):\n        mAPs = []\n\n        # Use a random subset of the data when a limit is defined\n        np.random.shuffle(self.dataset_image_ids)\n\n        for image_id in self.dataset_image_ids:\n            image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(self.dataset, self.inference_model.config,\n                                                                             image_id)\n            #molded_images = np.expand_dims(mold_image(image, self.inference_model.config), 0)\n            results = self.inference_model.detect([image], verbose=0)\n            r = results[0]\n            AP_range = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r[\"rois\"],\n                                           r[\"class_ids\"], r[\"scores\"], r['masks'], verbose=0)\n            #range : Default is 0.5 to 0.95 with increments of 0.05 if you want to view the AP at each\n            #        range then use verbose=1\n            mAPs.append(AP_range)\n\n        return np.array(mAPs)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:25:57.483647Z","iopub.execute_input":"2021-11-02T12:25:57.483924Z","iopub.status.idle":"2021-11-02T12:25:57.507572Z","shell.execute_reply.started":"2021-11-02T12:25:57.483892Z","shell.execute_reply":"2021-11-02T12:25:57.506278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='brown'>BEGIN THE TRAINING AND SAVE THE WEIGHTS</h3>**","metadata":{}},{"cell_type":"code","source":"import logging\nlogging.getLogger('tensorflow').disabled = True","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:11:41.172863Z","iopub.execute_input":"2021-11-02T12:11:41.173338Z","iopub.status.idle":"2021-11-02T12:11:41.183831Z","shell.execute_reply.started":"2021-11-02T12:11:41.173300Z","shell.execute_reply":"2021-11-02T12:11:41.182858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Train Set \nn_epochs = 6\ntrain_dataset = KaggleSartoriusDataset()\ntrain_dataset.load_dataset(dataset_dir='./dataset', is_train=True)\ntrain_dataset.prepare()\n\n# Define Validation Set\nvalidation_dataset = KaggleSartoriusDataset()\nvalidation_dataset.load_dataset(dataset_dir='./dataset', is_train=False)\nvalidation_dataset.prepare()\n\n# Build the Mask R-CNN Model Architecture\ntrain_model = modellib.MaskRCNN(mode='training', \n                                 model_dir='./trained_mask_rcnn/', \n                                 config=kaggle_sartorius_model_config)\ninfer_model = modellib.MaskRCNN(mode=\"inference\", \n                                 model_dir='./infer_mask_rcnn/',\n                                 config=kaggle_sartorius_model_config)\nmean_average_precision_callback = MeanAveragePrecisionCallback(train_model, infer_model, validation_dataset)\n\n#load coco-weights\ntrain_model.load_weights(filepath='./mask_rcnn_coco.h5', \n                   by_name=True, \n                   exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\",  \"mrcnn_bbox\", \"mrcnn_mask\"])\ntrain_model.train(train_dataset=train_dataset, \n                  val_dataset=validation_dataset, \n                  learning_rate=kaggle_sartorius_model_config.LEARNING_RATE, \n                  epochs=n_epochs, \n                  custom_callbacks=[mean_average_precision_callback] ,\n                  layers='heads')\n\nmodel_weights_path = f'./model_{n_epochs}.h5'\ntrain_model.keras_model.save_weights(model_weights_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:11:41.187777Z","iopub.execute_input":"2021-11-02T12:11:41.188916Z","iopub.status.idle":"2021-11-02T12:20:59.535458Z","shell.execute_reply.started":"2021-11-02T12:11:41.188637Z","shell.execute_reply":"2021-11-02T12:20:59.534639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h3><font color='indigo'>Inference To View How the model did</h3>**","metadata":{}},{"cell_type":"code","source":"#Load Model in Inference Mode\ninfer_model.load_weights(train_model.find_last(), by_name=True)\nprint('Weights Loaded')","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:20:59.537466Z","iopub.execute_input":"2021-11-02T12:20:59.537790Z","iopub.status.idle":"2021-11-02T12:21:00.224567Z","shell.execute_reply.started":"2021-11-02T12:20:59.537751Z","shell.execute_reply":"2021-11-02T12:21:00.223822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test on a random image\nfrom mrcnn import visualize\nfrom mrcnn.model import log\nfrom mrcnn import model as modellib\n\ndef get_ax(rows=1, cols=1, size=8):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \"\"\"\n    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return ax\n\ndef visualize_model_performence(image_id :int):\n    #image_id = np.random.choice(validation_dataset.image_ids)\n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(validation_dataset, kaggle_sartorius_model_config, image_id,)\n\n\n    print('GROUND TRUTH MASK:->', end='\\n')\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                validation_dataset.class_names, figsize=(8, 8), show_bbox=False)\n\n    results = infer_model.detect([original_image], verbose=1)\n    r = results[0]\n    print(\"MODEL OUTPUT MASK:->\")\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                validation_dataset.class_names, r['scores'], ax=get_ax(),show_bbox=False)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:21:46.125752Z","iopub.execute_input":"2021-11-02T12:21:46.126023Z","iopub.status.idle":"2021-11-02T12:21:46.142153Z","shell.execute_reply.started":"2021-11-02T12:21:46.125994Z","shell.execute_reply":"2021-11-02T12:21:46.141436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Id's Available:- \", validation_dataset.image_ids)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:21:48.941179Z","iopub.execute_input":"2021-11-02T12:21:48.941450Z","iopub.status.idle":"2021-11-02T12:21:48.947355Z","shell.execute_reply.started":"2021-11-02T12:21:48.941419Z","shell.execute_reply":"2021-11-02T12:21:48.946550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_model_performence(5)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:21:50.583444Z","iopub.execute_input":"2021-11-02T12:21:50.584357Z","iopub.status.idle":"2021-11-02T12:22:16.819110Z","shell.execute_reply.started":"2021-11-02T12:21:50.584312Z","shell.execute_reply":"2021-11-02T12:22:16.818432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_model_performence(20)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T12:22:16.820774Z","iopub.execute_input":"2021-11-02T12:22:16.821121Z","iopub.status.idle":"2021-11-02T12:22:20.253965Z","shell.execute_reply.started":"2021-11-02T12:22:16.821086Z","shell.execute_reply":"2021-11-02T12:22:20.253314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**<h1>Work in progress 🚧</h1>**\n\n**<h1>TO-DO List</h1>**\n<h2>1)ADD \"mAP\" METRIC(✓)<br>\n2)A Test Notebook (✓)<br>\n3)W&B Support</h2>","metadata":{}},{"cell_type":"markdown","source":"**<h3>references</h3>**\nref:- https://github.com/leekunhee/Mask_RCNN","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}