{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detectron2 for Happy Whales 🐋\n![Whale](https://c.tenor.com/6_3NxH30Ud4AAAAM/whale-blue-whale.gif)\n\nThis notebook is for training the segmentation model to segment out whales & dolphins out of the images.\nNote - This notebook has been written & the model has been trained only with whales.","metadata":{"id":"QHnVupBBn9eR"}},{"cell_type":"markdown","source":"## Install detectron2","metadata":{"id":"vM54r6jlKTII"}},{"cell_type":"code","source":"!pip3 install pyyaml==5.1\n!python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"id":"FsePPpwZSmqt","outputId":"09f9973a-2813-40a9-eeeb-24e0cdedf98a","execution":{"iopub.status.busy":"2022-02-14T17:02:18.782564Z","iopub.execute_input":"2022-02-14T17:02:18.782843Z","iopub.status.idle":"2022-02-14T17:05:40.130695Z","shell.execute_reply.started":"2022-02-14T17:02:18.782812Z","shell.execute_reply":"2022-02-14T17:05:40.129855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Common Required Libraries for the notebook 🤔","metadata":{}},{"cell_type":"code","source":"import torch\nimport detectron2\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\nimport numpy as np\nimport os, json, cv2, random\n\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog","metadata":{"id":"ZyAvNCJMmvFF","execution":{"iopub.status.busy":"2022-02-14T17:07:40.075468Z","iopub.execute_input":"2022-02-14T17:07:40.075841Z","iopub.status.idle":"2022-02-14T17:07:41.592593Z","shell.execute_reply.started":"2022-02-14T17:07:40.075790Z","shell.execute_reply":"2022-02-14T17:07:41.591859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing the dataset and structuring it for training 😬\n\nWe were too lazy to do train-test split since we were trying out the notebook just for fun xD\n\nThe images are annotated using [LabelMe](http://https://github.com/wkentaro/labelme)","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/happywhales-labelme-segmentation-dataset . ","metadata":{"id":"wKyL-YwBGagD","outputId":"2ccb4e31-f9d7-4960-b54b-5282dd12c64a","execution":{"iopub.status.busy":"2022-02-14T17:07:44.147123Z","iopub.execute_input":"2022-02-14T17:07:44.147694Z","iopub.status.idle":"2022-02-14T17:07:47.539597Z","shell.execute_reply.started":"2022-02-14T17:07:44.147657Z","shell.execute_reply":"2022-02-14T17:07:47.538704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir Final_Data\n!mkdir Final_Data/train/\n!mkdir Final_Data/val/","metadata":{"id":"aWf7y7pHGlTH","execution":{"iopub.status.busy":"2022-02-14T17:07:49.423975Z","iopub.execute_input":"2022-02-14T17:07:49.424830Z","iopub.status.idle":"2022-02-14T17:07:51.497575Z","shell.execute_reply.started":"2022-02-14T17:07:49.424789Z","shell.execute_reply":"2022-02-14T17:07:51.496559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ./happywhales-labelme-segmentation-dataset/test/*.jpg Final_Data/train \n!cp ./happywhales-labelme-segmentation-dataset/test/*.json Final_Data/train\n!cp ./happywhales-labelme-segmentation-dataset/test/*.jpg Final_Data/val\n!cp ./happywhales-labelme-segmentation-dataset/test/*.json Final_Data/val","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:07:58.514437Z","iopub.execute_input":"2022-02-14T17:07:58.514705Z","iopub.status.idle":"2022-02-14T17:08:01.434545Z","shell.execute_reply.started":"2022-02-14T17:07:58.514675Z","shell.execute_reply":"2022-02-14T17:08:01.433567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To convert the labelme annotations to coco for the reference of this training notebook.\n!git clone https://github.com/Tony607/labelme2coco","metadata":{"id":"gNkP5gbWGqB5","outputId":"4987d69b-ee3e-4b08-b06e-198e9ef1f1d4","execution":{"iopub.status.busy":"2022-02-14T17:08:01.436716Z","iopub.execute_input":"2022-02-14T17:08:01.436948Z","iopub.status.idle":"2022-02-14T17:08:03.355715Z","shell.execute_reply.started":"2022-02-14T17:08:01.436910Z","shell.execute_reply":"2022-02-14T17:08:03.354836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 install labelme","metadata":{"id":"oiyuPaJHGrpq","outputId":"90970224-816b-4004-bc52-6f277ed42cd4","execution":{"iopub.status.busy":"2022-02-14T17:08:03.357133Z","iopub.execute_input":"2022-02-14T17:08:03.357397Z","iopub.status.idle":"2022-02-14T17:08:37.042660Z","shell.execute_reply.started":"2022-02-14T17:08:03.357359Z","shell.execute_reply":"2022-02-14T17:08:37.041743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm Final_Data/train/train.json Final_Data/train/val.json  \n!python3 labelme2coco/labelme2coco.py /kaggle/working/Final_Data/train/ --output ./test_whale.json","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:08:37.045099Z","iopub.execute_input":"2022-02-14T17:08:37.045403Z","iopub.status.idle":"2022-02-14T17:08:51.051318Z","shell.execute_reply.started":"2022-02-14T17:08:37.045362Z","shell.execute_reply":"2022-02-14T17:08:51.050453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the dataset 🧐","metadata":{}},{"cell_type":"code","source":"from detectron2.data.datasets import register_coco_instances\n\nfor d in [\"train\", \"val\"]:\n    register_coco_instances(f\"whale_{d}\", {}, f\"test_whale.json\", f\"Final_Data/{d}\")","metadata":{"id":"llQeP_RpGtpx","execution":{"iopub.status.busy":"2022-02-14T17:09:00.764480Z","iopub.execute_input":"2022-02-14T17:09:00.764759Z","iopub.status.idle":"2022-02-14T17:09:00.769884Z","shell.execute_reply.started":"2022-02-14T17:09:00.764726Z","shell.execute_reply":"2022-02-14T17:09:00.768822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualising the annotations 😇","metadata":{}},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\n\ndataset_dicts = DatasetCatalog.get(\"whale_train\")\nwhale_metadata = MetadataCatalog.get(\"whale_train\")\n\nfor d in random.sample(dataset_dicts, 3):\n    img = cv2.imread(d[\"file_name\"])\n    v = Visualizer(img[:, :, ::-1], metadata=whale_metadata, scale=0.5)\n    v = v.draw_dataset_dict(d)\n    plt.figure(figsize = (14, 10))\n    plt.imshow(cv2.cvtColor(v.get_image()[:, :, ::-1], cv2.COLOR_BGR2RGB))\n    plt.show()","metadata":{"id":"TZoWwLtpG1o0","outputId":"d3c92ba1-8b45-4ca3-d1a9-9da92cfda868","execution":{"iopub.status.busy":"2022-02-14T17:09:04.499948Z","iopub.execute_input":"2022-02-14T17:09:04.500784Z","iopub.status.idle":"2022-02-14T17:09:07.073273Z","shell.execute_reply.started":"2022-02-14T17:09:04.500747Z","shell.execute_reply":"2022-02-14T17:09:07.072590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training 😭\n### The model used here is Mask RCNN R50\n\np.s. - Takes about 30 minutes to train. Holod your horses.","metadata":{}},{"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\nfrom detectron2.config import get_cfg\nimport os\n\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"whale_train\",)\ncfg.DATASETS.TEST = ()\ncfg.DATALOADER.NUM_WORKERS = 4\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\ncfg.SOLVER.IMS_PER_BATCH = 8\ncfg.SOLVER.BASE_LR = 0.00010\ncfg.SOLVER.MAX_ITER = 500\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 1\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = DefaultTrainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"id":"0CMQ6mfGHD9-","outputId":"459562ec-bd72-4c02-f7fe-2144ea44fdb4","execution":{"iopub.status.busy":"2022-02-14T17:09:20.006477Z","iopub.execute_input":"2022-02-14T17:09:20.006749Z","iopub.status.idle":"2022-02-14T17:27:07.221151Z","shell.execute_reply.started":"2022-02-14T17:09:20.006720Z","shell.execute_reply":"2022-02-14T17:27:07.220371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference 🚀\n### Passing single image and visualising the output","metadata":{}},{"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\n\ncfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4\npredictor = DefaultPredictor(cfg)\n\n#Passing random image path as input for test :)\nim = cv2.imread(\"../input/happy-whale-and-dolphin/test_images/000e246888710c.jpg\")\noutputs = predictor(im)\nprint(outputs)\nv = Visualizer(im[:, :, ::-1],\n               metadata=whale_metadata, \n               scale=0.8, \n               instance_mode=ColorMode.IMAGE_BW \n)\n\n\nv = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\nplt.figure(figsize = (14, 10))\nplt.imshow(cv2.cvtColor(v.get_image()[:, :, ::-1], cv2.COLOR_BGR2RGB))\nplt.show()","metadata":{"id":"IafjO2nLHWAv","outputId":"6a6e1c7b-2abe-4484-884a-0ef6ff021ea3","execution":{"iopub.status.busy":"2022-02-14T17:27:16.986079Z","iopub.execute_input":"2022-02-14T17:27:16.986622Z","iopub.status.idle":"2022-02-14T17:27:18.938835Z","shell.execute_reply.started":"2022-02-14T17:27:16.986581Z","shell.execute_reply":"2022-02-14T17:27:18.938176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Under Construction 🥷\n## Using segments to plot the mask on a binary image and EDA.","metadata":{}}]}