{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook contains data and code for Kaggle competition: [TensorFlow - Help Protect the Great Barrier Reef](https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview) and follows documentation: [Training Custom Object Detector](https://tensorflow-object-detection-api-tutorial.readthedocs.io/en/latest/training.html) from TensorFlow Object Detection API Tutorial.\n\nAdditional code references:  \nhttps://www.kaggle.com/khanhlvg/cots-detection-w-tensorflow-object-detection-api#Prepare-the-training-dataset  \nhttps://www.kaggle.com/andradaolteanu/greatbarrierreef-full-guide-to-bboxaugmentation","metadata":{}},{"cell_type":"markdown","source":"### Install Object Detection API in Kaggle notebook\n- To successfully install and import Object Detection API, I followed codes below:  \n\n```python\n!pip uninstall tf-models-official --yes\n!pip install tensorflow==2.8.0\n!pip install tf-models-official==2.8.0\n\n!git clone --depth 1 https://github.com/tensorflow/models\n\n%%bash\ncd models/research/\nprotoc object_detection/protos/*.proto --python_out=.\n\n%%bash\ncd models/research/\ncp object_detection/packages/tf2/setup.py .\npython -m pip install --use-feature=2020-resolver .\n\n### Now the API installed\n### you can run model_builder_tf2_test.py to check installation here\n\n### httplib2 needed to import\n!pip3 install httplib2==0.20.2\n```","metadata":{}},{"cell_type":"markdown","source":"## Import dependencies","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport time\nimport os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n%matplotlib inline\nimport io\nfrom PIL import Image\ntf.get_logger().setLevel('ERROR')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-15T08:49:12.446642Z","iopub.execute_input":"2022-02-15T08:49:12.446912Z","iopub.status.idle":"2022-02-15T08:49:12.454400Z","shell.execute_reply.started":"2022-02-15T08:49:12.446883Z","shell.execute_reply":"2022-02-15T08:49:12.453502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Load data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\nprint(train_df.head())\nprint(train_df.info())\nprint(\"object dtype check for `image_id` and `annotations`\")\nprint(type(train_df.image_id[0]), type(train_df.annotations[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:13.382915Z","iopub.execute_input":"2022-02-15T08:49:13.383424Z","iopub.status.idle":"2022-02-15T08:49:13.463742Z","shell.execute_reply.started":"2022-02-15T08:49:13.383386Z","shell.execute_reply":"2022-02-15T08:49:13.462476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print random sample\nprint(train_df.sample(5)) # .iloc[:, [0, 2, 4]]","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:14.282178Z","iopub.execute_input":"2022-02-15T08:49:14.284237Z","iopub.status.idle":"2022-02-15T08:49:14.293208Z","shell.execute_reply.started":"2022-02-15T08:49:14.284199Z","shell.execute_reply":"2022-02-15T08:49:14.292405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Loading `train.csv` file, `image_id` and `annotations` are converted to `str` type object. `image_id` is represented as `video_id-video_frame`-like encoding for all observations, while `annotations` is represented as a list of dict-type annotations. Re-open data using `ast.literal_eval` as converter and drop redundant column.","metadata":{}},{"cell_type":"code","source":"from ast import literal_eval\ntrain_df = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\",\n                    converters={\"annotations\": literal_eval})\ntrain_df.drop(['image_id'], axis=1, inplace=True)\nprint(train_df.info())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:16.122921Z","iopub.execute_input":"2022-02-15T08:49:16.123367Z","iopub.status.idle":"2022-02-15T08:49:16.640910Z","shell.execute_reply.started":"2022-02-15T08:49:16.123329Z","shell.execute_reply":"2022-02-15T08:49:16.640146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Grouped by `video_id` and `sequence`\nprint(train_df.groupby(['video_id', 'sequence']).size())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:16.642501Z","iopub.execute_input":"2022-02-15T08:49:16.642743Z","iopub.status.idle":"2022-02-15T08:49:16.653472Z","shell.execute_reply.started":"2022-02-15T08:49:16.642709Z","shell.execute_reply":"2022-02-15T08:49:16.652581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize image sequence with annotation bboxes","metadata":{}},{"cell_type":"code","source":"def show_image(path, annot, axs=None):\n    '''Shows an image and marks any starfish annotated within the frame.\n    path: full path to the .jpg image\n    annot: list of the annotation for the coordinates of starfish'''\n    \n    # This is in case we plot only 1 image\n    if axs==None:\n        fig, axs = plt.subplots(figsize=(23, 8))\n    img = plt.imread(path)\n    axs.imshow(img)\n    if annot:\n        for a in annot:\n            rect = patches.Rectangle((a[\"x\"], a[\"y\"]), a[\"width\"], a[\"height\"], \n                                     linewidth=3, edgecolor=\"#FF6103\", facecolor='none')\n            axs.add_patch(rect)\n    axs.axis(\"off\")\n\ndef show_multiple_images(seq_id, frame_no): # 6 images\n    '''Shows multiple images within a sequence.\n    seq_id: a number corresponding with the sequence unique ID\n    frame_no: a number of first frame to plot'''\n    \n    # Select image paths & their annotations\n    sample = train_df[(train_df[\"sequence\"]==seq_id) &\n                      (train_df[\"sequence_frame\"]>=frame_no) &\n                      (train_df[\"sequence_frame\"]<=frame_no+5)]\n    paths = []\n    annotations = []\n    IMAGE_DIR = \"../input/tensorflow-great-barrier-reef/train_images\"\n    for vid, _, vframe, _, annot in sample.values:\n        paths.append(os.path.join(IMAGE_DIR, \"video_\"+str(vid), str(vframe)+'.jpg'))\n        annotations.append(annot)    \n    # Plot\n    fig, axs = plt.subplots(2, 3, figsize=(23, 10))\n    axs = axs.flatten()\n    fig.suptitle(f\"Showing consecutive frames for Sequence ID: {seq_id}\", fontsize = 20)\n\n    for k, (path, annot) in enumerate(zip(paths, annotations)):\n        axs[k].set_title(f\"Frame No: {frame_no+k}\", fontsize = 12)\n        show_image(path, annot, axs[k])\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:19.463563Z","iopub.execute_input":"2022-02-15T08:49:19.463899Z","iopub.status.idle":"2022-02-15T08:49:19.479363Z","shell.execute_reply.started":"2022-02-15T08:49:19.463863Z","shell.execute_reply":"2022-02-15T08:49:19.478522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_id, frame_no = 60754, 698\nshow_multiple_images(seq_id, frame_no)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:20.605051Z","iopub.execute_input":"2022-02-15T08:49:20.605674Z","iopub.status.idle":"2022-02-15T08:49:22.456335Z","shell.execute_reply.started":"2022-02-15T08:49:20.605631Z","shell.execute_reply":"2022-02-15T08:49:22.455494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Prepare dataset for Tensorflow Object Detection API","metadata":{}},{"cell_type":"markdown","source":"- Generate label_map.pbtxt","metadata":{}},{"cell_type":"code","source":"# Create a label map to map between label index and human-readable label name.\nlabel_map_str = \"\"\"item {\n  id: 1\n  name: 'starfish'\n}\"\"\"\nwith open('label_map.pbtxt', 'w') as f: # created on current working dir\n    f.write(label_map_str)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:25.160102Z","iopub.execute_input":"2022-02-15T08:49:25.160365Z","iopub.status.idle":"2022-02-15T08:49:25.165311Z","shell.execute_reply.started":"2022-02-15T08:49:25.160336Z","shell.execute_reply":"2022-02-15T08:49:25.164624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- From dataset `train_df`, extract annotated observations and generate tfrecord file\n    - Save 20% of dataset for evaluation","metadata":{}},{"cell_type":"code","source":"mask = train_df.annotations.isin([[]])\nannot_df = train_df[~mask]\nfrom sklearn.model_selection import train_test_split\ntrain_annot_df, test_annot_df = train_test_split(annot_df, test_size=0.2, random_state=1243)\nprint(\"Total annotated images:{} \\nNumber of Train images:{} \\nNumber of Test images:{}\".format(len(annot_df), len(train_annot_df), len(test_annot_df)))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:26.512310Z","iopub.execute_input":"2022-02-15T08:49:26.512883Z","iopub.status.idle":"2022-02-15T08:49:26.642010Z","shell.execute_reply.started":"2022-02-15T08:49:26.512841Z","shell.execute_reply":"2022-02-15T08:49:26.641261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# created on current working dir\ntrain_annot_df.to_csv('train_annot_df.csv', index=None)\ntest_annot_df.to_csv('test_annot_df.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:27.691962Z","iopub.execute_input":"2022-02-15T08:49:27.692750Z","iopub.status.idle":"2022-02-15T08:49:27.749291Z","shell.execute_reply.started":"2022-02-15T08:49:27.692700Z","shell.execute_reply":"2022-02-15T08:49:27.748579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Convert the training and validation dataset into TFRecord format as required by the TensorFlow Object Detection API. You can find [installation guide](https://tensorflow-object-detection-api-tutorial.readthedocs.io/en/latest/install.html) here.\n\n- We will use pretrained model EfficientDet-D0:  \n        Map bounding box {xmin, ymin, width, height} for xmin, ymin in range of $[0, 255)$, as {xmin, ymin, xmax, ymax} for values in range of $[0, 1)$ dividing by corresponding axis value.","metadata":{}},{"cell_type":"code","source":"from object_detection.utils import dataset_util\ndef class_text_to_int(cname):\n    if cname == 'starfish':\n        return 1\n    else:\n        None\ndef generate_tfrecord(dataframe, output_path):\n    writer = tf.io.TFRecordWriter(output_path)\n    for v in dataframe.values:\n        vid, _, fpath, _, annot = v\n        IMAGE_DIR = \"../input/tensorflow-great-barrier-reef/train_images\"\n        fpath = os.path.join(IMAGE_DIR, 'video_'+str(vid), str(fpath)+'.jpg')\n        with tf.io.gfile.GFile(fpath, 'rb') as fid:\n            encoded_jpg = fid.read()\n        encoded_jpg_io = io.BytesIO(encoded_jpg)\n        image = Image.open(encoded_jpg_io)\n        width, height = image.size # (1280, 720)\n        filename = fpath.split(os.sep)[-1]\n        filename = filename.encode('utf8')\n        cname = 'starfish'\n        image_format = b'jpg'\n        xmins = []\n        xmaxs = []\n        ymins = []\n        ymaxs = []\n        classes_text = []\n        classes = []\n        for box in annot:\n            xmin, ymin, w, h = box.values()\n            xmax, ymax = xmin + w, ymin + h\n            xmins.append(xmin / width)\n            xmaxs.append(xmax / width)\n            ymins.append(ymin / height)\n            ymaxs.append(ymax / height)\n            classes_text.append(cname.encode('utf8'))\n            classes.append(class_text_to_int(cname))\n        tf_example = tf.train.Example(features=tf.train.Features(feature={\n            'image/height': dataset_util.int64_feature(height),\n            'image/width': dataset_util.int64_feature(width),\n            'image/filename': dataset_util.bytes_feature(filename),\n            'image/source_id': dataset_util.bytes_feature(filename),\n            'image/encoded': dataset_util.bytes_feature(encoded_jpg),\n            'image/format': dataset_util.bytes_feature(image_format),\n            'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n            'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n            'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n            'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n            'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n            'image/object/class/label': dataset_util.int64_list_feature(classes),\n        }))\n        writer.write(tf_example.SerializeToString())\n    writer.close()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:36.400819Z","iopub.execute_input":"2022-02-15T08:49:36.401551Z","iopub.status.idle":"2022-02-15T08:49:36.413884Z","shell.execute_reply.started":"2022-02-15T08:49:36.401514Z","shell.execute_reply":"2022-02-15T08:49:36.413224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# created on current working dir\n# generate_tfrecord(train_annot_df, \"train.record\")\n# generate_tfrecord(test_annot_df, \"test.record\")","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:49:39.887663Z","iopub.execute_input":"2022-02-15T08:49:39.888382Z","iopub.status.idle":"2022-02-15T08:49:39.891369Z","shell.execute_reply.started":"2022-02-15T08:49:39.888346Z","shell.execute_reply":"2022-02-15T08:49:39.890693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Train an object detection model\n\n### Local environment settings:  \n- CUDA on WSL: Windows 11 > Docker Desktop > tensorflow-gpu  \n- GPU: RTX 3070  \n\n###  Model: EfficientDet-D0  \n- Download pretrained model\n```python\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz\n!tar -xvzf efficientdet_d0_coco17_tpu-32.tar.gz\n```\n- pipeline.config\n    - num_classes: 1\n    - image_resizer: 768x768 # GPU memory limit\n    - batch_size: 2\n    - num_steps: 20000\n    - use_bfloat16: false # TPU disabled","metadata":{}},{"cell_type":"markdown","source":"### Command line codes\n```python\n### train: took less then 2 hours on my local environment\n!python model_main_tf2.py --model_dir=models/my_efficientDet \\\n    --pipeline_config_path=models/my_efficientDet/pipeline.config\n\n### evaluation\n!python model_main_tf2.py --model_dir=models/my_efficientDet \\\n    --pipeline_config_path=models/my_efficientDet/pipeline.config \\\n    --checkpoint_dir=models/my_efficientDet\n\n### export\n!python exporter_main_v2.py --input_type image_tensor \\\n    --pipeline_config_path models/my_efficientDet/pipeline.config \\\n    --trained_checkpoint_dir models/my_efficientDet \\\n    --output_directory exported-models/my_efficientDet\n```","metadata":{}},{"cell_type":"markdown","source":"### Evalutaion result\n```\nAccumulating evaluation results...\nDONE (t=0.52s).\n Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.365\n Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.712\n Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.314\n Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.097\n Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.427\n Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.602\n Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.220\n Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.433\n Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.492\n Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.262\n Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.548\n Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.665\nINFO:tensorflow:Eval metrics at step 20000\nI0214 20:29:34.667814 139819700154944 model_lib_v2.py:1015] Eval metrics at step 20000\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP: 0.364686\nI0214 20:29:34.711701 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP: 0.364686\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP@.50IOU: 0.712469\nI0214 20:29:34.713693 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP@.50IOU: 0.712469\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP@.75IOU: 0.313904\nI0214 20:29:34.715082 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP@.75IOU: 0.313904\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP (small): 0.097257\nI0214 20:29:34.716626 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP (small): 0.097257\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP (medium): 0.426918\nI0214 20:29:34.717952 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP (medium): 0.426918\nINFO:tensorflow:        + DetectionBoxes_Precision/mAP (large): 0.601684\nI0214 20:29:34.719259 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Precision/mAP (large): 0.601684\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@1: 0.219556\nI0214 20:29:34.720585 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@1: 0.219556\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@10: 0.433014\nI0214 20:29:34.721925 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@10: 0.433014\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@100: 0.491768\nI0214 20:29:34.723235 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@100: 0.491768\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@100 (small): 0.262473\nI0214 20:29:34.724538 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@100 (small): 0.262473\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@100 (medium): 0.548051\nI0214 20:29:34.725793 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@100 (medium): 0.548051\nINFO:tensorflow:        + DetectionBoxes_Recall/AR@100 (large): 0.664706\nI0214 20:29:34.727559 139819700154944 model_lib_v2.py:1018]     + DetectionBoxes_Recall/AR@100 (large): 0.664706\nINFO:tensorflow:        + Loss/localization_loss: 0.163040\nI0214 20:29:34.728614 139819700154944 model_lib_v2.py:1018]     + Loss/localization_loss: 0.163040\nINFO:tensorflow:        + Loss/classification_loss: 0.422540\nI0214 20:29:34.729672 139819700154944 model_lib_v2.py:1018]     + Loss/classification_loss: 0.422540\nINFO:tensorflow:        + Loss/regularization_loss: 0.032527\nI0214 20:29:34.730714 139819700154944 model_lib_v2.py:1018]     + Loss/regularization_loss: 0.032527\nINFO:tensorflow:        + Loss/total_loss: 0.618107\nI0214 20:29:34.731794 139819700154944 model_lib_v2.py:1018]     + Loss/total_loss: 0.618107\n```\n### F2 score(evalutation metric for competition)  \n    DetectionBoxes_Precision/mAP : 0.364686  \n    DetectionBoxes_Recall/AR@100 : 0.491768  \n    Expected F2 score for IoU=0.50:0.95 : 0.459727  ","metadata":{}},{"cell_type":"markdown","source":"## 4. Run inference and visualization\n- Fine tuned model uploaded to dir: /eff-768-v1  \n    Test on unused dataset: test_annot_df.csv","metadata":{}},{"cell_type":"code","source":"#PATH_TO_MODEL_DIR = os.path.join(os.getcwd(),'exported-models', 'my_efficientDet') # in local env\nPATH_TO_MODEL_DIR = \"../input/eff-768-v1\"","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:50:38.208676Z","iopub.execute_input":"2022-02-15T08:50:38.209257Z","iopub.status.idle":"2022-02-15T08:50:38.215387Z","shell.execute_reply.started":"2022-02-15T08:50:38.209216Z","shell.execute_reply":"2022-02-15T08:50:38.214633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TO_SAVED_MODEL = PATH_TO_MODEL_DIR # + 'saved_model' in local env\n\nprint('Loading model...', end='')\nstart_time = time.time()\n\n# Load saved model and build the detection function\ndetect_fn = tf.saved_model.load(PATH_TO_SAVED_MODEL)\n\nend_time = time.time()\nelapsed_time = end_time - start_time\nprint('Done! Took {} seconds'.format(elapsed_time))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:50:39.794109Z","iopub.execute_input":"2022-02-15T08:50:39.794783Z","iopub.status.idle":"2022-02-15T08:51:04.629066Z","shell.execute_reply.started":"2022-02-15T08:50:39.794744Z","shell.execute_reply":"2022-02-15T08:51:04.628156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_DIR = \"../input/tensorflow-great-barrier-reef/train_images\"\n\ndata = pd.read_csv(\"test_annot_df.csv\",\n                    converters={\"annotations\": literal_eval})","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:51:04.630738Z","iopub.execute_input":"2022-02-15T08:51:04.630998Z","iopub.status.idle":"2022-02-15T08:51:04.683775Z","shell.execute_reply.started":"2022-02-15T08:51:04.630961Z","shell.execute_reply":"2022-02-15T08:51:04.682986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from object_detection.utils import label_map_util\nfrom object_detection.utils import visualization_utils as viz_utils\nPATH_TO_LABELS = 'label_map.pbtxt'\ncategory_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS,\n                                                                    use_display_name=True)\ndef load_image_into_numpy_array(path):\n    return np.array(Image.open(path))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:51:04.685042Z","iopub.execute_input":"2022-02-15T08:51:04.685382Z","iopub.status.idle":"2022-02-15T08:51:04.722306Z","shell.execute_reply.started":"2022-02-15T08:51:04.685343Z","shell.execute_reply":"2022-02-15T08:51:04.721467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %matplotlib inline\ntest_data = data.sample(5)\nfor vid, _, vframe, _, annot in test_data.values:\n    if not annot:\n        continue\n    image_path = os.path.join(IMAGE_DIR, 'video_' + str(vid), str(vframe) + '.jpg')\n    print('Running inference for {}... '.format(image_path), end='')\n    image_np = load_image_into_numpy_array(image_path)\n    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.uint8)\n    detections = detect_fn(input_tensor)\n\n    # All outputs are batches tensors.\n    # Convert to numpy arrays, and take index [0] to remove the batch dimension.\n    # We're only interested in the first num_detections.\n    num_detections = int(detections.pop('num_detections'))\n    detections = {key: value[0, :num_detections].numpy()\n                  for key, value in detections.items()}\n    detections['num_detections'] = num_detections\n\n    # detection_classes should be ints.\n    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n\n    label_id_offset = 0\n    image_np_with_detections = image_np.copy()\n\n    viz_utils.visualize_boxes_and_labels_on_image_array(\n            image_np_with_detections,\n            detections['detection_boxes'],\n            detections['detection_classes']+label_id_offset,\n            detections['detection_scores'],\n            category_index,\n            use_normalized_coordinates=True,\n            max_boxes_to_draw=20,\n            min_score_thresh=0.1,\n            agnostic_mode=False,\n            line_thickness=3\n            )\n    height, width = image_np.shape[:2]\n    if annot:\n        boxes = []\n        for box in annot:\n            xmin, ymin, w, h = box.values()\n            boxes.append([ymin/height,xmin/width,(ymin+h)/height,(xmin+w)/width])\n        image_np_with_detections = tf.image.draw_bounding_boxes(\n                image_np_with_detections.reshape([1, *image_np_with_detections.shape]), np.array(boxes)[tf.newaxis,...], [[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]])\n        image_np_with_detections = np.array(image_np_with_detections, dtype=np.uint8)\n        image_np_with_detections = image_np_with_detections[0,:]\n        plt.figure(figsize=(16,9))\n        plt.imshow(image_np_with_detections)\n        print('Done')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T08:51:54.540060Z","iopub.execute_input":"2022-02-15T08:51:54.540479Z","iopub.status.idle":"2022-02-15T08:52:01.058778Z","shell.execute_reply.started":"2022-02-15T08:51:54.540419Z","shell.execute_reply":"2022-02-15T08:52:01.058013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Note that:\n    Black boxes are actual bounding boxes  \n    Green boxes are predicted bounding boxes with score threshold 0.1  \n    Model works pretty well on large objects, but struggles to manage smaller ones  \n- Ideas may improve the performance:  \n    - Higher resolution, rather than 768x768 (proper physical device may needed)\n    - Data augmentation\n    - Better base model architecture\n    - Train longer\n    - etc.","metadata":{}}]}