{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook contains code to train a crown-of-thorns starfish (COTS) detection model to serve as a baseline model for [this competition](https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview). We use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) to apply transfer learning on an [EfficientDet-D0](https://arxiv.org/abs/1911.09070) pretrained model. ","metadata":{"papermill":{"duration":0.023199,"end_time":"2021-11-19T08:38:07.360759","exception":false,"start_time":"2021-11-19T08:38:07.33756","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Install TensorFlow Object Detection API\n\nPip may report some dependency errors. You can safely ignore these errors and proceed if all tests in `model_builder_tf2_test.py` passed. ","metadata":{"papermill":{"duration":0.021045,"end_time":"2021-11-19T08:38:07.403389","exception":false,"start_time":"2021-11-19T08:38:07.382344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models && git checkout ac8d06519","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":25.822618,"end_time":"2021-11-19T08:38:33.247836","exception":false,"start_time":"2021-11-19T08:38:07.425218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:41:30.815848Z","iopub.execute_input":"2021-11-26T00:41:30.816304Z","iopub.status.idle":"2021-11-26T00:41:57.61409Z","shell.execute_reply.started":"2021-11-26T00:41:30.81622Z","shell.execute_reply":"2021-11-26T00:41:57.613245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd models/research\n\n# Compile protos.\nprotoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow to 2.6.0 to make it compatible with the CUDA version preinstalled on Kaggle.\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\nwget https://storage.googleapis.com/odml-dataset/others/setup.py\npip install -q --user .\n\n# Test if the Object Dectection API is working correctly\npython object_detection/builders/model_builder_tf2_test.py","metadata":{"papermill":{"duration":79.154739,"end_time":"2021-11-19T08:39:52.448588","exception":false,"start_time":"2021-11-19T08:38:33.293849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:41:57.616021Z","iopub.execute_input":"2021-11-26T00:41:57.616454Z","iopub.status.idle":"2021-11-26T00:43:15.859692Z","shell.execute_reply.started":"2021-11-26T00:41:57.616414Z","shell.execute_reply":"2021-11-26T00:43:15.858733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import dependencies","metadata":{}},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport pandas as pd\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'\nsys.path.insert(0, INPUT_DIR)\nimport greatbarrierreef","metadata":{"papermill":{"duration":1.526137,"end_time":"2021-11-19T08:39:54.022253","exception":false,"start_time":"2021-11-19T08:39:52.496116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:15.861484Z","iopub.execute_input":"2021-11-26T00:43:15.86186Z","iopub.status.idle":"2021-11-26T00:43:17.539832Z","shell.execute_reply.started":"2021-11-26T00:43:15.861818Z","shell.execute_reply":"2021-11-26T00:43:17.539017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The notebook is supposed to run with TF 2.6.0\nprint(tf.__version__)\nprint(tf.test.is_gpu_available())\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"papermill":{"duration":0.729459,"end_time":"2021-11-19T08:39:54.798333","exception":false,"start_time":"2021-11-19T08:39:54.068874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:17.543686Z","iopub.execute_input":"2021-11-26T00:43:17.543883Z","iopub.status.idle":"2021-11-26T00:43:18.209487Z","shell.execute_reply.started":"2021-11-26T00:43:17.543858Z","shell.execute_reply":"2021-11-26T00:43:18.208767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the training dataset","metadata":{"papermill":{"duration":0.04958,"end_time":"2021-11-19T08:39:54.895616","exception":false,"start_time":"2021-11-19T08:39:54.846036","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Split the `train` folder into training dataset and validation dataset. ","metadata":{"papermill":{"duration":0.788121,"end_time":"2021-11-19T08:40:46.513739","exception":false,"start_time":"2021-11-19T08:40:45.725618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAINING_RATIO = 0.8\n\ndata_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n\n# Split the dataset so that no sequence is leaked from the training dataset into the validation dataset.\nsplit_index = int(TRAINING_RATIO * len(data_df))\nwhile data_df.iloc[split_index - 1].sequence == data_df.iloc[split_index].sequence:\n    split_index += 1\n\n# Shuffle both the training and validation datasets.\ntrain_data_df = data_df.iloc[:split_index].sample(frac=1).reset_index(drop=True)\nval_data_df = data_df.iloc[split_index:].sample(frac=1).reset_index(drop=True)\n\ntrain_positive_count = len(train_data_df[train_data_df.annotations != '[]'])\nval_positive_count = len(val_data_df[val_data_df.annotations != '[]'])\n\nprint('Training ratio (all samples):', \n      float(len(train_data_df)) / (len(train_data_df) + len(val_data_df)))\nprint('Training ratio (positive samples):', \n      float(train_positive_count) / (train_positive_count + val_positive_count))","metadata":{"papermill":{"duration":1.175804,"end_time":"2021-11-19T08:40:48.461269","exception":false,"start_time":"2021-11-19T08:40:47.285465","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:18.211125Z","iopub.execute_input":"2021-11-26T00:43:18.211736Z","iopub.status.idle":"2021-11-26T00:43:18.379386Z","shell.execute_reply.started":"2021-11-26T00:43:18.211694Z","shell.execute_reply":"2021-11-26T00:43:18.378629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To save time for demonstration purpose, here we'll only take the positive images (images that contain at least 1 starfish) for training. TensorFlow Object Detection API will take the areas in the images that aren't annotated as containing a starfish to use as negative samples.","metadata":{}},{"cell_type":"code","source":"# Take only the positive images for training and validation\ntrain_data_df = train_data_df[train_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for training:', len(train_data_df))\nval_data_df = val_data_df[val_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for validation:', len(val_data_df))","metadata":{"papermill":{"duration":0.240932,"end_time":"2021-11-19T08:40:48.926677","exception":false,"start_time":"2021-11-19T08:40:48.685745","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:18.380686Z","iopub.execute_input":"2021-11-26T00:43:18.380922Z","iopub.status.idle":"2021-11-26T00:43:18.39512Z","shell.execute_reply.started":"2021-11-26T00:43:18.380889Z","shell.execute_reply":"2021-11-26T00:43:18.394262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize one randomly selected image from the training data to see if the annotation looks correct.","metadata":{"papermill":{"duration":0.196154,"end_time":"2021-11-19T08:40:49.387603","exception":false,"start_time":"2021-11-19T08:40:49.191449","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def image_with_annotation(video_id, video_frame, data_df, image_path):\n    \"\"\"Visualize annotations of a given image.\"\"\"\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    image = Image.open(full_path)\n    draw = ImageDraw.Draw(image)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            draw.rectangle((\n                annotation['x'], \n                annotation['y'],\n                (annotation['x'] + annotation['width']), \n                (annotation['y'] + annotation['height']),\n                ), outline=(255, 255, 0))\n        \n    buf = io.BytesIO()\n    image.save(buf, 'PNG')\n    data = buf.getvalue()\n\n    return data\n\n# Test visualization of a randomly selected image\nimage_path = os.path.join(INPUT_DIR, 'train_images')\ntest_index = 38\nvideo_id = train_data_df.iloc[test_index].video_id\nvideo_frame = train_data_df.iloc[test_index].video_frame\nIPython.display.Image(image_with_annotation(video_id, video_frame, data_df, image_path))","metadata":{"papermill":{"duration":0.694191,"end_time":"2021-11-19T08:40:50.258113","exception":false,"start_time":"2021-11-19T08:40:49.563922","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:18.396729Z","iopub.execute_input":"2021-11-26T00:43:18.397076Z","iopub.status.idle":"2021-11-26T00:43:18.757634Z","shell.execute_reply.started":"2021-11-26T00:43:18.397038Z","shell.execute_reply":"2021-11-26T00:43:18.756846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Convert the training and validation dataset into TFRecord format as required by the TensorFlow Object Detection API.","metadata":{"papermill":{"duration":0.115403,"end_time":"2021-11-19T08:40:50.494003","exception":false,"start_time":"2021-11-19T08:40:50.3786","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from object_detection.utils import dataset_util\nfrom object_detection.dataset_tools import tf_record_creation_util\n\n\ndef create_tf_example(video_id, video_frame, data_df, image_path):\n    \"\"\"Create a tf.Example entry for a given training image.\"\"\"\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    with tf.io.gfile.GFile(full_path, 'rb') as fid:\n        encoded_jpg = fid.read()\n    encoded_jpg_io = io.BytesIO(encoded_jpg)\n    image = Image.open(encoded_jpg_io)\n    if image.format != 'JPEG':\n        raise ValueError('Image format not JPEG')\n\n    height = image.size[1] # Image height\n    width = image.size[0] # Image width\n    filename = f'{video_id}:{video_frame}'.encode('utf8') # Unique id of the image.\n    encoded_image_data = None # Encoded image bytes\n    image_format = 'jpeg'.encode('utf8') # b'jpeg' or b'png'\n\n    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)\n    xmaxs = [] # List of normalized right x coordinates in bounding box\n             # (1 per box)\n    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)\n    ymaxs = [] # List of normalized bottom y coordinates in bounding box\n             # (1 per box)\n    classes_text = [] # List of string class name of bounding box (1 per box)\n    classes = [] # List of integer class id of bounding box (1 per box)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            xmins.append(annotation['x'] / width) \n            xmaxs.append((annotation['x'] + annotation['width']) / width) \n            ymins.append(annotation['y'] / height) \n            ymaxs.append((annotation['y'] + annotation['height']) / height) \n\n            classes_text.append('COTS'.encode('utf8'))\n            classes.append(1)\n\n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n      'image/height': dataset_util.int64_feature(height),\n      'image/width': dataset_util.int64_feature(width),\n      'image/filename': dataset_util.bytes_feature(filename),\n      'image/source_id': dataset_util.bytes_feature(filename),\n      'image/encoded': dataset_util.bytes_feature(encoded_jpg),\n      'image/format': dataset_util.bytes_feature(image_format),\n      'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n      'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n      'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n      'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n      'image/object/class/label': dataset_util.int64_list_feature(classes),\n    }))\n    \n    return tf_example\n\ndef convert_to_tfrecord(data_df, tfrecord_filebase, image_path, num_shards = 10):\n    \"\"\"Convert the object detection dataset to TFRecord as required by the TF ODT API.\"\"\"\n    with contextlib2.ExitStack() as tf_record_close_stack:\n        output_tfrecords = tf_record_creation_util.open_sharded_output_tfrecords(\n            tf_record_close_stack, tfrecord_filebase, num_shards)\n        \n        for index, row in data_df.iterrows():\n            if index % 500 == 0:\n                print('Processed {0} images.'.format(index))\n            tf_example = create_tf_example(row.video_id, row.video_frame, data_df, image_path)\n            output_shard_index = index % num_shards\n            output_tfrecords[output_shard_index].write(tf_example.SerializeToString())\n        \n        print('Completed processing {0} images.'.format(len(data_df)))\n\n!mkdir dataset\nimage_path = os.path.join(INPUT_DIR, 'train_images')\n\n# Convert train images to TFRecord\nprint('Converting TRAIN images...')\nconvert_to_tfrecord(\n  train_data_df,\n  'dataset/cots_train',\n  image_path,\n  num_shards = 4\n)\n\n# Convert validation images to TFRecord\nprint('Converting VALIDATION images...')\nconvert_to_tfrecord(\n  val_data_df,\n  'dataset/cots_val',\n  image_path,\n  num_shards = 4\n)","metadata":{"papermill":{"duration":38.916848,"end_time":"2021-11-19T08:41:29.641271","exception":false,"start_time":"2021-11-19T08:40:50.724423","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:43:18.758843Z","iopub.execute_input":"2021-11-26T00:43:18.759149Z","iopub.status.idle":"2021-11-26T00:44:24.473895Z","shell.execute_reply.started":"2021-11-26T00:43:18.75911Z","shell.execute_reply":"2021-11-26T00:44:24.473007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a label map to map between label index and human-readable label name.\n\nlabel_map_str = \"\"\"item {\n  id: 1\n  name: 'COTS'\n}\"\"\"\n\nwith open('dataset/label_map.pbtxt', 'w') as f:\n  f.write(label_map_str)\n\n!more dataset/label_map.pbtxt","metadata":{"papermill":{"duration":16.05844,"end_time":"2021-11-19T08:41:45.838392","exception":false,"start_time":"2021-11-19T08:41:29.779952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:44:24.475671Z","iopub.execute_input":"2021-11-26T00:44:24.477093Z","iopub.status.idle":"2021-11-26T00:44:25.157161Z","shell.execute_reply.started":"2021-11-26T00:44:24.477057Z","shell.execute_reply":"2021-11-26T00:44:25.156306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train an object detection model\n\nWe'll use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) and an EfficientDet-D0 base model and apply transfer learning to train a COTS detection model. EfficientDet-D0 is the smallest model in the EfficientDet model family and we pick it to reduce training time for demonstration purpose. You can probably increase accuracy by switch to using a larger EfficientDet model.","metadata":{"papermill":{"duration":3.355764,"end_time":"2021-11-19T08:41:50.909779","exception":false,"start_time":"2021-11-19T08:41:47.554015","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Download the pretrained EfficientDet-D0 checkpoint\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz\n!tar -xvzf efficientdet_d0_coco17_tpu-32.tar.gz","metadata":{"papermill":{"duration":3.256545,"end_time":"2021-11-19T08:41:54.358822","exception":false,"start_time":"2021-11-19T08:41:51.102277","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:44:25.1609Z","iopub.execute_input":"2021-11-26T00:44:25.161418Z","iopub.status.idle":"2021-11-26T00:44:27.475947Z","shell.execute_reply.started":"2021-11-26T00:44:25.161384Z","shell.execute_reply":"2021-11-26T00:44:27.475146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from string import Template\n\nconfig_file_template = \"\"\"\n# SSD with EfficientNet-b0 + BiFPN feature extractor,\n# shared box predictor and focal loss (a.k.a EfficientDet-d0).\n# See EfficientDet, Tan et al, https://arxiv.org/abs/1911.09070\n# See Lin et al, https://arxiv.org/abs/1708.02002\n# Initialized from an EfficientDet-D0 checkpoint.\n#\n# Train on GPU\n\nmodel {\n  ssd {\n    inplace_batchnorm_update: true\n    freeze_batchnorm: false\n    num_classes: 1\n    add_background_class: false\n    box_coder {\n      faster_rcnn_box_coder {\n        y_scale: 10.0\n        x_scale: 10.0\n        height_scale: 5.0\n        width_scale: 5.0\n      }\n    }\n    matcher {\n      argmax_matcher {\n        matched_threshold: 0.5\n        unmatched_threshold: 0.5\n        ignore_thresholds: false\n        negatives_lower_than_unmatched: true\n        force_match_for_each_row: true\n        use_matmul_gather: true\n      }\n    }\n    similarity_calculator {\n      iou_similarity {\n      }\n    }\n    encode_background_as_zeros: true\n    anchor_generator {\n      multiscale_anchor_generator {\n        min_level: 3\n        max_level: 7\n        anchor_scale: 4.0\n        aspect_ratios: [1.0, 2.0, 0.5]\n        scales_per_octave: 3\n      }\n    }\n    image_resizer {\n      keep_aspect_ratio_resizer {\n        min_dimension: 1280\n        max_dimension: 1280\n        pad_to_max_dimension: true\n        }\n    }\n    box_predictor {\n      weight_shared_convolutional_box_predictor {\n        depth: 64\n        class_prediction_bias_init: -4.6\n        conv_hyperparams {\n          force_use_bias: true\n          activation: SWISH\n          regularizer {\n            l2_regularizer {\n              weight: 0.00004\n            }\n          }\n          initializer {\n            random_normal_initializer {\n              stddev: 0.01\n              mean: 0.0\n            }\n          }\n          batch_norm {\n            scale: true\n            decay: 0.99\n            epsilon: 0.001\n          }\n        }\n        num_layers_before_predictor: 3\n        kernel_size: 3\n        use_depthwise: true\n      }\n    }\n    feature_extractor {\n      type: 'ssd_efficientnet-b0_bifpn_keras'\n      bifpn {\n        min_level: 3\n        max_level: 7\n        num_iterations: 3\n        num_filters: 64\n      }\n      conv_hyperparams {\n        force_use_bias: true\n        activation: SWISH\n        regularizer {\n          l2_regularizer {\n            weight: 0.00004\n          }\n        }\n        initializer {\n          truncated_normal_initializer {\n            stddev: 0.03\n            mean: 0.0\n          }\n        }\n        batch_norm {\n          scale: true,\n          decay: 0.99,\n          epsilon: 0.001,\n        }\n      }\n    }\n    loss {\n      classification_loss {\n        weighted_sigmoid_focal {\n          alpha: 0.25\n          gamma: 1.5\n        }\n      }\n      localization_loss {\n        weighted_smooth_l1 {\n        }\n      }\n      classification_weight: 1.0\n      localization_weight: 1.0\n    }\n    normalize_loss_by_num_matches: true\n    normalize_loc_loss_by_codesize: true\n    post_processing {\n      batch_non_max_suppression {\n        score_threshold: 1e-8\n        iou_threshold: 0.5\n        max_detections_per_class: 100\n        max_total_detections: 100\n      }\n      score_converter: SIGMOID\n    }\n  }\n}\n\ntrain_config: {\n  fine_tune_checkpoint: \"efficientdet_d0_coco17_tpu-32/checkpoint/ckpt-0\"\n  fine_tune_checkpoint_version: V2\n  fine_tune_checkpoint_type: \"detection\"\n  batch_size: 2\n  sync_replicas: false\n  startup_delay_steps: 0\n  replicas_to_aggregate: 1\n  use_bfloat16: false\n  num_steps: $training_steps\n  data_augmentation_options {\n    random_horizontal_flip {\n    }\n  }\n  data_augmentation_options {\n    random_scale_crop_and_pad_to_square {\n      output_size: 1280\n      scale_min: 0.5\n      scale_max: 2.0\n    }\n  }\n  optimizer {\n    momentum_optimizer: {\n      learning_rate: {\n        cosine_decay_learning_rate {\n          learning_rate_base: 5e-3\n          total_steps: $training_steps\n          warmup_learning_rate: 5e-4\n          warmup_steps: $warmup_steps\n        }\n      }\n      momentum_optimizer_value: 0.9\n    }\n    use_moving_average: false\n  }\n  max_number_of_boxes: 100\n  unpad_groundtruth_tensors: false\n}\n\ntrain_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  tf_record_input_reader {\n    input_path: \"dataset/cots_train-?????-of-00004\"\n  }\n}\n\neval_config: {\n  metrics_set: \"coco_detection_metrics\"\n  use_moving_averages: false\n  batch_size: 2;\n}\n\neval_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  shuffle: false\n  num_epochs: 1\n  tf_record_input_reader {\n    input_path: \"dataset/cots_val-?????-of-00004\"\n  }\n}\n\"\"\"","metadata":{"papermill":{"duration":0.133468,"end_time":"2021-11-19T08:41:54.609774","exception":false,"start_time":"2021-11-19T08:41:54.476306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:44:27.477682Z","iopub.execute_input":"2021-11-26T00:44:27.477969Z","iopub.status.idle":"2021-11-26T00:44:27.486722Z","shell.execute_reply.started":"2021-11-26T00:44:27.477916Z","shell.execute_reply":"2021-11-26T00:44:27.485904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the training pipeline\n\nTRAINING_STEPS = 20000\nWARMUP_STEPS = 2000\nPIPELINE_CONFIG_PATH='dataset/pipeline.config'\n\npipeline = Template(config_file_template).substitute(\n    training_steps=TRAINING_STEPS, warmup_steps=WARMUP_STEPS)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{"papermill":{"duration":0.121946,"end_time":"2021-11-19T08:41:54.846958","exception":false,"start_time":"2021-11-19T08:41:54.725012","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:44:27.488444Z","iopub.execute_input":"2021-11-26T00:44:27.489118Z","iopub.status.idle":"2021-11-26T00:44:27.500089Z","shell.execute_reply.started":"2021-11-26T00:44:27.489067Z","shell.execute_reply":"2021-11-26T00:44:27.499359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR='cots_efficientdet_d0'\n!mkdir {MODEL_DIR}\n!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --alsologtostderr","metadata":{"papermill":{"duration":18463.625406,"end_time":"2021-11-19T13:49:38.586506","exception":false,"start_time":"2021-11-19T08:41:54.9611","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-26T00:44:27.503177Z","iopub.execute_input":"2021-11-26T00:44:27.503861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the object detection model","metadata":{"papermill":{"duration":0.226793,"end_time":"2021-11-19T13:49:39.043281","exception":false,"start_time":"2021-11-19T13:49:38.816488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --checkpoint_dir={MODEL_DIR} \\\n    --eval_timeout=0 \\\n    --alsologtostderr","metadata":{"papermill":{"duration":323.73381,"end_time":"2021-11-19T13:55:03.004285","exception":false,"start_time":"2021-11-19T13:49:39.270475","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export as SavedModel for inference","metadata":{"papermill":{"duration":0.239823,"end_time":"2021-11-19T13:55:03.483464","exception":false,"start_time":"2021-11-19T13:55:03.243641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python models/research/object_detection/exporter_main_v2.py \\\n    --input_type image_tensor \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --trained_checkpoint_dir={MODEL_DIR} \\\n    --output_directory={MODEL_DIR}/output","metadata":{"papermill":{"duration":122.093553,"end_time":"2021-11-19T13:57:05.815158","exception":false,"start_time":"2021-11-19T13:55:03.721605","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {MODEL_DIR}/output","metadata":{"papermill":{"duration":0.931074,"end_time":"2021-11-19T13:57:06.998967","exception":false,"start_time":"2021-11-19T13:57:06.067893","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run inference on test images and create the submission file","metadata":{"papermill":{"duration":0.24428,"end_time":"2021-11-19T13:57:07.485159","exception":false,"start_time":"2021-11-19T13:57:07.240879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load the TensorFlow COTS detection model into memory.\nstart_time = time.time()\ntf.keras.backend.clear_session()\ndetect_fn_tf_odt = tf.saved_model.load(os.path.join(os.path.join(MODEL_DIR, 'output'), 'saved_model'))\nend_time = time.time()\nelapsed_time = end_time - start_time\nprint('Elapsed time: ' + str(elapsed_time) + 's')","metadata":{"papermill":{"duration":28.095431,"end_time":"2021-11-19T13:57:35.83037","exception":false,"start_time":"2021-11-19T13:57:07.734939","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define some utils method for prediction.\n\ndef load_image_into_numpy_array(path):\n    \"\"\"Load an image from file into a numpy array.\n\n    Puts image into numpy array to feed into tensorflow graph.\n    Note that by convention we put it into a numpy array with shape\n    (height, width, channels), where channels=3 for RGB.\n\n    Args:\n    path: a file path (this can be local or on colossus)\n\n    Returns:\n    uint8 numpy array with shape (img_height, img_width, 3)\n    \"\"\"\n    img_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(io.BytesIO(img_data))\n    (im_width, im_height) = image.size\n    \n    return np.array(image.getdata()).reshape(\n      (im_height, im_width, 3)).astype(np.uint8)\n\ndef detect(image_np):\n    \"\"\"Detect COTS from a given numpy image.\"\"\"\n\n    input_tensor = np.expand_dims(image_np, 0)\n    start_time = time.time()\n    detections = detect_fn_tf_odt(input_tensor)\n    return detections","metadata":{"papermill":{"duration":0.417072,"end_time":"2021-11-19T13:57:36.586877","exception":false,"start_time":"2021-11-19T13:57:36.169805","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"papermill":{"duration":0.354385,"end_time":"2021-11-19T13:57:37.442493","exception":false,"start_time":"2021-11-19T13:57:37.088108","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DETECTION_THRESHOLD = 0.3\n\nsubmission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n    height, width, _ = image_np.shape\n    \n    # Run object detection using the TensorFlow model.\n    detections = detect(image_np)\n    \n    # Parse the detection result and generate a prediction string.\n    num_detections = detections['num_detections'][0].numpy().astype(np.int32)\n    predictions = []\n    for index in range(num_detections):\n        score = detections['detection_scores'][0][index].numpy()\n        if score < DETECTION_THRESHOLD:\n            continue\n\n        bbox = detections['detection_boxes'][0][index].numpy()\n        y_min = int(bbox[0] * height)\n        x_min = int(bbox[1] * width)\n        y_max = int(bbox[2] * height)\n        x_max = int(bbox[3] * width)\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    # Generate the submission data.\n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"papermill":{"duration":4.526271,"end_time":"2021-11-19T13:57:42.213925","exception":false,"start_time":"2021-11-19T13:57:37.687654","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean up","metadata":{"papermill":{"duration":0.264983,"end_time":"2021-11-19T13:57:42.731628","exception":false,"start_time":"2021-11-19T13:57:42.466645","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Remove the dataset files to save space.\n!rm -rf dataset\n!rm -rf train_images\n!rm tensorflow-great-barrier-reef.zip\n\n# Remove other data downloaded during training.\n!rm -rf models\n!rm efficientdet_d0_coco17_tpu-32.tar.gz","metadata":{"papermill":{"duration":4.604917,"end_time":"2021-11-19T13:57:47.580583","exception":false,"start_time":"2021-11-19T13:57:42.975666","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}