{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[Original Notebook](https://www.kaggle.com/khanhlvg/cots-detection-w-tensorflow-object-detection-api)\n\nThis notebook contains code to train a crown-of-thorns starfish (COTS) detection model to serve as a baseline model for [this competition](https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview). We use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) to apply transfer learning on an [EfficientDet-D0](https://arxiv.org/abs/1911.09070) pretrained model. ","metadata":{"papermill":{"duration":0.023199,"end_time":"2021-11-19T08:38:07.360759","exception":false,"start_time":"2021-11-19T08:38:07.33756","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Install TensorFlow Object Detection API\n\nPip may report some dependency errors. You can safely ignore these errors and proceed if all tests in `model_builder_tf2_test.py` passed. ","metadata":{"papermill":{"duration":0.021045,"end_time":"2021-11-19T08:38:07.403389","exception":false,"start_time":"2021-11-19T08:38:07.382344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":25.822618,"end_time":"2021-11-19T08:38:33.247836","exception":false,"start_time":"2021-11-19T08:38:07.425218","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-20T05:08:27.095381Z","iopub.execute_input":"2021-12-20T05:08:27.095918Z","iopub.status.idle":"2021-12-20T05:08:56.848791Z","shell.execute_reply.started":"2021-12-20T05:08:27.095764Z","shell.execute_reply":"2021-12-20T05:08:56.847309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd models/research\n\n# Compile protos.\nprotoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow to 2.6.0 to make it compatible with the CUDA version preinstalled on Kaggle.\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\nwget https://storage.googleapis.com/odml-dataset/others/setup.py\npip install -q --user .\n\n# Test if the Object Dectection API is working correctly\npython object_detection/builders/model_builder_tf2_test.py","metadata":{"papermill":{"duration":79.154739,"end_time":"2021-11-19T08:39:52.448588","exception":false,"start_time":"2021-11-19T08:38:33.293849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:08:56.852653Z","iopub.execute_input":"2021-12-20T05:08:56.853387Z","iopub.status.idle":"2021-12-20T05:10:34.540409Z","shell.execute_reply.started":"2021-12-20T05:08:56.853337Z","shell.execute_reply":"2021-12-20T05:10:34.539154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import dependencies","metadata":{}},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport pandas as pd\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'\nsys.path.append(INPUT_DIR)\nimport greatbarrierreef","metadata":{"papermill":{"duration":1.526137,"end_time":"2021-11-19T08:39:54.022253","exception":false,"start_time":"2021-11-19T08:39:52.496116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:10:34.543171Z","iopub.execute_input":"2021-12-20T05:10:34.543527Z","iopub.status.idle":"2021-12-20T05:10:36.385970Z","shell.execute_reply.started":"2021-12-20T05:10:34.543478Z","shell.execute_reply":"2021-12-20T05:10:36.384927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The notebook is supposed to run with TF 2.6.0\nprint(tf.__version__)\nprint(tf.test.is_gpu_available())\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"papermill":{"duration":0.729459,"end_time":"2021-11-19T08:39:54.798333","exception":false,"start_time":"2021-11-19T08:39:54.068874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:10:36.390647Z","iopub.execute_input":"2021-12-20T05:10:36.390889Z","iopub.status.idle":"2021-12-20T05:10:37.251865Z","shell.execute_reply.started":"2021-12-20T05:10:36.390860Z","shell.execute_reply":"2021-12-20T05:10:37.250033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-20T05:10:37.254187Z","iopub.execute_input":"2021-12-20T05:10:37.254825Z","iopub.status.idle":"2021-12-20T05:10:37.328268Z","shell.execute_reply.started":"2021-12-20T05:10:37.254777Z","shell.execute_reply":"2021-12-20T05:10:37.327119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the training dataset","metadata":{"papermill":{"duration":0.04958,"end_time":"2021-11-19T08:39:54.895616","exception":false,"start_time":"2021-11-19T08:39:54.846036","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Split the `train` folder into training dataset and validation dataset. ","metadata":{"papermill":{"duration":0.788121,"end_time":"2021-11-19T08:40:46.513739","exception":false,"start_time":"2021-11-19T08:40:45.725618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAINING_RATIO = 0.8\n\ndata_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n\n# Split the dataset so that no sequence is leaked from the training dataset into the validation dataset.\nsplit_index = int(TRAINING_RATIO * len(data_df))\nwhile data_df.iloc[split_index - 1].sequence == data_df.iloc[split_index].sequence:\n    split_index += 1\n\n# Shuffle both the training and validation datasets.\ntrain_data_df = data_df.iloc[:split_index].sample(frac=1).reset_index(drop=True)\nval_data_df = data_df.iloc[split_index:].sample(frac=1).reset_index(drop=True)\n\ntrain_positive_count = len(train_data_df[train_data_df.annotations != '[]'])\nval_positive_count = len(val_data_df[val_data_df.annotations != '[]'])\n\nprint('Training ratio (all samples):', \n      float(len(train_data_df)) / (len(train_data_df) + len(val_data_df)))\nprint('Training ratio (positive samples):', \n      float(train_positive_count) / (train_positive_count + val_positive_count))","metadata":{"papermill":{"duration":1.175804,"end_time":"2021-11-19T08:40:48.461269","exception":false,"start_time":"2021-11-19T08:40:47.285465","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:10:37.330119Z","iopub.execute_input":"2021-12-20T05:10:37.330506Z","iopub.status.idle":"2021-12-20T05:10:37.527828Z","shell.execute_reply.started":"2021-12-20T05:10:37.330462Z","shell.execute_reply":"2021-12-20T05:10:37.526784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nTo save time for demonstration purpose, here we'll only take the positive images (images that contain at least 1 starfish) for training. TensorFlow Object Detection API will take the areas in the images that aren't annotated as containing a starfish to use as negative samples.\n","metadata":{}},{"cell_type":"code","source":"\n\n# Take only the positive images for training and validation\ntrain_data_df = train_data_df[train_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for training:', len(train_data_df))\nval_data_df = val_data_df[val_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for validation:', len(val_data_df))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-20T05:10:37.529357Z","iopub.execute_input":"2021-12-20T05:10:37.530163Z","iopub.status.idle":"2021-12-20T05:10:37.546764Z","shell.execute_reply.started":"2021-12-20T05:10:37.530116Z","shell.execute_reply":"2021-12-20T05:10:37.545430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize one randomly selected image from the training data to see if the annotation looks correct.","metadata":{"papermill":{"duration":0.196154,"end_time":"2021-11-19T08:40:49.387603","exception":false,"start_time":"2021-11-19T08:40:49.191449","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def image_with_annotation(video_id, video_frame, data_df, image_path):\n    \"\"\"Visualize annotations of a given image.\"\"\"\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    image = Image.open(full_path)\n    draw = ImageDraw.Draw(image)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            draw.rectangle((\n                annotation['x'], \n                annotation['y'],\n                (annotation['x'] + annotation['width']), \n                (annotation['y'] + annotation['height']),\n                ), outline=(255, 255, 0))\n        \n    buf = io.BytesIO()\n    image.save(buf, 'PNG')\n    data = buf.getvalue()\n\n    return data\n\n# Test visualization of a randomly selected image\nimage_path = os.path.join(INPUT_DIR, 'train_images')\ntest_index = 4\nvideo_id = train_data_df.iloc[test_index].video_id\nvideo_frame = train_data_df.iloc[test_index].video_frame\nIPython.display.Image(image_with_annotation(video_id, video_frame, data_df, image_path))","metadata":{"papermill":{"duration":0.694191,"end_time":"2021-11-19T08:40:50.258113","exception":false,"start_time":"2021-11-19T08:40:49.563922","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:10:37.548568Z","iopub.execute_input":"2021-12-20T05:10:37.549099Z","iopub.status.idle":"2021-12-20T05:10:37.963552Z","shell.execute_reply.started":"2021-12-20T05:10:37.549024Z","shell.execute_reply":"2021-12-20T05:10:37.961100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **tf_record**\nModels based on the TensorFlow object detection API need a special format for all input data, called TFRecord file format, Tensorflow’s own binary storage format.\n\nIf you are working with large datasets, using a binary file format for storage of your data can have a significant impact on the performance of your import pipeline and as a consequence on the training time of your model. Binary data takes up less space on disk, takes less time to copy and can be read much more efficiently from disk.\n\nIt is optimized for use with Tensorflow in multiple ways. To start with, it makes it easy to combine multiple datasets and integrates seamlessly with the data import and preprocessing functionality provided by the library.Especially for datasets that are too large to be stored fully in memory this is an advantage as only the data that is required at the time (e.g. a batch) is loaded from disk and then processed.\n\n#### A TFRecord file contains a sequence of records. The file can only be read sequentially.\n**[Official Tutorial](https://www.tensorflow.org/tutorials/load_data/tfrecord)**","metadata":{}},{"cell_type":"code","source":"BytesList = tf.train.BytesList\nFloatList = tf.train.FloatList\nIntList = tf.train.Int64List\n\nfrom object_detection.utils import dataset_util\n\ndef image_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(\n        bytes_list=BytesList(value=[tf.io.encode_jpeg(value).numpy()])\n    )\n\ndef bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=BytesList(value=[value.encode()]))\n\ndef bytes_feature_list(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=BytesList(value=value))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=FloatList(value=[value]))\n\n\ndef float_feature_list(value):\n    \"\"\"Returns a list of float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=IntList(value=[value]))\n\ndef _int64_feature_list(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=IntList(value=value))\n\n\n\ndef create_tf_example(video_id, video_frame, data_df, image_dir_path):\n    full_path = os.path.join(image_dir_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    img = open(full_path,'rb').read()\n    filename = f'{video_id}:{video_frame}'.encode('utf8')\n    image = Image.open(full_path)\n    image_format = 'jpeg'.encode('utf8')\n    \n    height = image.size[1] # Image height\n    width = image.size[0] # Image width\n    \n    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)\n    xmaxs = [] # List of normalized right x coordinates in bounding box\n             # (1 per box)\n    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)\n    ymaxs = [] # List of normalized bottom y coordinates in bounding box\n             # (1 per box)\n    classes_text = [] # List of string class name of bounding box (1 per box)\n    classes = [] # List of integer class id of bounding box (1 per box)\n    \n    \n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            xmins.append(annotation['x'] / width) \n            xmaxs.append((annotation['x'] + annotation['width']) / width) \n            ymins.append(annotation['y'] / height) \n            ymaxs.append((annotation['y'] + annotation['height']) / height) \n\n            classes_text.append('COTS'.encode('utf8'))\n            classes.append(1)\n            \n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n      'image/height': dataset_util.int64_feature(height),\n      'image/width': dataset_util.int64_feature(width),\n      'image/filename': dataset_util.bytes_feature(filename),\n      'image/source_id': dataset_util.bytes_feature(filename),\n      'image/encoded': dataset_util.bytes_feature(img),\n      'image/format': dataset_util.bytes_feature(image_format),\n      'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n      'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n      'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n      'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n      'image/object/class/label': dataset_util.int64_list_feature(classes),\n    }))\n    \n    return tf_example\n            \n# tf_example = create_tf_example(0, 13, data_df, os.path.join(INPUT_DIR, 'train_images'))\n\ndef convert_to_tfrecord(data_df, tf_record_file, image_dir_path):\n    with tf.io.TFRecordWriter(tf_record_file) as writer:\n        for index, row in data_df.iterrows():\n            if index % 500 == 0:\n                print('Processed {0} images.'.format(index))\n            tf_example = create_tf_example(row.video_id, row.video_frame, data_df, image_path)\n            writer.write(tf_example.SerializeToString())\n        print('Completed processing {0} images.'.format(len(data_df)))\n\n        \nimage_path = os.path.join(INPUT_DIR, 'train_images')\n!mkdir dataset\n\nprint('Converting TRAIN images...')\nconvert_to_tfrecord(train_data_df, 'dataset/cots_train.tfrecord',image_path)\n\nprint('Converting validation images...')\nconvert_to_tfrecord(val_data_df, 'dataset/cots_val.tfrecord',image_path)","metadata":{"papermill":{"duration":38.916848,"end_time":"2021-11-19T08:41:29.641271","exception":false,"start_time":"2021-11-19T08:40:50.724423","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:10:37.965060Z","iopub.execute_input":"2021-12-20T05:10:37.965562Z","iopub.status.idle":"2021-12-20T05:11:41.360739Z","shell.execute_reply.started":"2021-12-20T05:10:37.965520Z","shell.execute_reply":"2021-12-20T05:11:41.359657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a label map to map between label index and human-readable label name.\n\nlabel_map_str = \"\"\"item {\n  id: 1\n  name: 'COTS'\n}\"\"\"\n\nwith open('dataset/label_map.pbtxt', 'w') as f:\n  f.write(label_map_str)\n\n!more dataset/label_map.pbtxt","metadata":{"papermill":{"duration":16.05844,"end_time":"2021-11-19T08:41:45.838392","exception":false,"start_time":"2021-11-19T08:41:29.779952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:11:41.365998Z","iopub.execute_input":"2021-12-20T05:11:41.366327Z","iopub.status.idle":"2021-12-20T05:11:42.232965Z","shell.execute_reply.started":"2021-12-20T05:11:41.366294Z","shell.execute_reply":"2021-12-20T05:11:42.231865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train an object detection model\n\n### **Model selection**\n\nOne of the coolest features of the TensorFlow Object Detection API is the opportunity to work with a set of state of the art models, pre-trained on the COCO dataset! We can fine-tune these models for our purposes and get great results.\n\nWe'll use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) and an EfficientDet-D4 model and apply transfer learning to train a COTS detection model. You can probably increase accuracy by switch to using a larger EfficientDet model.","metadata":{"papermill":{"duration":3.355764,"end_time":"2021-11-19T08:41:50.909779","exception":false,"start_time":"2021-11-19T08:41:47.554015","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d2_coco17_tpu-32.tar.gz\n!tar -xvzf efficientdet_d2_coco17_tpu-32.tar.gz","metadata":{"papermill":{"duration":3.256545,"end_time":"2021-11-19T08:41:54.358822","exception":false,"start_time":"2021-11-19T08:41:51.102277","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:11:42.235126Z","iopub.execute_input":"2021-12-20T05:11:42.235830Z","iopub.status.idle":"2021-12-20T05:11:46.111906Z","shell.execute_reply.started":"2021-12-20T05:11:42.235779Z","shell.execute_reply":"2021-12-20T05:11:46.110721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz\n!tar -xvzf efficientdet_d0_coco17_tpu-32.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-12-20T05:11:46.114356Z","iopub.execute_input":"2021-12-20T05:11:46.114788Z","iopub.status.idle":"2021-12-20T05:11:49.237871Z","shell.execute_reply.started":"2021-12-20T05:11:46.114726Z","shell.execute_reply":"2021-12-20T05:11:49.236758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model configuration**\nWe downloaded and extracted a pre-trained model of our choice. Now we want to configure it.\n\n**Model configuration** is a process that lets us tailor model-related artifacts (e.g. hyperparameters, loss function, etc) so that it can be trained (fine-tuned) to tackle detection for the objects that we’re interested in. That’s it.\n\nThe TensorFlow Object Detection API allows model configuration via the **pipeline.config** file that goes along with the pre-trained model. ","metadata":{}},{"cell_type":"code","source":"example1 = \"efficientdet_d2_coco17_tpu-32/pipeline.config\"\nfile1 = open(example1, \"r\") \nFileContent = file1.read()\nprint(FileContent)","metadata":{"execution":{"iopub.status.busy":"2021-12-20T05:11:49.240540Z","iopub.execute_input":"2021-12-20T05:11:49.241567Z","iopub.status.idle":"2021-12-20T05:11:49.249273Z","shell.execute_reply.started":"2021-12-20T05:11:49.241489Z","shell.execute_reply":"2021-12-20T05:11:49.248104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example2 = \"efficientdet_d0_coco17_tpu-32/pipeline.config\"\nfile1 = open(example2, \"r\") \nFileContent = file1.read()\nprint(FileContent)","metadata":{"execution":{"iopub.status.busy":"2021-12-20T05:11:49.250959Z","iopub.execute_input":"2021-12-20T05:11:49.251607Z","iopub.status.idle":"2021-12-20T05:11:49.262715Z","shell.execute_reply.started":"2021-12-20T05:11:49.251560Z","shell.execute_reply":"2021-12-20T05:11:49.261696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from string import Template\n\nconfig_file_template = \"\"\"\n# SSD with EfficientNet-b0 + BiFPN feature extractor,\n# shared box predictor and focal loss (a.k.a EfficientDet-d0).\n# See EfficientDet, Tan et al, https://arxiv.org/abs/1911.09070\n# See Lin et al, https://arxiv.org/abs/1708.02002\n# Initialized from an EfficientDet-D0 checkpoint.\n#\n# Train on GPU\n\nmodel {\n  ssd {\n    num_classes: 1\n    box_coder {\n      faster_rcnn_box_coder {\n        y_scale: 10.0\n        x_scale: 10.0\n        height_scale: 5.0\n        width_scale: 5.0\n      }\n    }\n    \n    image_resizer {\n      keep_aspect_ratio_resizer {\n        min_dimension: 1280\n        max_dimension: 1280\n        pad_to_max_dimension: true\n      }\n    }\n    feature_extractor {\n      type: \"ssd_efficientnet-b2_bifpn_keras\"\n      conv_hyperparams {\n        regularizer {\n          l2_regularizer {\n            weight: 3.9999998989515007e-05\n          }\n        }\n        initializer {\n          truncated_normal_initializer {\n            mean: 0.0\n            stddev: 0.029999999329447746\n          }\n        }\n        activation: SWISH\n        batch_norm {\n          decay: 0.9900000095367432\n          scale: true\n          epsilon: 0.0010000000474974513\n        }\n        force_use_bias: true\n      }\n      bifpn {\n        min_level: 3\n        max_level: 7\n        num_iterations: 5\n        num_filters: 112\n      }\n    }\n    matcher {\n      argmax_matcher {\n        matched_threshold: 0.5\n        unmatched_threshold: 0.5\n        ignore_thresholds: false\n        negatives_lower_than_unmatched: true\n        force_match_for_each_row: true\n        use_matmul_gather: true\n      }\n    }\n    similarity_calculator {\n      iou_similarity {\n      }\n    }\n    box_predictor {\n      weight_shared_convolutional_box_predictor {\n        conv_hyperparams {\n          regularizer {\n            l2_regularizer {\n              weight: 3.9999998989515007e-05\n            }\n          }\n          initializer {\n            random_normal_initializer {\n              mean: 0.0\n              stddev: 0.01\n            }\n          }\n          activation: SWISH\n          batch_norm {\n            decay: 0.9900000095367432\n            scale: true\n            epsilon: 0.0010000000474974513\n          }\n          force_use_bias: true\n        }\n        depth: 112\n        num_layers_before_predictor: 3\n        kernel_size: 3\n        class_prediction_bias_init: -4.599999904632568\n        use_depthwise: true\n      }\n    }\n    anchor_generator {\n      multiscale_anchor_generator {\n        min_level: 3\n        max_level: 7\n        anchor_scale: 4.0\n        aspect_ratios: 1.0\n        aspect_ratios: 2.0\n        aspect_ratios: 0.5\n        scales_per_octave: 3\n      }\n    }\n    post_processing {\n      batch_non_max_suppression {\n        score_threshold: 9.99999993922529e-09\n        iou_threshold: 0.5\n        max_detections_per_class: 100\n        max_total_detections: 100\n      }\n      score_converter: SIGMOID\n    }\n    normalize_loss_by_num_matches: true\n    loss {\n      localization_loss {\n        weighted_smooth_l1 {\n        }\n      }\n      classification_loss {\n        weighted_sigmoid_focal {\n          alpha: 0.25\n          gamma: 1.5\n        }\n      }\n      classification_weight: 1.0\n      localization_weight: 1.0\n    }\n    encode_background_as_zeros: true\n    normalize_loc_loss_by_codesize: true\n    inplace_batchnorm_update: true\n    freeze_batchnorm: false\n    add_background_class: false\n  }\n}\ntrain_config {\n  batch_size: 2\n  data_augmentation_options {\n    random_horizontal_flip {\n    }\n  }\n  data_augmentation_options {\n    random_scale_crop_and_pad_to_square {\n      output_size: 1280\n      scale_min: 0.5\n      scale_max: 2.0\n    }\n  }\n  sync_replicas: false\n  optimizer {\n    momentum_optimizer: {\n      learning_rate: {\n        cosine_decay_learning_rate {\n          learning_rate_base: 5e-3\n          total_steps: $training_steps\n          warmup_learning_rate: 5e-4\n          warmup_steps: $warmup_steps\n        }\n      }\n      momentum_optimizer_value: 0.9\n    }\n    use_moving_average: false\n  }\n  fine_tune_checkpoint: \"efficientdet_d2_coco17_tpu-32/checkpoint/ckpt-0\"\n  num_steps: $training_steps\n  startup_delay_steps: 0.0\n  replicas_to_aggregate: 1\n  max_number_of_boxes: 100\n  unpad_groundtruth_tensors: false\n  fine_tune_checkpoint_type: \"detection\"\n  use_bfloat16: false\n  fine_tune_checkpoint_version: V2\n}\ntrain_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  tf_record_input_reader {\n    input_path: \"dataset/cots_train.tfrecord\"\n  }\n}\n\neval_config: {\n  metrics_set: \"coco_detection_metrics\"\n  use_moving_averages: false\n  batch_size: 1;\n}\n\neval_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  shuffle: false\n  num_epochs: 1\n  tf_record_input_reader {\n    input_path: \"dataset/cots_val.tfrecord\"\n  }\n}\n\n\"\"\"","metadata":{"papermill":{"duration":0.133468,"end_time":"2021-11-19T08:41:54.609774","exception":false,"start_time":"2021-11-19T08:41:54.476306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:11:49.264737Z","iopub.execute_input":"2021-12-20T05:11:49.265556Z","iopub.status.idle":"2021-12-20T05:11:49.276586Z","shell.execute_reply.started":"2021-12-20T05:11:49.265498Z","shell.execute_reply":"2021-12-20T05:11:49.275600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the training pipeline\n\nTRAINING_STEPS = 20000\nWARMUP_STEPS = 2000\nPIPELINE_CONFIG_PATH='dataset/pipeline.config'\n\npipeline = Template(config_file_template).substitute(\n    training_steps=TRAINING_STEPS, warmup_steps=WARMUP_STEPS)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{"papermill":{"duration":0.121946,"end_time":"2021-11-19T08:41:54.846958","exception":false,"start_time":"2021-11-19T08:41:54.725012","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:11:49.278397Z","iopub.execute_input":"2021-12-20T05:11:49.279362Z","iopub.status.idle":"2021-12-20T05:11:49.293723Z","shell.execute_reply.started":"2021-12-20T05:11:49.279317Z","shell.execute_reply":"2021-12-20T05:11:49.292604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR='cots_efficientdet_d2'\n!mkdir {MODEL_DIR}\n!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --alsologtostderr","metadata":{"papermill":{"duration":18463.625406,"end_time":"2021-11-19T13:49:38.586506","exception":false,"start_time":"2021-11-19T08:41:54.9611","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T05:11:49.297344Z","iopub.execute_input":"2021-12-20T05:11:49.297921Z","iopub.status.idle":"2021-12-20T13:25:54.286736Z","shell.execute_reply.started":"2021-12-20T05:11:49.297880Z","shell.execute_reply":"2021-12-20T13:25:54.284242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the object detection model","metadata":{"papermill":{"duration":0.226793,"end_time":"2021-11-19T13:49:39.043281","exception":false,"start_time":"2021-11-19T13:49:38.816488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"k\")","metadata":{"execution":{"iopub.status.busy":"2021-12-20T13:25:54.292813Z","iopub.execute_input":"2021-12-20T13:25:54.293949Z","iopub.status.idle":"2021-12-20T13:25:54.306025Z","shell.execute_reply.started":"2021-12-20T13:25:54.293914Z","shell.execute_reply":"2021-12-20T13:25:54.304652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --checkpoint_dir={MODEL_DIR} \\\n    --eval_timeout=0 \\\n    --alsologtostderr","metadata":{"papermill":{"duration":323.73381,"end_time":"2021-11-19T13:55:03.004285","exception":false,"start_time":"2021-11-19T13:49:39.270475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:25:54.307646Z","iopub.execute_input":"2021-12-20T13:25:54.308301Z","iopub.status.idle":"2021-12-20T13:31:23.967107Z","shell.execute_reply.started":"2021-12-20T13:25:54.308254Z","shell.execute_reply":"2021-12-20T13:31:23.966064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export as SavedModel for inference","metadata":{"papermill":{"duration":0.239823,"end_time":"2021-11-19T13:55:03.483464","exception":false,"start_time":"2021-11-19T13:55:03.243641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python models/research/object_detection/exporter_main_v2.py \\\n    --input_type image_tensor \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --trained_checkpoint_dir={MODEL_DIR} \\\n    --output_directory={MODEL_DIR}/output","metadata":{"papermill":{"duration":122.093553,"end_time":"2021-11-19T13:57:05.815158","exception":false,"start_time":"2021-11-19T13:55:03.721605","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:31:23.969375Z","iopub.execute_input":"2021-12-20T13:31:23.969741Z","iopub.status.idle":"2021-12-20T13:34:55.388175Z","shell.execute_reply.started":"2021-12-20T13:31:23.969698Z","shell.execute_reply":"2021-12-20T13:34:55.387005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {MODEL_DIR}/output","metadata":{"papermill":{"duration":0.931074,"end_time":"2021-11-19T13:57:06.998967","exception":false,"start_time":"2021-11-19T13:57:06.067893","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:34:55.392618Z","iopub.execute_input":"2021-12-20T13:34:55.392910Z","iopub.status.idle":"2021-12-20T13:34:56.197758Z","shell.execute_reply.started":"2021-12-20T13:34:55.392859Z","shell.execute_reply":"2021-12-20T13:34:56.196622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run inference on test images and create the submission file","metadata":{"papermill":{"duration":0.24428,"end_time":"2021-11-19T13:57:07.485159","exception":false,"start_time":"2021-11-19T13:57:07.240879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load the TensorFlow COTS detection model into memory.\nstart_time = time.time()\ntf.keras.backend.clear_session()\ndetect_fn_tf_odt = tf.saved_model.load(os.path.join(os.path.join(MODEL_DIR, 'output'), 'saved_model'))\nend_time = time.time()\nelapsed_time = end_time - start_time\nprint('Elapsed time: ' + str(elapsed_time) + 's')","metadata":{"papermill":{"duration":28.095431,"end_time":"2021-11-19T13:57:35.83037","exception":false,"start_time":"2021-11-19T13:57:07.734939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:34:56.200139Z","iopub.execute_input":"2021-12-20T13:34:56.200507Z","iopub.status.idle":"2021-12-20T13:35:49.353729Z","shell.execute_reply.started":"2021-12-20T13:34:56.200457Z","shell.execute_reply":"2021-12-20T13:35:49.352381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define some utils method for prediction.\n\ndef load_image_into_numpy_array(path):\n    \"\"\"Load an image from file into a numpy array.\n\n    Puts image into numpy array to feed into tensorflow graph.\n    Note that by convention we put it into a numpy array with shape\n    (height, width, channels), where channels=3 for RGB.\n\n    Args:\n    path: a file path (this can be local or on colossus)\n\n    Returns:\n    uint8 numpy array with shape (img_height, img_width, 3)\n    \"\"\"\n    img_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(io.BytesIO(img_data))\n    (im_width, im_height) = image.size\n    \n    return np.array(image.getdata()).reshape(\n      (im_height, im_width, 3)).astype(np.uint8)\n\ndef detect(image_np):\n    \"\"\"Detect COTS from a given numpy image.\"\"\"\n\n    input_tensor = np.expand_dims(image_np, 0)\n    start_time = time.time()\n    detections = detect_fn_tf_odt(input_tensor)\n    return detections","metadata":{"papermill":{"duration":0.417072,"end_time":"2021-11-19T13:57:36.586877","exception":false,"start_time":"2021-11-19T13:57:36.169805","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:35:49.355653Z","iopub.execute_input":"2021-12-20T13:35:49.357769Z","iopub.status.idle":"2021-12-20T13:35:49.367261Z","shell.execute_reply.started":"2021-12-20T13:35:49.357718Z","shell.execute_reply":"2021-12-20T13:35:49.365964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"papermill":{"duration":0.354385,"end_time":"2021-11-19T13:57:37.442493","exception":false,"start_time":"2021-11-19T13:57:37.088108","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:35:49.369070Z","iopub.execute_input":"2021-12-20T13:35:49.369629Z","iopub.status.idle":"2021-12-20T13:35:49.391643Z","shell.execute_reply.started":"2021-12-20T13:35:49.369582Z","shell.execute_reply":"2021-12-20T13:35:49.390451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DETECTION_THRESHOLD = 0.3\n\nsubmission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n    height, width, _ = image_np.shape\n    \n    # Run object detection using the TensorFlow model.\n    detections = detect(image_np)\n    \n    # Parse the detection result and generate a prediction string.\n    num_detections = detections['num_detections'][0].numpy().astype(np.int32)\n    predictions = []\n    for index in range(num_detections):\n        score = detections['detection_scores'][0][index].numpy()\n        if score < DETECTION_THRESHOLD:\n            continue\n\n        bbox = detections['detection_boxes'][0][index].numpy()\n        y_min = int(bbox[0] * height)\n        x_min = int(bbox[1] * width)\n        y_max = int(bbox[2] * height)\n        x_max = int(bbox[3] * width)\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    # Generate the submission data.\n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"papermill":{"duration":4.526271,"end_time":"2021-11-19T13:57:42.213925","exception":false,"start_time":"2021-11-19T13:57:37.687654","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:35:49.393439Z","iopub.execute_input":"2021-12-20T13:35:49.393835Z","iopub.status.idle":"2021-12-20T13:35:55.544780Z","shell.execute_reply.started":"2021-12-20T13:35:49.393787Z","shell.execute_reply":"2021-12-20T13:35:55.543779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean up","metadata":{"papermill":{"duration":0.264983,"end_time":"2021-11-19T13:57:42.731628","exception":false,"start_time":"2021-11-19T13:57:42.466645","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Remove the dataset files to save space.\n!rm -rf dataset\n!rm -rf train_images\n!rm tensorflow-great-barrier-reef.zip\n\n# Remove other data downloaded during training.\n!rm -rf models\n!rm efficientdet_d0_coco17_tpu-32.tar.gz","metadata":{"papermill":{"duration":4.604917,"end_time":"2021-11-19T13:57:47.580583","exception":false,"start_time":"2021-11-19T13:57:42.975666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-20T13:35:55.546485Z","iopub.execute_input":"2021-12-20T13:35:55.546996Z","iopub.status.idle":"2021-12-20T13:36:00.326390Z","shell.execute_reply.started":"2021-12-20T13:35:55.546938Z","shell.execute_reply":"2021-12-20T13:36:00.325021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}