{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##### [Original Notebook](https://www.kaggle.com/khanhlvg/cots-detection-w-tensorflow-object-detection-api)\n##### This notebook contains code to train a crown-of-thorns starfish (COTS) detection model to serve as a baseline model for [this competition](https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview). \n##### We use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) to apply transfer learning on an [EfficientDet-D0](https://arxiv.org/abs/1911.09070) pretrained model. ","metadata":{}},{"cell_type":"markdown","source":"### Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport pandas as pd\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\nINPUT_DIR = \"../input/tensorflow-great-barrier-reef\"\nsys.path.append(INPUT_DIR)\nimport greatbarrierreef","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:18:48.335431Z","iopub.execute_input":"2022-02-14T07:18:48.335886Z","iopub.status.idle":"2022-02-14T07:18:53.430971Z","shell.execute_reply.started":"2022-02-14T07:18:48.335759Z","shell.execute_reply":"2022-02-14T07:18:53.429884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:18:53.432746Z","iopub.execute_input":"2022-02-14T07:18:53.433061Z","iopub.status.idle":"2022-02-14T07:18:53.504946Z","shell.execute_reply.started":"2022-02-14T07:18:53.433019Z","shell.execute_reply":"2022-02-14T07:18:53.503983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))\ntest","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:18:53.507330Z","iopub.execute_input":"2022-02-14T07:18:53.507854Z","iopub.status.idle":"2022-02-14T07:18:53.527952Z","shell.execute_reply.started":"2022-02-14T07:18:53.507806Z","shell.execute_reply":"2022-02-14T07:18:53.526990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Installing TensorFlow Object Detection API","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models && git checkout ac8d06519\n","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:18:53.531284Z","iopub.execute_input":"2022-02-14T07:18:53.531656Z","iopub.status.idle":"2022-02-14T07:19:22.805101Z","shell.execute_reply.started":"2022-02-14T07:18:53.531614Z","shell.execute_reply":"2022-02-14T07:19:22.803864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd models/research\n\n# Compile protos.\nprotoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow and keras to 2.7.0 to be compatible with the current TF ODT API snapshot\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\nwget -O setup.py https://storage.googleapis.com/odml-dataset/others/setup_tf27.py\npip install -q --user .\n\n# Test if the Object Dectection API is working correctly\npython object_detection/builders/model_builder_tf2_test.py","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:19:22.807611Z","iopub.execute_input":"2022-02-14T07:19:22.807953Z","iopub.status.idle":"2022-02-14T07:22:15.052384Z","shell.execute_reply.started":"2022-02-14T07:19:22.807884Z","shell.execute_reply":"2022-02-14T07:22:15.051045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)\nprint(tf.test.is_gpu_available())\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:15.054553Z","iopub.execute_input":"2022-02-14T07:22:15.054797Z","iopub.status.idle":"2022-02-14T07:22:17.752807Z","shell.execute_reply.started":"2022-02-14T07:22:15.054763Z","shell.execute_reply":"2022-02-14T07:22:17.749323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparing the training dataset","metadata":{}},{"cell_type":"markdown","source":"#### Split the train folder into training and validation datasets","metadata":{}},{"cell_type":"code","source":"TRAINING_RATIO = 0.8\n\ndata_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n\n# Split the dataset so that no sequence is leaked from the training dataset into the validation dataset.\nsplit_index = int(TRAINING_RATIO * len(data_df))\nwhile data_df.iloc[split_index - 1].sequence == data_df.iloc[split_index].sequence:\n    split_index += 1\n\n# Shuffle both the training and validation datasets.\ntrain_data_df = data_df.iloc[:split_index].sample(frac=1).reset_index(drop=True)\nval_data_df = data_df.iloc[split_index:].sample(frac=1).reset_index(drop=True)\n\ntrain_positive_count = len(train_data_df[train_data_df.annotations != '[]'])\nval_positive_count = len(val_data_df[val_data_df.annotations != '[]'])\n\nprint('Training ratio (all samples):', \n      float(len(train_data_df)) / (len(train_data_df) + len(val_data_df)))\nprint('Training ratio (positive samples):', \n      float(train_positive_count) / (train_positive_count + val_positive_count))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:17.754348Z","iopub.execute_input":"2022-02-14T07:22:17.754876Z","iopub.status.idle":"2022-02-14T07:22:17.970543Z","shell.execute_reply.started":"2022-02-14T07:22:17.754813Z","shell.execute_reply":"2022-02-14T07:22:17.969381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Just taking into account the images with annotations from both the train and validation dataset","metadata":{}},{"cell_type":"code","source":"train_data_df = train_data_df[train_data_df.annotations != '[]'].reset_index()\nprint(\"Positive Train Annotated Samples: \", len(train_data_df))\n\nval_data_df = val_data_df[val_data_df.annotations !='[]'].reset_index()\nprint(\"Positive Validation Annotated Samples: \", len(val_data_df))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:17.972193Z","iopub.execute_input":"2022-02-14T07:22:17.972520Z","iopub.status.idle":"2022-02-14T07:22:17.991056Z","shell.execute_reply.started":"2022-02-14T07:22:17.972475Z","shell.execute_reply":"2022-02-14T07:22:17.989685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Visualize one randomly selected image from the training set to see if annotations looks correct","metadata":{}},{"cell_type":"code","source":"train_data_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:17.993065Z","iopub.execute_input":"2022-02-14T07:22:17.993408Z","iopub.status.idle":"2022-02-14T07:22:18.008182Z","shell.execute_reply.started":"2022-02-14T07:22:17.993362Z","shell.execute_reply":"2022-02-14T07:22:18.006981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:18.012919Z","iopub.execute_input":"2022-02-14T07:22:18.013603Z","iopub.status.idle":"2022-02-14T07:22:18.028965Z","shell.execute_reply.started":"2022-02-14T07:22:18.013555Z","shell.execute_reply":"2022-02-14T07:22:18.027493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_with_annotations(video_id, video_frame, data_df, image_path):\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}',f'{video_frame}.jpg'))\n    image = Image.open(full_path)\n    draw = ImageDraw.Draw(image)\n    \n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\",'\"'))\n        for annotation in annotations:\n            draw.rectangle((\n                annotation['x'],\n                annotation['y'],\n                (annotation['x'] + annotation['width']),\n                (annotation['y'] + annotation['height']),\n            ), outline = (255, 0, 0))\n    buf = io.BytesIO()\n    image.save(buf, 'png')\n    data = buf.getvalue()\n    return data\n\n# Test visualisation of a randomly selected image\nimage_path = os.path.join(INPUT_DIR, 'train_images')\ntest_index = 20\nvideo_id = train_data_df.iloc[test_index].video_id\nvideo_frame = train_data_df.iloc[test_index].video_frame\nIPython.display.Image(image_with_annotations(video_id, video_frame, train_data_df, image_path))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:18.031122Z","iopub.execute_input":"2022-02-14T07:22:18.031617Z","iopub.status.idle":"2022-02-14T07:22:18.412442Z","shell.execute_reply.started":"2022-02-14T07:22:18.031570Z","shell.execute_reply":"2022-02-14T07:22:18.411283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df['annotations'][20]","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:18.414217Z","iopub.execute_input":"2022-02-14T07:22:18.414549Z","iopub.status.idle":"2022-02-14T07:22:18.422261Z","shell.execute_reply.started":"2022-02-14T07:22:18.414508Z","shell.execute_reply":"2022-02-14T07:22:18.421365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Converting To TFRecord","metadata":{}},{"cell_type":"code","source":"BytesList = tf.train.BytesList\nFloatList = tf.train.FloatList\nIntList = tf.train.Int64List\n\nfrom object_detection.utils import dataset_util\n\ndef image_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(\n        bytes_list=BytesList(value=[tf.io.encode_jpeg(value).numpy()])\n    )\n\ndef bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=BytesList(value=[value.encode()]))\n\ndef bytes_feature_list(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=BytesList(value=value))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=FloatList(value=[value]))\n\n\ndef float_feature_list(value):\n    \"\"\"Returns a list of float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=IntList(value=[value]))\n\ndef _int64_feature_list(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=IntList(value=value))\n\n\n\ndef create_tf_example(video_id, video_frame, data_df, image_dir_path):\n    full_path = os.path.join(image_dir_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    img = open(full_path,'rb').read()\n    filename = f'{video_id}:{video_frame}'.encode('utf8')\n    image = Image.open(full_path)\n    image_format = 'jpeg'.encode('utf8')\n    \n    height = image.size[1] # Image height\n    width = image.size[0] # Image width\n    \n    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)\n    xmaxs = [] # List of normalized right x coordinates in bounding box\n             # (1 per box)\n    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)\n    ymaxs = [] # List of normalized bottom y coordinates in bounding box\n             # (1 per box)\n    classes_text = [] # List of string class name of bounding box (1 per box)\n    classes = [] # List of integer class id of bounding box (1 per box)\n    \n    \n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            xmins.append(annotation['x'] / width) \n            xmaxs.append((annotation['x'] + annotation['width']) / width) \n            ymins.append(annotation['y'] / height) \n            ymaxs.append((annotation['y'] + annotation['height']) / height) \n\n            classes_text.append('COTS'.encode('utf8'))\n            classes.append(1)\n            \n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n      'image/height': dataset_util.int64_feature(height),\n      'image/width': dataset_util.int64_feature(width),\n      'image/filename': dataset_util.bytes_feature(filename),\n      'image/source_id': dataset_util.bytes_feature(filename),\n      'image/encoded': dataset_util.bytes_feature(img),\n      'image/format': dataset_util.bytes_feature(image_format),\n      'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n      'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n      'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n      'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n      'image/object/class/label': dataset_util.int64_list_feature(classes),\n    }))\n    \n    return tf_example\n            \n# tf_example = create_tf_example(0, 13, data_df, os.path.join(INPUT_DIR, 'train_images'))\n\ndef convert_to_tfrecord(data_df, tf_record_file, image_dir_path):\n    with tf.io.TFRecordWriter(tf_record_file) as writer:\n        for index, row in data_df.iterrows():\n            if index % 500 == 0:\n                print('Processed {0} images.'.format(index))\n            tf_example = create_tf_example(row.video_id, row.video_frame, data_df, image_path)\n            writer.write(tf_example.SerializeToString())\n        print('Completed processing {0} images.'.format(len(data_df)))\n\n        \nimage_path = os.path.join(INPUT_DIR, 'train_images')\n!mkdir dataset\n\nprint('Converting TRAIN images...')\nconvert_to_tfrecord(train_data_df, 'dataset/cots_train.tfrecord',image_path)\n\nprint('Converting validation images...')\nconvert_to_tfrecord(val_data_df, 'dataset/cots_val.tfrecord',image_path)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:22:18.424282Z","iopub.execute_input":"2022-02-14T07:22:18.425051Z","iopub.status.idle":"2022-02-14T07:23:23.499893Z","shell.execute_reply.started":"2022-02-14T07:22:18.424876Z","shell.execute_reply":"2022-02-14T07:23:23.498667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a label map to map between label index and human-readble label name\n\nlabel_map_str = \"\"\"item{\nid:1\nname: 'COTS'\n}\n\n\"\"\"\nwith open('dataset/label_map.pbtxt', 'w') as f:\n    f.write(label_map_str)\n!more dataset/label_map.pbtxt","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:23.501840Z","iopub.execute_input":"2022-02-14T07:23:23.503001Z","iopub.status.idle":"2022-02-14T07:23:24.301108Z","shell.execute_reply.started":"2022-02-14T07:23:23.502918Z","shell.execute_reply":"2022-02-14T07:23:24.299725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train an object detection model","metadata":{"execution":{"iopub.status.busy":"2022-01-09T10:05:47.302443Z","iopub.execute_input":"2022-01-09T10:05:47.302737Z","iopub.status.idle":"2022-01-09T10:05:47.306659Z","shell.execute_reply.started":"2022-01-09T10:05:47.302691Z","shell.execute_reply":"2022-01-09T10:05:47.30598Z"}}},{"cell_type":"code","source":"!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz\n!tar -xvzf faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:24.308069Z","iopub.execute_input":"2022-02-14T07:23:24.320296Z","iopub.status.idle":"2022-02-14T07:23:35.892191Z","shell.execute_reply.started":"2022-02-14T07:23:24.320226Z","shell.execute_reply":"2022-02-14T07:23:35.890809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:35.894911Z","iopub.execute_input":"2022-02-14T07:23:35.895317Z","iopub.status.idle":"2022-02-14T07:23:36.798685Z","shell.execute_reply.started":"2022-02-14T07:23:35.895252Z","shell.execute_reply":"2022-02-14T07:23:36.797421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example1=\"faster_rcnn_resnet101_v1_640x640_coco17_tpu-8/pipeline.config\"\nfile1 = open(example1, 'r')\nFileContent = file1.read()\nprint(FileContent)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:36.801434Z","iopub.execute_input":"2022-02-14T07:23:36.802275Z","iopub.status.idle":"2022-02-14T07:23:36.810814Z","shell.execute_reply.started":"2022-02-14T07:23:36.802220Z","shell.execute_reply":"2022-02-14T07:23:36.809562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from string import Template\n\nconfig_file_template = \"\"\"\n# Faster R-CNN with Resnet-50 (v1)\n# Trained on COCO, initialized from Imagenet classification checkpoint\n#\n# Train on TPU-8\n#\n# Achieves 31.8 mAP on COCO17 val\n\nmodel {\n  faster_rcnn {\n    num_classes: 1\n    image_resizer {\n      keep_aspect_ratio_resizer {\n        min_dimension: 960\n        max_dimension: 960\n        pad_to_max_dimension: true\n      }\n    }\n    feature_extractor {\n      type: 'faster_rcnn_resnet101_keras'\n      batch_norm_trainable: true\n    }\n    first_stage_anchor_generator {\n      grid_anchor_generator {\n        scales: [0.25, 0.5, 1.0, 2.0]\n        aspect_ratios: [0.5, 1.0, 2.0]\n        height_stride: 16\n        width_stride: 16\n      }\n    }\n    first_stage_box_predictor_conv_hyperparams {\n      op: CONV\n      regularizer {\n        l2_regularizer {\n          weight: 0.0\n        }\n      }\n      initializer {\n        truncated_normal_initializer {\n          stddev: 0.01\n        }\n      }\n    }\n    first_stage_nms_score_threshold: 0.0\n    first_stage_nms_iou_threshold: 0.7\n    first_stage_max_proposals: 300\n    first_stage_localization_loss_weight: 2.0\n    first_stage_objectness_loss_weight: 1.0\n    initial_crop_size: 14\n    maxpool_kernel_size: 2\n    maxpool_stride: 2\n    second_stage_box_predictor {\n      mask_rcnn_box_predictor {\n        use_dropout: false\n        dropout_keep_probability: 1.0\n        fc_hyperparams {\n          op: FC\n          regularizer {\n            l2_regularizer {\n              weight: 0.0\n            }\n          }\n          initializer {\n            variance_scaling_initializer {\n              factor: 1.0\n              uniform: true\n              mode: FAN_AVG\n            }\n          }\n        }\n        share_box_across_classes: true\n      }\n    }\n    second_stage_post_processing {\n      batch_non_max_suppression {\n        score_threshold: 0.0\n        iou_threshold: 0.6\n        max_detections_per_class: 100\n        max_total_detections: 300\n      }\n      score_converter: SOFTMAX\n    }\n    second_stage_localization_loss_weight: 2.0\n    second_stage_classification_loss_weight: 1.0\n    use_static_shapes: true\n    use_matmul_crop_and_resize: true\n    clip_anchors_to_image: true\n    use_static_balanced_label_sampler: true\n    use_matmul_gather_in_matcher: true\n  }\n}\n\ntrain_config: {\n  fine_tune_checkpoint: \"faster_rcnn_resnet101_v1_640x640_coco17_tpu-8/checkpoint/ckpt-0\"\n  fine_tune_checkpoint_version: V2\n  fine_tune_checkpoint_type: \"detection\"\n  batch_size: 1\n  sync_replicas: false\n  startup_delay_steps: 0\n  replicas_to_aggregate: 1\n  use_bfloat16: false\n  num_steps: $training_steps\n  data_augmentation_options {\n    random_horizontal_flip {\n    }\n  }\n  data_augmentation_options {\n    random_scale_crop_and_pad_to_square {\n      output_size: 1280\n      scale_min: 0.5\n      scale_max: 2.0\n    }\n  }\n  optimizer {\n    momentum_optimizer: {\n      learning_rate: {\n        cosine_decay_learning_rate {\n          learning_rate_base: 5e-3\n          total_steps: $training_steps\n          warmup_learning_rate: 5e-4\n          warmup_steps: $warmup_steps\n        }\n      }\n      momentum_optimizer_value: 0.9\n    }\n    use_moving_average: false\n  }\n  max_number_of_boxes: 100\n  unpad_groundtruth_tensors: false\n}\n\ntrain_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  tf_record_input_reader {\n    input_path: \"dataset/cots_train.tfrecord\"\n  }\n}\n\neval_config: {\n  metrics_set: \"coco_detection_metrics\"\n  use_moving_averages: false\n  batch_size: 2;\n}\n\neval_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  shuffle: false\n  num_epochs: 1\n  tf_record_input_reader {\n    input_path: \"dataset/cots_val.tfrecord\"\n  }\n}\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:36.813262Z","iopub.execute_input":"2022-02-14T07:23:36.814403Z","iopub.status.idle":"2022-02-14T07:23:36.825119Z","shell.execute_reply.started":"2022-02-14T07:23:36.814314Z","shell.execute_reply":"2022-02-14T07:23:36.823755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the training pipeline\n\nTRAINING_STEPS = 500\nWARMUP_STEPS = 100\nPIPELINE_CONFIG_PATH = 'dataset/pipeline.config'\n\npipeline = Template(config_file_template).substitute(\n    training_steps = TRAINING_STEPS, warmup_steps=WARMUP_STEPS)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:36.827194Z","iopub.execute_input":"2022-02-14T07:23:36.828140Z","iopub.status.idle":"2022-02-14T07:23:36.841522Z","shell.execute_reply.started":"2022-02-14T07:23:36.828088Z","shell.execute_reply":"2022-02-14T07:23:36.840356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR='cots_faster_rcnn_resnet101'\n!mkdir {MODEL_DIR}","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:36.845736Z","iopub.execute_input":"2022-02-14T07:23:36.846159Z","iopub.status.idle":"2022-02-14T07:23:37.672486Z","shell.execute_reply.started":"2022-02-14T07:23:36.846125Z","shell.execute_reply":"2022-02-14T07:23:37.670856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --alsologtostderr","metadata":{"execution":{"iopub.status.busy":"2022-02-14T07:23:37.675082Z","iopub.execute_input":"2022-02-14T07:23:37.675475Z","iopub.status.idle":"2022-02-14T11:24:56.211539Z","shell.execute_reply.started":"2022-02-14T07:23:37.675413Z","shell.execute_reply":"2022-02-14T11:24:56.210425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluating the Model","metadata":{}},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --checkpoint_dir={MODEL_DIR} \\\n    --eval_timeout=0 \\\n    --alsologtostderr","metadata":{"execution":{"iopub.status.busy":"2022-02-14T11:27:46.306185Z","iopub.execute_input":"2022-02-14T11:27:46.306534Z","iopub.status.idle":"2022-02-14T13:15:16.118594Z","shell.execute_reply.started":"2022-02-14T11:27:46.306482Z","shell.execute_reply":"2022-02-14T13:15:16.117365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exporting the model","metadata":{}},{"cell_type":"code","source":"!python models/research/object_detection/exporter_main_v2.py \\\n    --input_type image_tensor \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --trained_checkpoint_dir={MODEL_DIR} \\\n    --output_directory={MODEL_DIR}/output","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:15:16.121800Z","iopub.execute_input":"2022-02-14T13:15:16.122728Z","iopub.status.idle":"2022-02-14T13:16:58.345913Z","shell.execute_reply.started":"2022-02-14T13:15:16.122675Z","shell.execute_reply":"2022-02-14T13:16:58.344757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {MODEL_DIR}/output","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:16:58.348560Z","iopub.execute_input":"2022-02-14T13:16:58.348993Z","iopub.status.idle":"2022-02-14T13:16:59.116256Z","shell.execute_reply.started":"2022-02-14T13:16:58.348920Z","shell.execute_reply":"2022-02-14T13:16:59.115114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Run inference on test images and create the submission file","metadata":{}},{"cell_type":"code","source":"# Load the TensorFlow COTS detection model into memory.\nstart_time = time.time()\n#tf.keras.backend.clear_session()\ndetect_fn_tf_odt = tf.saved_model.load(os.path.join(os.path.join(MODEL_DIR, 'output'), 'saved_model'))\nend_time = time.time()\nelapsed_time = end_time - start_time\nprint('Elapsed time: ' + str(elapsed_time) + 's')","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:16:59.120195Z","iopub.execute_input":"2022-02-14T13:16:59.121096Z","iopub.status.idle":"2022-02-14T13:17:17.362256Z","shell.execute_reply.started":"2022-02-14T13:16:59.121059Z","shell.execute_reply":"2022-02-14T13:17:17.360116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define some utils method for prediction.\n\ndef load_image_into_numpy_array(path):\n    \n    img_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(io.BytesIO(img_data))\n    (im_width, im_height) = image.size\n    \n    return np.array(image.getdata()).reshape(\n      (im_height, im_width, 3)).astype(np.uint8)\n\ndef detect(image_np):\n    \"\"\"Detect COTS from a given numpy image.\"\"\"\n\n    input_tensor = np.expand_dims(image_np, 0)\n    start_time = time.time()\n    detections = detect_fn_tf_odt(input_tensor)\n    return detections","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:23:39.218185Z","iopub.execute_input":"2022-02-14T13:23:39.218789Z","iopub.status.idle":"2022-02-14T13:23:39.227283Z","shell.execute_reply.started":"2022-02-14T13:23:39.218750Z","shell.execute_reply":"2022-02-14T13:23:39.225656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:23:42.903915Z","iopub.execute_input":"2022-02-14T13:23:42.904639Z","iopub.status.idle":"2022-02-14T13:23:42.910486Z","shell.execute_reply.started":"2022-02-14T13:23:42.904602Z","shell.execute_reply":"2022-02-14T13:23:42.908918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DETECTION_THRESHOLD = 0.3\n\nsubmission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n    height, width, _ = image_np.shape\n    \n    # Run object detection using the TensorFlow model.\n    detections = detect(image_np)\n    \n    # Parse the detection result and generate a prediction string.\n    num_detections = detections['num_detections'][0].numpy().astype(np.int32)\n    predictions = []\n    for index in range(num_detections):\n        score = detections['detection_scores'][0][index].numpy()\n        if score < DETECTION_THRESHOLD:\n            continue\n\n        bbox = detections['detection_boxes'][0][index].numpy()\n        y_min = int(bbox[0] * height)\n        x_min = int(bbox[1] * width)\n        y_max = int(bbox[2] * height)\n        x_max = int(bbox[3] * width)\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    # Generate the submission data.\n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:25:25.428314Z","iopub.execute_input":"2022-02-14T13:25:25.428608Z","iopub.status.idle":"2022-02-14T13:25:25.439512Z","shell.execute_reply.started":"2022-02-14T13:25:25.428576Z","shell.execute_reply":"2022-02-14T13:25:25.438249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf dataset/\n!rm -rf faster_rcnn_resnet101_v1_640x640_coco17_tpu-8/\n!rm -rf models/","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:26:05.600547Z","iopub.execute_input":"2022-02-14T13:26:05.600864Z","iopub.status.idle":"2022-02-14T13:26:08.522393Z","shell.execute_reply.started":"2022-02-14T13:26:05.600816Z","shell.execute_reply":"2022-02-14T13:26:08.521171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}