{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This notebook contains code to train a crown-of-thorns starfish (COTS) detection model to serve as a baseline model for [this competition](https://www.kaggle.com/c/tensorflow-great-barrier-reef/overview). We use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) to apply transfer learning on an [EfficientDet-D4](https://arxiv.org/abs/1911.09070) pretrained model. ","metadata":{"papermill":{"duration":0.023199,"end_time":"2021-11-19T08:38:07.360759","exception":false,"start_time":"2021-11-19T08:38:07.33756","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Install TensorFlow Object Detection API\n\nPip may report some dependency errors. You can safely ignore these errors and proceed if all tests in `model_builder_tf2_test.py` passed. ","metadata":{"papermill":{"duration":0.021045,"end_time":"2021-11-19T08:38:07.403389","exception":false,"start_time":"2021-11-19T08:38:07.382344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models && git checkout ac8d06519","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":25.822618,"end_time":"2021-11-19T08:38:33.247836","exception":false,"start_time":"2021-11-19T08:38:07.425218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-25T07:33:51.796186Z","iopub.execute_input":"2021-11-25T07:33:51.796456Z","iopub.status.idle":"2021-11-25T07:34:11.222181Z","shell.execute_reply.started":"2021-11-25T07:33:51.796428Z","shell.execute_reply":"2021-11-25T07:34:11.221276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd models/research\n\n# Compile protos.\n!protoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow and keras to 2.4.1 to be compatible with the current Kaggle TPU coordinator version\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\n!wget -O setup.py https://storage.googleapis.com/odml-dataset/others/setup_tf27.py\n!pip install -q --user .\n\n# Test if the Object Dectection API is working correctly\n!python object_detection/builders/model_builder_tf2_test.py\n\n# Upgrade tensorflow_gcs_config to the same version as Tensorflow.\n!pip install tensorflow_gcs_config==2.7.0\n\n%cd ../..","metadata":{"papermill":{"duration":79.154739,"end_time":"2021-11-19T08:39:52.448588","exception":false,"start_time":"2021-11-19T08:38:33.293849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-25T07:34:11.224186Z","iopub.execute_input":"2021-11-25T07:34:11.224453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import dependencies","metadata":{}},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'\nsys.path.insert(0, INPUT_DIR)\nimport greatbarrierreef","metadata":{"papermill":{"duration":1.526137,"end_time":"2021-11-19T08:39:54.022253","exception":false,"start_time":"2021-11-19T08:39:52.496116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T08:35:26.252493Z","iopub.execute_input":"2021-11-24T08:35:26.252794Z","iopub.status.idle":"2021-11-24T08:35:26.259909Z","shell.execute_reply.started":"2021-11-24T08:35:26.252759Z","shell.execute_reply":"2021-11-24T08:35:26.259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The notebook is supposed to run with TF 2.4.1\n!pip install -q --user cloud_tpu_client\nfrom cloud_tpu_client import Client\n\nc = Client()\nc.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n\nprint(tf.__version__)\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nprint(\"All devices: \", tf.config.list_logical_devices('TPU'))","metadata":{"papermill":{"duration":0.729459,"end_time":"2021-11-19T08:39:54.798333","exception":false,"start_time":"2021-11-19T08:39:54.068874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T07:43:41.701243Z","iopub.execute_input":"2021-11-24T07:43:41.701488Z","iopub.status.idle":"2021-11-24T07:43:55.311545Z","shell.execute_reply.started":"2021-11-24T07:43:41.701461Z","shell.execute_reply":"2021-11-24T07:43:55.310445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)\nuser_secrets.set_gcloud_credentials()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:27:44.277649Z","iopub.execute_input":"2021-11-24T10:27:44.27799Z","iopub.status.idle":"2021-11-24T10:27:50.965331Z","shell.execute_reply.started":"2021-11-24T10:27:44.277907Z","shell.execute_reply":"2021-11-24T10:27:50.964649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load the credentials from Kaggle Secrets.\n# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n\n# # Activate the service account stored in the Kaggle Secrets to access Cloud Storage\n# with open('/tmp/service_account.json', 'w') as f:\n#     f.write(user_secrets.get_secret('service_account_json'))\n# !gcloud auth activate-service-account --key-file=/tmp/service_account.json --no-user-output-enabled","metadata":{"execution":{"iopub.status.busy":"2021-11-24T07:43:55.313671Z","iopub.execute_input":"2021-11-24T07:43:55.313916Z","iopub.status.idle":"2021-11-24T07:43:58.546049Z","shell.execute_reply.started":"2021-11-24T07:43:55.313886Z","shell.execute_reply":"2021-11-24T07:43:58.544763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a folder on Google Cloud Storage to store the checkpoints.\nimport pytz\nfrom datetime import datetime\nJST = pytz.timezone('Asia/Tokyo')\nutc_dt = datetime.now()\nFOLDER_NAME = utc_dt.astimezone(JST).strftime(\"%Y%m%d-%H%M\")\nBUCKET_NAME = user_secrets.get_secret('gcs_bucket_name')\nGCS_OUTPUT_PATH = f\"gs://{BUCKET_NAME}/kaggle/{FOLDER_NAME}\"\nprint(GCS_OUTPUT_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T08:37:45.063897Z","iopub.execute_input":"2021-11-24T08:37:45.064484Z","iopub.status.idle":"2021-11-24T08:37:45.26257Z","shell.execute_reply.started":"2021-11-24T08:37:45.064428Z","shell.execute_reply":"2021-11-24T08:37:45.26136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the training dataset","metadata":{"papermill":{"duration":0.04958,"end_time":"2021-11-19T08:39:54.895616","exception":false,"start_time":"2021-11-19T08:39:54.846036","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"See this [notebook](https://www.kaggle.com/khanhlvg/tensorflow-prepare-cots-dataset-for-tpu/) to learn how to convert the training images to the TFRecord format required by TensorFlow Object Detection API. We'll use the output of the conversion notebook here and start training a model.","metadata":{"papermill":{"duration":0.788121,"end_time":"2021-11-19T08:40:46.513739","exception":false,"start_time":"2021-11-19T08:40:45.725618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load the path to the preprocessed dataset.\nfrom kaggle_datasets import KaggleDatasets\nGCS_TFRECORD_BUCKET_PATH = KaggleDatasets().get_gcs_path('crown-of-thorns-starfish-dataset-in-tfrecord')\n# GCS_TFRECORD_BUCKET_PATH='gs://cots_data_public'","metadata":{"execution":{"iopub.status.busy":"2021-11-23T15:29:06.003321Z","iopub.execute_input":"2021-11-23T15:29:06.003636Z","iopub.status.idle":"2021-11-23T15:29:25.115273Z","shell.execute_reply.started":"2021-11-23T15:29:06.003601Z","shell.execute_reply":"2021-11-23T15:29:25.114225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train an object detection model\n\nWe'll use [TensorFlow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) and an EfficientDet-D0 base model and apply transfer learning to train a COTS detection model. EfficientDet-D0 is the smallest model in the EfficientDet model family and we pick it to reduce training time for demonstration purpose. You can probably increase accuracy by switch to using a larger EfficientDet model.","metadata":{"papermill":{"duration":3.355764,"end_time":"2021-11-19T08:41:50.909779","exception":false,"start_time":"2021-11-19T08:41:47.554015","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from string import Template\n\nconfig_file_template = \"\"\"\nmodel {\n  ssd {\n    num_classes: 1\n    image_resizer {\n      keep_aspect_ratio_resizer {\n        min_dimension: 1280\n        max_dimension: 1280\n        pad_to_max_dimension: true\n      }\n    }\n    feature_extractor {\n      type: \"ssd_efficientnet-b4_bifpn_keras\"\n      conv_hyperparams {\n        regularizer {\n          l2_regularizer {\n            weight: 3.9999998989515007e-05\n          }\n        }\n        initializer {\n          truncated_normal_initializer {\n            mean: 0.0\n            stddev: 0.029999999329447746\n          }\n        }\n        activation: SWISH\n        batch_norm {\n          decay: 0.9900000095367432\n          scale: true\n          epsilon: 0.0010000000474974513\n        }\n        force_use_bias: true\n      }\n      bifpn {\n        min_level: 3\n        max_level: 7\n        num_iterations: 7\n        num_filters: 224\n      }\n    }\n    box_coder {\n      faster_rcnn_box_coder {\n        y_scale: 1.0\n        x_scale: 1.0\n        height_scale: 1.0\n        width_scale: 1.0\n      }\n    }\n    matcher {\n      argmax_matcher {\n        matched_threshold: 0.5\n        unmatched_threshold: 0.5\n        ignore_thresholds: false\n        negatives_lower_than_unmatched: true\n        force_match_for_each_row: true\n        use_matmul_gather: true\n      }\n    }\n    similarity_calculator {\n      iou_similarity {\n      }\n    }\n    box_predictor {\n      weight_shared_convolutional_box_predictor {\n        conv_hyperparams {\n          regularizer {\n            l2_regularizer {\n              weight: 3.9999998989515007e-05\n            }\n          }\n          initializer {\n            random_normal_initializer {\n              mean: 0.0\n              stddev: 0.009999999776482582\n            }\n          }\n          activation: SWISH\n          batch_norm {\n            decay: 0.9900000095367432\n            scale: true\n            epsilon: 0.0010000000474974513\n          }\n          force_use_bias: true\n        }\n        depth: 224\n        num_layers_before_predictor: 4\n        kernel_size: 3\n        class_prediction_bias_init: -4.599999904632568\n        use_depthwise: true\n      }\n    }\n    anchor_generator {\n      multiscale_anchor_generator {\n        min_level: 3\n        max_level: 7\n        anchor_scale: 4.0\n        aspect_ratios: 1.0\n        aspect_ratios: 2.0\n        aspect_ratios: 0.5\n        scales_per_octave: 3\n      }\n    }\n    post_processing {\n      batch_non_max_suppression {\n        score_threshold: 9.99999993922529e-09\n        iou_threshold: 0.5\n        max_detections_per_class: 100\n        max_total_detections: 100\n      }\n      score_converter: SIGMOID\n    }\n    normalize_loss_by_num_matches: true\n    loss {\n      localization_loss {\n        weighted_smooth_l1 {\n        }\n      }\n      classification_loss {\n        weighted_sigmoid_focal {\n          gamma: 1.5\n          alpha: 0.25\n        }\n      }\n      classification_weight: 1.0\n      localization_weight: 1.0\n    }\n    encode_background_as_zeros: true\n    normalize_loc_loss_by_codesize: true\n    inplace_batchnorm_update: true\n    freeze_batchnorm: false\n    add_background_class: false\n  }\n}\ntrain_config {\n  batch_size: $batch_size\n  data_augmentation_options {\n    random_horizontal_flip {\n    }\n  }\n  data_augmentation_options {\n    random_scale_crop_and_pad_to_square {\n      output_size: 1280\n      scale_min: 0.5\n      scale_max: 2.0\n    }\n  }\n  sync_replicas: true\n  optimizer {\n    momentum_optimizer {\n      learning_rate {\n        cosine_decay_learning_rate {\n          learning_rate_base: 0.007999999821186066\n          total_steps: $training_steps\n          warmup_learning_rate: 0.00050000000474974513\n          warmup_steps: $warmup_steps\n        }\n      }\n      momentum_optimizer_value: 0.8999999761581421\n    }\n    use_moving_average: false\n  }\n  fine_tune_checkpoint: \"$fine_tune_checkpoint\"\n  num_steps: $training_steps\n  startup_delay_steps: 0.0\n  replicas_to_aggregate: 8\n  max_number_of_boxes: 100\n  unpad_groundtruth_tensors: false\n  fine_tune_checkpoint_type: \"detection\"\n  use_bfloat16: true\n  fine_tune_checkpoint_version: V2\n}\ntrain_input_reader: {\n  label_map_path: \"$label_map_path\"\n  tf_record_input_reader {\n    input_path: \"$train_input_path\"\n  }\n}\n\neval_config: {\n  metrics_set: \"coco_detection_metrics\"\n  use_moving_averages: false\n  batch_size: 1;\n}\n\neval_input_reader: {\n  label_map_path: \"$label_map_path\"\n  shuffle: false\n  num_epochs: 1\n  tf_record_input_reader {\n    input_path: \"$val_input_path\"\n  }\n}\n\"\"\"","metadata":{"papermill":{"duration":0.133468,"end_time":"2021-11-19T08:41:54.609774","exception":false,"start_time":"2021-11-19T08:41:54.476306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T08:37:53.659002Z","iopub.execute_input":"2021-11-24T08:37:53.659338Z","iopub.status.idle":"2021-11-24T08:37:53.669323Z","shell.execute_reply.started":"2021-11-24T08:37:53.659303Z","shell.execute_reply":"2021-11-24T08:37:53.668366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the training pipeline\n\nBATCH_SIZE=16\nTRAINING_STEPS = 5000\nWARMUP_STEPS = 500\nPIPELINE_CONFIG_PATH='pipeline.config'\n\nGCS_TFRECORD_TRAIN_PATH = GCS_TFRECORD_BUCKET_PATH + '/tfrecord/cots_train-?????-of-00008'\nGCS_TFRECORD_VAL_PATH = GCS_TFRECORD_BUCKET_PATH + '/tfrecord/cots_val-?????-of-00008'\nGCS_PRETRAINED_CHECKPOINT_PATH = GCS_TFRECORD_BUCKET_PATH + '/efficientdet_d4_coco17_tpu-32/checkpoint/ckpt-0'\nGCS_LABEL_MAP_PATH = GCS_TFRECORD_BUCKET_PATH + '/label_map.pbtxt'\n\npipeline = Template(config_file_template).substitute(\n    batch_size=BATCH_SIZE,\n    training_steps=TRAINING_STEPS, \n    warmup_steps=WARMUP_STEPS,\n    label_map_path=GCS_LABEL_MAP_PATH,\n    train_input_path=GCS_TFRECORD_TRAIN_PATH,\n    val_input_path=GCS_TFRECORD_VAL_PATH,\n    fine_tune_checkpoint=GCS_PRETRAINED_CHECKPOINT_PATH\n)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)\n\nMODEL_DIR=GCS_OUTPUT_PATH + '/cots_efficientdet_d4'","metadata":{"papermill":{"duration":0.121946,"end_time":"2021-11-19T08:41:54.846958","exception":false,"start_time":"2021-11-19T08:41:54.725012","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T08:37:53.979585Z","iopub.execute_input":"2021-11-24T08:37:53.979875Z","iopub.status.idle":"2021-11-24T08:37:53.987946Z","shell.execute_reply.started":"2021-11-24T08:37:53.979846Z","shell.execute_reply":"2021-11-24T08:37:53.987336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --use_tpu=true \\\n    --alsologtostderr","metadata":{"papermill":{"duration":18463.625406,"end_time":"2021-11-19T13:49:38.586506","exception":false,"start_time":"2021-11-19T08:41:54.9611","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-23T15:59:11.628883Z","iopub.execute_input":"2021-11-23T15:59:11.62929Z","iopub.status.idle":"2021-11-23T16:24:08.934985Z","shell.execute_reply.started":"2021-11-23T15:59:11.629265Z","shell.execute_reply":"2021-11-23T16:24:08.934199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the object detection model","metadata":{"papermill":{"duration":0.226793,"end_time":"2021-11-19T13:49:39.043281","exception":false,"start_time":"2021-11-19T13:49:38.816488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --checkpoint_dir={MODEL_DIR} \\\n    --eval_timeout=0 \\\n    --alsologtostderr","metadata":{"papermill":{"duration":323.73381,"end_time":"2021-11-19T13:55:03.004285","exception":false,"start_time":"2021-11-19T13:49:39.270475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T08:37:57.255478Z","iopub.execute_input":"2021-11-24T08:37:57.255918Z","iopub.status.idle":"2021-11-24T08:54:12.354147Z","shell.execute_reply.started":"2021-11-24T08:37:57.255888Z","shell.execute_reply":"2021-11-24T08:54:12.341232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export as SavedModel for inference","metadata":{"papermill":{"duration":0.239823,"end_time":"2021-11-19T13:55:03.483464","exception":false,"start_time":"2021-11-19T13:55:03.243641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!python models/research/object_detection/exporter_main_v2.py \\\n    --input_type image_tensor \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --trained_checkpoint_dir={MODEL_DIR} \\\n    --output_directory=output","metadata":{"papermill":{"duration":122.093553,"end_time":"2021-11-19T13:57:05.815158","exception":false,"start_time":"2021-11-19T13:55:03.721605","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T07:56:20.205632Z","iopub.execute_input":"2021-11-24T07:56:20.206418Z","iopub.status.idle":"2021-11-24T08:09:39.712028Z","shell.execute_reply.started":"2021-11-24T07:56:20.206364Z","shell.execute_reply":"2021-11-24T08:09:39.710823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls output","metadata":{"papermill":{"duration":0.931074,"end_time":"2021-11-19T13:57:06.998967","exception":false,"start_time":"2021-11-19T13:57:06.067893","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-24T08:09:39.713938Z","iopub.execute_input":"2021-11-24T08:09:39.714261Z","iopub.status.idle":"2021-11-24T08:09:40.4852Z","shell.execute_reply.started":"2021-11-24T08:09:39.714223Z","shell.execute_reply":"2021-11-24T08:09:40.484146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean up","metadata":{"papermill":{"duration":0.24428,"end_time":"2021-11-19T13:57:07.485159","exception":false,"start_time":"2021-11-19T13:57:07.240879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Remove data downloaded during training.\n!rm -rf models","metadata":{"papermill":{"duration":4.604917,"end_time":"2021-11-19T13:57:47.580583","exception":false,"start_time":"2021-11-19T13:57:42.975666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-23T14:36:24.27105Z","iopub.status.idle":"2021-11-23T14:36:24.271475Z","shell.execute_reply.started":"2021-11-23T14:36:24.271293Z","shell.execute_reply":"2021-11-23T14:36:24.271308Z"},"trusted":true},"execution_count":null,"outputs":[]}]}