{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This notebook contains code to convert the positive training images into the TFRecord format that is required by the TensorFlow Object Detection API. You can use the output of this notebook and start training a model from there.\n\n# Install TensorFlow Object Detection API\n\nPip may report some dependency errors. You can safely ignore these errors and proceed if all tests in `model_builder_tf2_test.py` passed. ","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models && git checkout ac8d06519","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:26:38.043783Z","iopub.execute_input":"2021-11-23T13:26:38.044612Z","iopub.status.idle":"2021-11-23T13:26:59.131129Z","shell.execute_reply.started":"2021-11-23T13:26:38.044463Z","shell.execute_reply":"2021-11-23T13:26:59.130266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd models/research\n\n# Compile protos.\nprotoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow and keras to 2.7.0 to be compatible with the current TF ODT API snapshot\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\nwget -O setup.py https://storage.googleapis.com/odml-dataset/others/setup_tf27.py\npip install -q --user .\n\n# Test if the Object Dectection API is working correctly\npython object_detection/builders/model_builder_tf2_test.py","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-23T13:26:59.132882Z","iopub.execute_input":"2021-11-23T13:26:59.133104Z","iopub.status.idle":"2021-11-23T13:28:55.436229Z","shell.execute_reply.started":"2021-11-23T13:26:59.133077Z","shell.execute_reply":"2021-11-23T13:28:55.434752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport pandas as pd\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:28:56.362122Z","iopub.execute_input":"2021-11-23T13:28:56.362430Z","iopub.status.idle":"2021-11-23T13:28:59.142614Z","shell.execute_reply.started":"2021-11-23T13:28:56.362365Z","shell.execute_reply":"2021-11-23T13:28:59.141708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the training dataset\n\nSplit the `train` folder into training dataset and validation dataset. ","metadata":{}},{"cell_type":"code","source":"TRAINING_RATIO = 0.8\n\ndata_df = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n\n# Split the dataset so that no sequence is leaked from the training dataset into the validation dataset.\nsplit_index = int(TRAINING_RATIO * len(data_df))\nwhile data_df.iloc[split_index - 1].sequence == data_df.iloc[split_index].sequence:\n    split_index += 1\n\n# Shuffle both the training and validation datasets.\ntrain_data_df = data_df.iloc[:split_index].sample(frac=1).reset_index(drop=True)\nval_data_df = data_df.iloc[split_index:].sample(frac=1).reset_index(drop=True)\n\ntrain_positive_count = len(train_data_df[train_data_df.annotations != '[]'])\nval_positive_count = len(val_data_df[val_data_df.annotations != '[]'])\n\nprint('Training ratio (all samples):', \n      float(len(train_data_df)) / (len(train_data_df) + len(val_data_df)))\nprint('Training ratio (positive samples):', \n      float(train_positive_count) / (train_positive_count + val_positive_count))","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:28:59.144250Z","iopub.execute_input":"2021-11-23T13:28:59.144540Z","iopub.status.idle":"2021-11-23T13:28:59.315229Z","shell.execute_reply.started":"2021-11-23T13:28:59.144503Z","shell.execute_reply":"2021-11-23T13:28:59.314331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To save time for demonstration purpose, here we'll only take the positive images (images that contain at least 1 starfish) for training. TensorFlow Object Detection API will take the areas in the images that aren't annotated as containing a starfish to use as negative samples.","metadata":{}},{"cell_type":"code","source":"# Take only the positive images for training and validation\ntrain_data_df = train_data_df[train_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for training:', len(train_data_df))\nval_data_df = val_data_df[val_data_df.annotations != '[]'].reset_index()\nprint('Number of positive images used for validation:', len(val_data_df))","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:29:37.827190Z","iopub.execute_input":"2021-11-23T13:29:37.827844Z","iopub.status.idle":"2021-11-23T13:29:37.849402Z","shell.execute_reply.started":"2021-11-23T13:29:37.827804Z","shell.execute_reply":"2021-11-23T13:29:37.848489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_with_annotation(video_id, video_frame, data_df, image_path):\n    \"\"\"Visualize annotations of a given image.\"\"\"\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    image = Image.open(full_path)\n    draw = ImageDraw.Draw(image)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            draw.rectangle((\n                annotation['x'], \n                annotation['y'],\n                (annotation['x'] + annotation['width']), \n                (annotation['y'] + annotation['height']),\n                ), outline=(255, 255, 0))\n        \n    buf = io.BytesIO()\n    image.save(buf, 'PNG')\n    data = buf.getvalue()\n\n    return data\n\n# Test visualization of a randomly selected image\nimage_path = os.path.join(INPUT_DIR, 'train_images')\ntest_index = 1\nvideo_id = train_data_df.iloc[test_index].video_id\nvideo_frame = train_data_df.iloc[test_index].video_frame\nIPython.display.Image(image_with_annotation(video_id, video_frame, data_df, image_path))","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:29:55.120809Z","iopub.execute_input":"2021-11-23T13:29:55.121084Z","iopub.status.idle":"2021-11-23T13:29:55.660193Z","shell.execute_reply.started":"2021-11-23T13:29:55.121055Z","shell.execute_reply":"2021-11-23T13:29:55.658662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from object_detection.utils import dataset_util\nfrom object_detection.dataset_tools import tf_record_creation_util\n\n\ndef create_tf_example(video_id, video_frame, data_df, image_path):\n    \"\"\"Create a tf.Example entry for a given training image.\"\"\"\n    full_path = os.path.join(image_path, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    with tf.io.gfile.GFile(full_path, 'rb') as fid:\n        encoded_jpg = fid.read()\n    encoded_jpg_io = io.BytesIO(encoded_jpg)\n    image = Image.open(encoded_jpg_io)\n    if image.format != 'JPEG':\n        raise ValueError('Image format not JPEG')\n\n    height = image.size[1] # Image height\n    width = image.size[0] # Image width\n    filename = f'{video_id}:{video_frame}'.encode('utf8') # Unique id of the image.\n    encoded_image_data = None # Encoded image bytes\n    image_format = 'jpeg'.encode('utf8') # b'jpeg' or b'png'\n\n    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)\n    xmaxs = [] # List of normalized right x coordinates in bounding box\n             # (1 per box)\n    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)\n    ymaxs = [] # List of normalized bottom y coordinates in bounding box\n             # (1 per box)\n    classes_text = [] # List of string class name of bounding box (1 per box)\n    classes = [] # List of integer class id of bounding box (1 per box)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            xmins.append(annotation['x'] / width) \n            xmaxs.append((annotation['x'] + annotation['width']) / width) \n            ymins.append(annotation['y'] / height) \n            ymaxs.append((annotation['y'] + annotation['height']) / height) \n\n            classes_text.append('COTS'.encode('utf8'))\n            classes.append(1)\n\n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n      'image/height': dataset_util.int64_feature(height),\n      'image/width': dataset_util.int64_feature(width),\n      'image/filename': dataset_util.bytes_feature(filename),\n      'image/source_id': dataset_util.bytes_feature(filename),\n      'image/encoded': dataset_util.bytes_feature(encoded_jpg),\n      'image/format': dataset_util.bytes_feature(image_format),\n      'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n      'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n      'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n      'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n      'image/object/class/label': dataset_util.int64_list_feature(classes),\n    }))\n    \n    return tf_example\n\ndef convert_to_tfrecord(data_df, tfrecord_filebase, image_path, num_shards = 10):\n    \"\"\"Convert the object detection dataset to TFRecord as required by the TF ODT API.\"\"\"\n    with contextlib2.ExitStack() as tf_record_close_stack:\n        output_tfrecords = tf_record_creation_util.open_sharded_output_tfrecords(\n            tf_record_close_stack, tfrecord_filebase, num_shards)\n        \n        for index, row in data_df.iterrows():\n            if index % 500 == 0:\n                print('Processed {0} images.'.format(index))\n            tf_example = create_tf_example(row.video_id, row.video_frame, data_df, image_path)\n            output_shard_index = index % num_shards\n            output_tfrecords[output_shard_index].write(tf_example.SerializeToString())\n        \n        print('Completed processing {0} images.'.format(len(data_df)))\n\nimage_path = os.path.join(INPUT_DIR, 'train_images')\n!mkdir tfrecord\n\n# Convert train images to TFRecord\nprint('Converting TRAIN images...')\nconvert_to_tfrecord(\n  train_data_df,\n  'tfrecord/cots_train',\n  image_path,\n  num_shards = 8\n)\n\n# Convert validation images to TFRecord\nprint('Converting VALIDATION images...')\nconvert_to_tfrecord(\n  val_data_df,\n  'tfrecord/cots_val',\n  image_path,\n  num_shards = 8\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:31:05.235735Z","iopub.execute_input":"2021-11-23T13:31:05.236051Z","iopub.status.idle":"2021-11-23T13:32:16.105850Z","shell.execute_reply.started":"2021-11-23T13:31:05.236014Z","shell.execute_reply":"2021-11-23T13:32:16.105059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a label map to map between label index and human-readable label name.\n\nlabel_map_str = \"\"\"item {\n  id: 1\n  name: 'COTS'\n}\"\"\"\n\nwith open('label_map.pbtxt', 'w') as f:\n    f.write(label_map_str)\n\n!more label_map.pbtxt","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download EfficientDet pretrained models\n\nHere I use an EfficientDet-D4 model but you can try using larger model to improve accuracy.","metadata":{}},{"cell_type":"code","source":"# Download the pretrained EfficientDet-D4 checkpoint\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d4_coco17_tpu-32.tar.gz\n!tar -xvzf efficientdet_d4_coco17_tpu-32.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:49:33.443343Z","iopub.execute_input":"2021-11-23T13:49:33.443830Z","iopub.status.idle":"2021-11-23T13:49:39.297514Z","shell.execute_reply.started":"2021-11-23T13:49:33.443661Z","shell.execute_reply":"2021-11-23T13:49:39.296598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean up","metadata":{}},{"cell_type":"code","source":"!rm *.tar.gz\n!rm -rf models","metadata":{"execution":{"iopub.status.busy":"2021-11-23T13:49:52.519649Z","iopub.execute_input":"2021-11-23T13:49:52.519976Z","iopub.status.idle":"2021-11-23T13:49:54.048283Z","shell.execute_reply.started":"2021-11-23T13:49:52.519929Z","shell.execute_reply":"2021-11-23T13:49:54.047083Z"},"trusted":true},"execution_count":null,"outputs":[]}]}