{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n[Check out the inference notebook here](https://www.kaggle.com/ravishah1/cots-faster-rcnn-inference-tf-2-0-od-api-0-474)\n\n[For details on how it works, see my explanation of the config file](https://www.kaggle.com/c/tensorflow-great-barrier-reef/discussion/293305)\n\n[I also took inspiration from this notebook by Khanh](https://www.kaggle.com/khanhlvg/cots-detection-w-tensorflow-object-detection-api)\n\n### General Info\n\nThis notebook is the training for a faster-rcnn using the TensorFlow Object Detection API. It uses TensorFlow 2.0 API making it eligable for the TensorFlow Performance Prize. It scores a 0.474\n\n### Cross Validation\n\n- I have written a cross validation pipeline file in this notebook\n- I am using a group k-fold on sequence (which currently performs best)\n\n### Dataset\n\n- I have written a dataset generation file in this notebook\n- I create the tfrecords dataset with 4 shards\n\n### The Model\n\n- Faster-RCNN\n- feature extractor: Resnet-101 \n\n### Augmentations\n\n- image resizer: pad min and max dimension to 1280\n- random_horizontal_flip\n- random_scale_crop_and_pad_to_square\n\n### Optimizer\n\n- momentum optimizer \n- cosine_decay_learning_rate\n- training steps = 20000\n- warmup steps = 2000","metadata":{}},{"cell_type":"code","source":"import contextlib2\nimport io\nimport IPython\nimport json\nimport numpy as np\nimport os\nimport pathlib\nimport pandas as pd\nimport sys\nimport tensorflow as tf\nimport time\n\nfrom PIL import Image, ImageDraw\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'\nsys.path.insert(0, INPUT_DIR)\nimport greatbarrierreef","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/tensorflow/models\n    \n# Check out a certain commit to ensure that future changes in the TF ODT API codebase won't affect this notebook.\n!cd models && git checkout ac8d06519","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd models/research\n\n# Compile protos.\nprotoc object_detection/protos/*.proto --python_out=.\n\n# Install TensorFlow Object Detection API.\n# Note: I fixed the version of some dependencies to make it work on Kaggle notebook. In particular:\n# * scipy==1.6.3 to avoid the missing GLIBCXX_3.4.26 error\n# * tensorflow to 2.6.0 to make it compatible with the CUDA version preinstalled on Kaggle.\n# When Kaggle notebook upgrade to TF 2.7, you can use the default setup.py script:\n# cp object_detection/packages/tf2/setup.py .\nwget https://storage.googleapis.com/odml-dataset/others/setup.py\npip install -q --user .\n\n# Test if the Object Dectection API is working correctly\npython object_detection/builders/model_builder_tf2_test.py","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The notebook is supposed to run with TF 2.6.0\nprint(tf.__version__)\nprint(tf.test.is_gpu_available())\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"execution":{"iopub.status.busy":"2021-12-23T02:59:43.898326Z","iopub.execute_input":"2021-12-23T02:59:43.898536Z","iopub.status.idle":"2021-12-23T02:59:43.928759Z","shell.execute_reply.started":"2021-12-23T02:59:43.898507Z","shell.execute_reply":"2021-12-23T02:59:43.927903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"%%writefile cross_validation_setup.py\n\"\"\"\nSplits a dataframe to a specified cross_validation framework\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport argparse\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\n\nparser = argparse.ArgumentParser(\n    description=\"Cross Validation Setup\")\nparser.add_argument(\"-in\",\n                    \"--input_dir\",\n                    help=\"The filepath to the input csv.\", type=str)\nparser.add_argument(\"-out\",\n                    \"--output_dir\",\n                    help=\"The filepath to the output csv.\", type=str)\nparser.add_argument(\"-type\",\n                    \"--type\",\n                    help=\"The type of cross_validation to use.\", type=str)\nparser.add_argument(\"-on\",\n                    \"--on\",\n                    help=\"The column to stratisfy on.\", type=str)\nparser.add_argument(\"-folds\",\n                    \"--folds\",\n                    help=\"The number of folds to create.\", type=int)\nparser.add_argument(\"-hold\",\n                    \"--holdout\",\n                    help=\"The fold to holdout.\", type=int)\n\nargs = parser.parse_args()\n\ndef cross_validation():\n    df = pd.read_csv(args.input_dir)\n    df = df[df.annotations!='[]']\n    df = df.reset_index(drop=True)\n    \n    if args.type=='kfold':\n        kf = KFold(n_splits=args.folds, shuffle=True, random_state=42)\n        for f, (t_, v_) in enumerate(kf.split(X=df)):\n            df.loc[v_, 'fold'] = f\n            \n    elif args.type=='skfold' and args.on=='video_id':\n        # uses skf to get even distrubution of data from different videos\n        kf = StratifiedKFold(n_splits=args.folds, shuffle=True, random_state=42)\n        for f, (t_, v_) in enumerate(kf.split(X=df, y=df.video_id)): \n            df.loc[v_, 'fold'] = f\n            \n    elif args.type=='gkfold' and args.on=='sequence':\n        kf = GroupKFold(n_splits=args.folds)\n        for f, (t_, v_) in enumerate(kf.split(X=df, y=df.video_id, groups=df.sequence)): \n            df.loc[v_, 'fold'] = f\n            \n    else:\n        raise Exception('Not Implemented')\n        \n    df.to_csv(args.output_dir, index=False)\n    \nif __name__ == '__main__':\n    cross_validation()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile generate_tfrecords.py\n\nimport os\nfrom os.path import exists\nimport glob\nimport pandas as pd\nimport io\nimport json\nimport xml.etree.ElementTree as ET\nimport contextlib2\nimport argparse\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'    # Suppress TensorFlow logging (1)\nimport tensorflow.compat.v1 as tf\nfrom PIL import Image\nfrom object_detection.utils import dataset_util, label_map_util\nfrom object_detection.dataset_tools import tf_record_creation_util\nfrom collections import namedtuple\n\n# Initiate argument parser\nparser = argparse.ArgumentParser(\n    description=\"TensorFlow TFRecord Generator\")\nparser.add_argument(\"-c\",\n                    \"--csv_path\",\n                    help=\"Path to the train.csv file.\", type=str)\nparser.add_argument(\"-o\",\n                    \"--output_path\",\n                    help=\"Path of output TFRecord (.record) file.\", type=str)\nparser.add_argument(\"-i\",\n                    \"--image_dir\",\n                    help=\"Path to the folder where the input image files are stored.\", type=str)\nparser.add_argument(\"-t\",\n                    \"--train\",\n                    help=\"True if this is a training dataset, false if it is a validation dataset.\", type=str)\nparser.add_argument(\"-s\",\n                    \"--shards\",\n                    help=\"The number of shards for the dataset\", type=int)\nparser.add_argument(\"-f\",\n                    \"--holdout_fold\",\n                    help=\"The fold to holdout.\", type=int)\n\nargs = parser.parse_args()\n\ndef create_tf_example(data_df: pd.DataFrame, video_id: int, video_frame: int):\n    \"\"\"\n    Create a tf.Example entry for a given training image.\n    \"\"\"\n    full_path = os.path.join(args.image_dir, os.path.join(f'video_{video_id}', f'{video_frame}.jpg'))\n    with tf.io.gfile.GFile(full_path, 'rb') as fid:\n        encoded_jpg = fid.read()\n    encoded_jpg_io = io.BytesIO(encoded_jpg)\n    image = Image.open(encoded_jpg_io)\n    if image.format != 'JPEG':\n        raise ValueError('Image format not JPEG')\n\n    height = image.size[1] # Image height\n    width = image.size[0] # Image width\n    filename = f'{video_id}:{video_frame}'.encode('utf8') # Unique id of the image.\n    encoded_image_data = None # Encoded image bytes\n    image_format = 'jpeg'.encode('utf8') # b'jpeg' or b'png'\n\n    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)\n    xmaxs = [] # List of normalized right x coordinates in bounding box\n             # (1 per box)\n    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)\n    ymaxs = [] # List of normalized bottom y coordinates in bounding box\n             # (1 per box)\n    classes_text = [] # List of string class name of bounding box (1 per box)\n    classes = [] # List of integer class id of bounding box (1 per box)\n\n    rows = data_df[(data_df.video_id == video_id) & (data_df.video_frame == video_frame)]\n    for _, row in rows.iterrows():\n        annotations = json.loads(row.annotations.replace(\"'\", '\"'))\n        for annotation in annotations:\n            xmins.append(annotation['x'] / width) \n            xmaxs.append((annotation['x'] + annotation['width']) / width) \n            ymins.append(annotation['y'] / height) \n            ymaxs.append((annotation['y'] + annotation['height']) / height) \n\n            classes_text.append('COTS'.encode('utf8'))\n            classes.append(1)\n\n    tf_example = tf.train.Example(features=tf.train.Features(feature={\n      'image/height': dataset_util.int64_feature(height),\n      'image/width': dataset_util.int64_feature(width),\n      'image/filename': dataset_util.bytes_feature(filename),\n      'image/source_id': dataset_util.bytes_feature(filename),\n      'image/encoded': dataset_util.bytes_feature(encoded_jpg),\n      'image/format': dataset_util.bytes_feature(image_format),\n      'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),\n      'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),\n      'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),\n      'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),\n      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),\n      'image/object/class/label': dataset_util.int64_list_feature(classes),\n    }))\n\n    return tf_example\n\ndef create_labels_file():\n    label_map_str = \"\"\"\n    item {\n        id: 1\n        name: 'COTS'\n        }\n                    \"\"\"\n\n    if exists('dataset/label_map.pbtxt') is False:\n        with open('dataset/label_map.pbtxt', 'w') as f:\n            f.write(label_map_str)\n        print('Successfully created label_map.pbtxt file')\n\nif __name__ == '__main__':\n\n    # label file\n    create_labels_file()\n    #writer = tf.python_io.TFRecordWriter(args.output_path)\n    \n    # setup df\n    data_df = pd.read_csv(args.csv_path)\n    if args.train =='train':\n        data_df = data_df[data_df.fold != args.holdout_fold].reset_index(drop=True)\n    else:\n        data_df = data_df[data_df.fold == args.holdout_fold].reset_index(drop=True)\n    \n    # make records\n    with contextlib2.ExitStack() as tf_record_close_stack:\n        output_tfrecords = tf_record_creation_util.open_sharded_output_tfrecords(\n            tf_record_close_stack, args.output_path, args.shards)\n        \n        for index, row in data_df.iterrows():\n            tf_example = create_tf_example(data_df, row.video_id, row.video_frame)\n            output_shard_index = index % args.shards\n            output_tfrecords[output_shard_index].write(tf_example.SerializeToString())\n    #writer.close()\n    print('Successfully created the TFRecord file: {}'.format(args.output_path))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\npython cross_validation_setup.py \\\n    -in ../input/tensorflow-great-barrier-reef/train.csv \\\n    -out train.csv \\\n    -type gkfold \\\n    -on sequence \\\n    -folds 10 \\\n    -hold 0\n\nmkdir dataset\n\npython generate_tfrecords.py \\\n    -c train.csv \\\n    -o dataset/cots_train \\\n    -i ../input/tensorflow-great-barrier-reef/train_images \\\n    -t train \\\n    -s 4 \\\n    -f 0\n\npython generate_tfrecords.py \\\n    -c train.csv \\\n    -o dataset/cots_val \\\n    -i ../input/tensorflow-great-barrier-reef/train_images \\\n    -t valid \\\n    -s 4 \\\n    -f 0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prep Train","metadata":{}},{"cell_type":"code","source":"# Download the pretrained EfficientDet-D0 checkpoint\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz\n!tar -xvzf faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from string import Template\n\nconfig_file_template = \"\"\"\n# Faster R-CNN with Resnet-50 (v1)\n# Trained on COCO, initialized from Imagenet classification checkpoint\n#\n# Train on TPU-8\n#\n# Achieves 31.8 mAP on COCO17 val\n\nmodel {\n  faster_rcnn {\n    num_classes: 1\n    image_resizer {\n      keep_aspect_ratio_resizer {\n        min_dimension: 1280\n        max_dimension: 1280\n        pad_to_max_dimension: true\n      }\n    }\n    feature_extractor {\n      type: 'faster_rcnn_resnet101_keras'\n      batch_norm_trainable: true\n    }\n    first_stage_anchor_generator {\n      grid_anchor_generator {\n        scales: [0.25, 0.5, 1.0, 2.0]\n        aspect_ratios: [0.5, 1.0, 2.0]\n        height_stride: 16\n        width_stride: 16\n      }\n    }\n    first_stage_box_predictor_conv_hyperparams {\n      op: CONV\n      regularizer {\n        l2_regularizer {\n          weight: 0.0\n        }\n      }\n      initializer {\n        truncated_normal_initializer {\n          stddev: 0.01\n        }\n      }\n    }\n    first_stage_nms_score_threshold: 0.0\n    first_stage_nms_iou_threshold: 0.7\n    first_stage_max_proposals: 300\n    first_stage_localization_loss_weight: 2.0\n    first_stage_objectness_loss_weight: 1.0\n    initial_crop_size: 14\n    maxpool_kernel_size: 2\n    maxpool_stride: 2\n    second_stage_box_predictor {\n      mask_rcnn_box_predictor {\n        use_dropout: false\n        dropout_keep_probability: 1.0\n        fc_hyperparams {\n          op: FC\n          regularizer {\n            l2_regularizer {\n              weight: 0.0\n            }\n          }\n          initializer {\n            variance_scaling_initializer {\n              factor: 1.0\n              uniform: true\n              mode: FAN_AVG\n            }\n          }\n        }\n        share_box_across_classes: true\n      }\n    }\n    second_stage_post_processing {\n      batch_non_max_suppression {\n        score_threshold: 0.0\n        iou_threshold: 0.6\n        max_detections_per_class: 100\n        max_total_detections: 300\n      }\n      score_converter: SOFTMAX\n    }\n    second_stage_localization_loss_weight: 2.0\n    second_stage_classification_loss_weight: 1.0\n    use_static_shapes: true\n    use_matmul_crop_and_resize: true\n    clip_anchors_to_image: true\n    use_static_balanced_label_sampler: true\n    use_matmul_gather_in_matcher: true\n  }\n}\n\ntrain_config: {\n  fine_tune_checkpoint: \"faster_rcnn_resnet101_v1_640x640_coco17_tpu-8/checkpoint/ckpt-0\"\n  fine_tune_checkpoint_version: V2\n  fine_tune_checkpoint_type: \"detection\"\n  batch_size: 2\n  sync_replicas: false\n  startup_delay_steps: 0\n  replicas_to_aggregate: 1\n  use_bfloat16: false\n  num_steps: $training_steps\n  data_augmentation_options {\n    random_horizontal_flip {\n    }\n  }\n  data_augmentation_options {\n    random_scale_crop_and_pad_to_square {\n      output_size: 1280\n      scale_min: 0.5\n      scale_max: 2.0\n    }\n  }\n  optimizer {\n    momentum_optimizer: {\n      learning_rate: {\n        cosine_decay_learning_rate {\n          learning_rate_base: 5e-3\n          total_steps: $training_steps\n          warmup_learning_rate: 5e-4\n          warmup_steps: $warmup_steps\n        }\n      }\n      momentum_optimizer_value: 0.9\n    }\n    use_moving_average: false\n  }\n  max_number_of_boxes: 100\n  unpad_groundtruth_tensors: false\n}\n\ntrain_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  tf_record_input_reader {\n    input_path: \"dataset/cots_train-?????-of-00004\"\n  }\n}\n\neval_config: {\n  metrics_set: \"coco_detection_metrics\"\n  use_moving_averages: false\n  batch_size: 2;\n}\n\neval_input_reader: {\n  label_map_path: \"dataset/label_map.pbtxt\"\n  shuffle: false\n  num_epochs: 1\n  tf_record_input_reader {\n    input_path: \"dataset/cots_val-?????-of-00004\"\n  }\n}\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-11-28T02:37:10.460705Z","iopub.execute_input":"2021-11-28T02:37:10.461188Z","iopub.status.idle":"2021-11-28T02:37:10.489182Z","shell.execute_reply.started":"2021-11-28T02:37:10.46108Z","shell.execute_reply":"2021-11-28T02:37:10.488578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the training pipeline\n\nTRAINING_STEPS = 20000\nWARMUP_STEPS = 2000\nPIPELINE_CONFIG_PATH='dataset/pipeline.config'\n\npipeline = Template(config_file_template).substitute(\n    training_steps=TRAINING_STEPS, warmup_steps=WARMUP_STEPS)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"MODEL_DIR='cots_faster_rcnn_resnet101'\n!mkdir {MODEL_DIR}\n!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --alsologtostderr","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval","metadata":{}},{"cell_type":"code","source":"!python models/research/object_detection/model_main_tf2.py \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --model_dir={MODEL_DIR} \\\n    --checkpoint_dir={MODEL_DIR} \\\n    --eval_timeout=0 \\\n    --alsologtostderr","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export","metadata":{}},{"cell_type":"code","source":"!python models/research/object_detection/exporter_main_v2.py \\\n    --input_type image_tensor \\\n    --pipeline_config_path={PIPELINE_CONFIG_PATH} \\\n    --trained_checkpoint_dir={MODEL_DIR} \\\n    --output_directory={MODEL_DIR}/output","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {MODEL_DIR}/output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean Up","metadata":{}},{"cell_type":"code","source":"# Remove the dataset files to save space.\n!rm -rf dataset\n!rm -rf train_images\n!rm tensorflow-great-barrier-reef.zip\n\n# Remove other data downloaded during training.\n!rm -rf models\n!rm faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz","metadata":{},"execution_count":null,"outputs":[]}]}