{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TensorFlow - Help Protect the Great Barrier Reef\n#### Detect crown-of-thorns starfish in underwater image data\n\n**Goal:** Build an object detection model trained on underwater videos of coral reefs to accurately identify starfish in real-time.\n\nThis work will help researchers identify species that are threatening Australia's Great Barrier Reef and take well-informed action to protect the reef for future generations.","metadata":{}},{"cell_type":"markdown","source":"# Introduction\nAustralia's stunningly beautiful Great Barrier Reef is the world’s largest coral reef and home to 1,500 species of fish, 400 species of corals, 130 species of sharks, rays, and a massive variety of other sea life.\n\nUnfortunately, the reef is under threat, in part because of the overpopulation of one particular starfish – the coral-eating crown-of-thorns starfish (or COTS for short). Scientists, tourism operators and reef managers established a large-scale intervention program to control COTS outbreaks to ecologically sustainable levels.\n\nTo know where the COTS are, a traditional reef survey method, called \"Manta Tow\", is performed by a snorkel diver. While towed by a boat, they visually assess the reef, stopping to record variables observed every 200m. While generally effective, this method faces clear limitations, including operational scalability, data resolution, reliability, and traceability.\n\nThe Great Barrier Reef Foundation established an innovation program to develop new survey and intervention methods to provide a step change in COTS Control. Underwater cameras will collect thousands of reef images and AI technology could drastically improve the efficiency and scale at which reef managers detect and control COTS outbreaks.\n\nTo scale up video-based surveying systems, Australia’s national science agency, CSIRO has teamed up with Google to develop innovative machine learning technology that can analyse large image datasets accurately, efficiently, and in near real-time.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport os\nimport sys\nimport tensorflow as tf\nimport time\n\n# Import the library that is used to submit the prediction result.\nINPUT_DIR = '../input/tensorflow-great-barrier-reef/'\nsys.path.insert(0, INPUT_DIR)\nimport greatbarrierreef","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:47:44.813167Z","iopub.execute_input":"2021-12-03T07:47:44.813838Z","iopub.status.idle":"2021-12-03T07:47:48.987849Z","shell.execute_reply.started":"2021-12-03T07:47:44.813688Z","shell.execute_reply":"2021-12-03T07:47:48.987103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf_train","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:47:48.991326Z","iopub.execute_input":"2021-12-03T07:47:48.991527Z","iopub.status.idle":"2021-12-03T07:47:49.053184Z","shell.execute_reply.started":"2021-12-03T07:47:48.991501Z","shell.execute_reply":"2021-12-03T07:47:49.052563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train.annotations.str.len() > 6]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:47:49.054544Z","iopub.execute_input":"2021-12-03T07:47:49.055174Z","iopub.status.idle":"2021-12-03T07:47:49.088258Z","shell.execute_reply.started":"2021-12-03T07:47:49.055135Z","shell.execute_reply":"2021-12-03T07:47:49.087617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\nast.literal_eval(df_train.iloc[16].annotations)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:47:49.090215Z","iopub.execute_input":"2021-12-03T07:47:49.090457Z","iopub.status.idle":"2021-12-03T07:47:49.098269Z","shell.execute_reply.started":"2021-12-03T07:47:49.090426Z","shell.execute_reply":"2021-12-03T07:47:49.097574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image, ImageDraw\n\ndef fetch_image_list(df_tmp, video_id, num_images, start_frame_idx):\n    def fetch_image(frame_id):\n        path_base = '/kaggle/input/tensorflow-great-barrier-reef/train_images/video_{}/{}.jpg'\n        raw_img = Image.open(path_base.format(video_id, frame_id))\n\n        row_frame = df_tmp[(df_tmp.video_id == video_id) & (df_tmp.video_frame == frame_id)].iloc[0]\n        bounding_boxes = ast.literal_eval(row_frame.annotations)\n\n        for box in bounding_boxes:\n            draw = ImageDraw.Draw(raw_img)\n            x0, y0, x1, y1 = (box['x'], box['y'], box['x']+box['width'], box['y']+box['height'])\n            draw.rectangle( (x0, y0, x1, y1), outline=180, width=3)\n        return raw_img\n\n    return [np.array(fetch_image(start_frame_idx + index)) for index in range(num_images)]\n\nimages = fetch_image_list(df_train, video_id = 0, num_images = 80, start_frame_idx = 25)\n\nprint(\"Num images: \", len(images))\nplt.imshow(images[0], interpolation='nearest')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:47:49.09958Z","iopub.execute_input":"2021-12-03T07:47:49.100113Z","iopub.status.idle":"2021-12-03T07:47:53.130316Z","shell.execute_reply.started":"2021-12-03T07:47:49.100075Z","shell.execute_reply":"2021-12-03T07:47:53.12955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import animation, rc\nrc('animation', html='jshtml')\n\ndef create_animation(ims):\n    fig = plt.figure(figsize=(9, 9))\n    plt.axis('off')\n    im = plt.imshow(ims[0])\n\n    def animate_func(i):\n        im.set_array(ims[i])\n        return [im]\n\n    return animation.FuncAnimation(fig, animate_func, frames = len(ims), interval = 1000//12)\n\ncreate_animation(images)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-12-03T07:47:53.131408Z","iopub.execute_input":"2021-12-03T07:47:53.131663Z","iopub.status.idle":"2021-12-03T07:48:02.749113Z","shell.execute_reply.started":"2021-12-03T07:47:53.131628Z","shell.execute_reply":"2021-12-03T07:48:02.748439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Tensorflow COTS Model to run inference","metadata":{}},{"cell_type":"code","source":"MODEL_DIR = '../input/cots-detection-w-tensorflow-object-detection-api/cots_efficientdet_d0'\nstart_time = time.time()\ntf.keras.backend.clear_session()\ndetect_fn_tf_odt = tf.saved_model.load(os.path.join(os.path.join(MODEL_DIR, 'output'), 'saved_model'))\nend_time = time.time()\nelapsed_time = end_time - start_time\nprint('Elapsed time: ' + str(elapsed_time) + 's')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:48:02.75038Z","iopub.execute_input":"2021-12-03T07:48:02.751094Z","iopub.status.idle":"2021-12-03T07:48:32.257289Z","shell.execute_reply.started":"2021-12-03T07:48:02.751055Z","shell.execute_reply":"2021-12-03T07:48:32.256538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image_into_numpy_array(path):\n    \"\"\"Load an image from file into a numpy array.\n\n    Puts image into numpy array to feed into tensorflow graph.\n    Note that by convention we put it into a numpy array with shape\n    (height, width, channels), where channels=3 for RGB.\n\n    Args:\n    path: a file path (this can be local or on colossus)\n\n    Returns:\n    uint8 numpy array with shape (img_height, img_width, 3)\n    \"\"\"\n    img_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(io.BytesIO(img_data))\n    (im_width, im_height) = image.size\n    \n    return np.array(image.getdata()).reshape(\n      (im_height, im_width, 3)).astype(np.uint8)\n\ndef detect(image_np):\n    \"\"\"Detect COTS from a given numpy image.\"\"\"\n\n    input_tensor = np.expand_dims(image_np, 0)\n    start_time = time.time()\n    detections = detect_fn_tf_odt(input_tensor)\n    return detections","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:48:32.258653Z","iopub.execute_input":"2021-12-03T07:48:32.259067Z","iopub.status.idle":"2021-12-03T07:48:32.266353Z","shell.execute_reply.started":"2021-12-03T07:48:32.25903Z","shell.execute_reply":"2021-12-03T07:48:32.265627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using the provided python time-series API to create submission file","metadata":{}},{"cell_type":"code","source":"env = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:48:32.26749Z","iopub.execute_input":"2021-12-03T07:48:32.267802Z","iopub.status.idle":"2021-12-03T07:48:32.276553Z","shell.execute_reply.started":"2021-12-03T07:48:32.267766Z","shell.execute_reply":"2021-12-03T07:48:32.275859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DETECTION_THRESHOLD = 0.19\n\nsubmission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\n\nfor (image_np, sample_prediction_df) in iter_test:\n    height, width, _ = image_np.shape\n    \n    # Run object detection using the TensorFlow model.\n    detections = detect(image_np)\n    \n    # Parse the detection result and generate a prediction string.\n    num_detections = detections['num_detections'][0].numpy().astype(np.int32)\n    predictions = []\n    for index in range(num_detections):\n        score = detections['detection_scores'][0][index].numpy()\n        if score < DETECTION_THRESHOLD:\n            continue\n\n        bbox = detections['detection_boxes'][0][index].numpy()\n        y_min = int(bbox[0] * height)\n        x_min = int(bbox[1] * width)\n        y_max = int(bbox[2] * height)\n        x_max = int(bbox[3] * width)\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    # Generate the submission data.\n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n\n    print('Prediction:', prediction_str)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T07:48:32.279157Z","iopub.execute_input":"2021-12-03T07:48:32.279503Z","iopub.status.idle":"2021-12-03T07:48:41.378478Z","shell.execute_reply.started":"2021-12-03T07:48:32.279467Z","shell.execute_reply":"2021-12-03T07:48:41.37773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}