{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n
\n\n

TF - Great Barrier Reef - EDA

\n
CREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n
\n πŸ›‘   WARNING:

THIS IS A WORK IN PROGRESS
\n
\n\n\n
\n πŸ‘   IF YOU FORK THIS OR FIND THIS HELPFUL   πŸ‘

PLEASE UPVOTE!

This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. πŸ˜…\n
\n\n\n","metadata":{"papermill":{"duration":0.120561,"end_time":"2021-11-06T21:15:09.611563","exception":false,"start_time":"2021-11-06T21:15:09.491002","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

\n\n

\n\n

TABLE OF CONTENTS

\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    BACKGROUND INFORMATION

\n\n---\n\n

2    SETUP

\n\n---\n\n

3    HELPER FUNCTIONS

\n\n---\n\n

4    DATASET CREATION AND EXPLORATION

\n\n---\n\n

5    MODELLING

\n\n---","metadata":{"papermill":{"duration":0.085591,"end_time":"2021-11-06T21:15:09.78303","exception":false,"start_time":"2021-11-06T21:15:09.697439","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

0  IMPORTS    ⤒

","metadata":{"papermill":{"duration":0.050527,"end_time":"2021-11-06T21:15:09.894476","exception":false,"start_time":"2021-11-06T21:15:09.843949","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n... PIP/APT INSTALLS AND DOWNLOADS/ZIP STARTING ...\")\n# !pip install -q tensorflow-model-optimization\n# !pip install -q neural-structured-learning\n# !pip install -q tensorflow_datasets\n\n## Only to be used if we can figure out that TPU bug\n# !pip install -q tf-nightly\n\n# Try to skip and disable so we can submit w/o internet\n# !pip install -q ../input/tensorflow-model-optimization/numpy-1.21.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n# !pip install -q ../input/tensorflow-model-optimization/dm_tree-0.1.6-cp37-cp37m-manylinux_2_24_x86_64.whl\n# !pip install -q ../input/tensorflow-model-optimization/six-1.16.0-py2.py3-none-any.whl\n# !pip install -q ../input/tensorflow-model-optimization/tensorflow_model_optimization-0.7.0-py2.py3-none-any.whl\n# !pip install ../input/neural-structued-learning/neural_structured_learning-1.3.1-py2.py3-none-any.whl\n\nprint(\"... PIP/APT INSTALLS COMPLETE ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\t– SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold;\n\n# This is necessary to force the TPU client to have the same TF version as TF-Nightly\nfrom cloud_tpu_client import Client\nc = Client(tpu=''); c.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n\n# Competition Specific\n# import greatbarrierreef\n# env = greatbarrierreef.make_env() # initialize the environment\n\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")\n \n# print(\"\\n... EFFICIENTDET SETUP STARTING ...\")\n\n# # SET LIBRARY DIRECTORY\n# LIB_DIR = \"/kaggle/input/google-automl-efficientdetefficientnet-oct-2021\"\n\n# # To give access to automl files\n# sys.path.insert(0, LIB_DIR)\n# sys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\"))\n# sys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\", \"efficientdet\"))\n# sys.path.insert(0, os.path.join(LIB_DIR, \"automl-master\", \"efficientdet\", \"tf2\"))\n \n# # EfficientDET Module Imports\n# import hparams_config\n# from tf2 import efficientdet_keras\n# from tf2 import train_lib\n# from tf2 import anchors\n# from tf2 import efficientdet_keras\n# from tf2 import label_util\n# from tf2 import postprocess\n# from tf2 import util_keras\n# from tf2.train import setup_model\n# from efficientdet import dataloader\n# from visualize import vis_utils\n# from inference import visualize_image\n# print(\"... EFFICIENTDET SETUP COMPLETE ...\\n\")\n\n# print(\"\\n... SEEDING FOR DETERMINISTIC BEHAVIOUR ...\\n\")\n# seed_it_all()","metadata":{"papermill":{"duration":162.144149,"end_time":"2021-11-06T21:17:52.087371","exception":false,"start_time":"2021-11-06T21:15:09.943222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:06:01.091201Z","iopub.execute_input":"2021-12-13T17:06:01.091705Z","iopub.status.idle":"2021-12-13T17:06:24.509964Z","shell.execute_reply.started":"2021-12-13T17:06:01.091616Z","shell.execute_reply":"2021-12-13T17:06:24.508901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

1  BACKGROUND INFORMATION    ⤒

\n\n---\n","metadata":{"papermill":{"duration":0.05019,"end_time":"2021-11-06T21:17:52.231372","exception":false,"start_time":"2021-11-06T21:17:52.181182","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.1 BASIC COMPETITION INFORMATION

\n\n---\n\n
PRIMARY TASK DESCRIPTION\n\nThe goal of this competition is to accurately **identify starfish** in real-time by **building an object detection model** trained on **underwater videos** of coral reefs.\n\nYour work will help researchers identify species that are threatening Australia's Great Barrier Reef and take well-informed action to protect the reef for future generations.\n\n
\n\n---\n\n
CONTEXT\n\nAustralia's stunningly beautiful Great Barrier Reef is the world’s largest coral reef and home to 1,500 species of fish, 400 species of corals, 130 species of sharks, rays, and a massive variety of other sea life.\n\nUnfortunately, the reef is under threat, in part because of the overpopulation of one particular starfish – the coral-eating crown-of-thorns starfish (or COTS for short). Scientists, tourism operators and reef managers established a large-scale intervention program to control COTS outbreaks to ecologically sustainable levels.\n\n\n\n---\n\n
MORE BACKGROUND INFORMATION\n\n\nTo know where the COTS (crown-of-thorns starfish) are, a traditional reef survey method, called \"Manta Tow\", is performed by a snorkel diver. While towed by a boat, they visually assess the reef, stopping to record variables observed every 200m. While generally effective, this method faces clear limitations, including operational scalability, data resolution, reliability, and traceability.\n\n\n\nThe Great Barrier Reef Foundation established an innovation program to develop new survey and intervention methods to provide a step change in COTS Control. Underwater cameras will collect thousands of reef images and AI technology could drastically improve the efficiency and scale at which reef managers detect and control COTS outbreaks.\n\nTo scale up video-based surveying systems, Australia’s national science agency, CSIRO has teamed up with Google to develop innovative machine learning technology that can analyse large image datasets accurately, efficiently, and in near real-time.","metadata":{"papermill":{"duration":0.053372,"end_time":"2021-11-06T21:17:52.337029","exception":false,"start_time":"2021-11-06T21:17:52.283657","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.2 COMPETITION EVALUATION

\n\n---\n\n
GENERAL EVALUATION INFORMATION\n\n\nThis competition is evaluated on the [**F2 Score**](https://en.wikipedia.org/wiki/F-score) at different intersection over union (IoU) thresholds\n* The F2 metric weights [recall](https://www.wikiwand.com/en/Precision_and_recall) more heavily than [precision](https://www.wikiwand.com/en/Precision_and_recall), as in this case it makes sense to tolerate some false positives in order to ensure very few starfish are missed.\n* The metric sweeps over IoU thresholds in the range of 0.3 to 0.8 with a step size of 0.05 calculating an F2 score at each threshold. \n * For example, at a threshold of 0.5, a predicted object is considered a \"hit\" if its IoU with a ground truth object is at least 0.5\n* A true positive is the first (in confidence order, see details below) submission box in a sample with an IoU greater than the threshold against an unmatched solution box.\n* Once all submission boxes have been evaluated, any unmatched submission boxes are false positives\n * Any remaining unmatched solution boxes are false negatives\n* The final F2 Score is calculated as the mean of the F2 scores at each IoU threshold. \n* Within each IoU threshold the competition metric uses micro averaging; \n * every true positive, false positive, and false negative has equal weight compared to each other true positive, false positive, and false negative.\n \n
SUBMISSION FORMAT\n\nThe submission format requires a space delimited set of bounding boxes. \n* For example:\n * `abc,0.5 0 0 100 100` indicates that image abc has a bounding box with a confidence of 0.5, at x == 0 and y == 0, with a width and height of 100.\n * wheras `def,0.3 0 0 50 50 0.5 10 10 30 30` indicates an image with two bounding boxes\n* Each prediction row needs to include all bounding boxes for the image.\n\n
\n\nThis competition uses a hidden test set that will be served by an API to ensure you evaluate the images in the same order they were recorded within each video. When your submitted notebook is scored, the actual test data (including a sample submission) will be availabe to your notebook. The following code snippet shows how to initialize the submission environment, generate the test set, and register your predictions on the test set\n\n```python\nimport greatbarrierreef\nenv = greatbarrierreef.make_env() # initialize the environment\niter_test = env.iter_test() # an iterator which loops over the test set and sample submission\nfor (pixel_array, sample_prediction_df) in iter_test:\n sample_prediction_df['annotations'] = 'abc,0.5 0 0 100 100' # make your predictions here\n env.predict(sample_prediction_df) # register your predictions\n```\n\n
IS THIS A CODE COMPETITION?\n\nYES!\n","metadata":{"papermill":{"duration":0.052259,"end_time":"2021-11-06T21:17:52.443218","exception":false,"start_time":"2021-11-06T21:17:52.390959","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.3 DATASET OVERVIEW

\n\n---\n\n
GENERAL INFORMATION\n\nIn this competition, you will predict the presence ***(detection)*** and position ***(localization)*** of crown-of-thorns starfish (COTS) in sequences of underwater images taken at various times and locations around the Great Barrier Reef. \n* Predictions take the form of a bounding box together with a confidence score for each identified starfish. \n* An image may contain zero or more starfish\n\nThis competition uses a hidden test set that will be served by an API to ensure you evaluate the images in the same order they were recorded within each video. When your submitted notebook is scored, the actual test data (including a sample submission) will be availabe to your notebook.\n\n
DISCOVERED INFORMATION [TENTATIVE]\n\nThis is the information I have discovered and believe to be very salient.\n\n* Image Shape\n * **`720x1280`**

\n* Bounding Box Format\n * **`(xmin, ymin, width, height)`**

\n* # Of Train Images\n * **``23,501**

\n* Approx # Of Test Images\n * **`13,000`**
\n* Minimum Number of Annotations Per Image\n * **`0`**
\n* Maximum Number of Annotations Per Image\n * **`18`**
\n\n
FILES\n\n**`train/`** - Folder containing training set photos of the form **`video_{video_id}/{video_frame_number}.jpg`**\n\n* **`[train/test].csv`** - Metadata for the images. \n * `video_id` \n * ID number of the video the image was part of. The video ids are not meaningfully ordered.\n * `video_frame`\n * The frame number of the image within the video. Expect to see occasional gaps in the frame number from when the diver surfaced.\n * `sequence`\n * ID of a gap-free subset of a given video. The sequence ids are not meaningfully ordered.\n * `sequence_frame`\n * The frame number within a given sequence.\n * `image_id`\n * ID code for the image, in the format '{video_id}-{video_frame}'\n * `annotations`\n * The bounding boxes of any starfish detections in a string format that can be evaluated directly with Python. \n * Does not use the same format as the predictions you will submit. \n * Not available in **`test.csv`**\n * A bounding box is described by the pixel coordinate (x_min, y_min) of its lower left corner (seems wrong) within the image together with its width and height in pixels. [x_min, y_min, width, height]\n\n\n* **`example_sample_submission.csv`** - A sample submission file in the correct format. \n * The actual sample submission will be provided by the API\n * This file is only provided to illustrate how to properly format predictions. \n * The submission format has been described in more detail above...\n\n\n* **`example_test.npy`** - Sample data that will be served by the example API.\n\n
\n\n**`greatbarrierreef/`** - The image delivery API that will serve the test set pixel arrays. \n* You may need Python 3.7 and a Linux environment to run the example offline without errors.\n\n
\n\n
TIME SERIES API DETAILS\n\n* The API serves the images one by one, in order by video and frame number, as pixel arrays.\n* Expect to see roughly 13,000 images in the test set.\n* The API will require roughly two GB of memory after initialization. \n * The initialization step (env.iter_test()) will require meaningfully more memory than that; we recommend you do not load your model until after making that call. \n * The API will also consume less than ten minutes of runtime for loading and serving the data.","metadata":{"papermill":{"duration":0.052579,"end_time":"2021-11-06T21:17:52.548406","exception":false,"start_time":"2021-11-06T21:17:52.495827","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

2  SETUP    ⤒

\n\n---\n","metadata":{"papermill":{"duration":0.052944,"end_time":"2021-11-06T21:17:52.65284","exception":false,"start_time":"2021-11-06T21:17:52.599896","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

2.1 ACCELERATOR DETECTION

\n\n---\n\nIn order to use **`TPU`**, we use **`TPUClusterResolver`** for the initialization which is necessary to connect to the remote cluster and initialize cloud TPUs. Let's go over two important points\n\n1. When using TPU on Kaggle, you don't need to specify arguments for **`TPUClusterResolver`**\n2. However, on **G**oogle **C**ompute **E**ngine (**GCE**), you will need to do the following:\n\n
\n\n```python\n# The name you gave to the TPU to use\nTPU_WORKER = 'my-tpu-name'\n\n# or you can also specify the grpc path directly\n# TPU_WORKER = 'grpc://xxx.xxx.xxx.xxx:8470'\n\n# The zone you chose when you created the TPU to use on GCP.\nZONE = 'us-east1-b'\n\n# The name of the GCP project where you created the TPU to use on GCP.\nPROJECT = 'my-tpu-project'\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER, zone=ZONE, project=PROJECT)\n```\n\n
\n πŸ›‘   WARNING:

- Although the Tensorflow documentation says it is the project name that should be provided for the argument `project`, it is actually the Project ID, that you should provide. This can be found on the GCP project dashboard page.
\n
\n\n
\n πŸ“–   REFERENCES:

\n - Guide - Use TPUs
\n - Doc - TPUClusterResolver
\n\n
","metadata":{"papermill":{"duration":0.053821,"end_time":"2021-11-06T21:17:52.761303","exception":false,"start_time":"2021-11-06T21:17:52.707482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"papermill":{"duration":0.07574,"end_time":"2021-11-06T21:17:52.892074","exception":false,"start_time":"2021-11-06T21:17:52.816334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:06:24.511423Z","iopub.execute_input":"2021-12-13T17:06:24.511978Z","iopub.status.idle":"2021-12-13T17:06:39.243944Z","shell.execute_reply.started":"2021-12-13T17:06:24.511930Z","shell.execute_reply":"2021-12-13T17:06:39.242709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.2 COMPETITION DATA ACCESS

\n\n---\n\nTPUs read data must be read directly from **G**oogle **C**loud **S**torage **(GCS)**. Kaggle provides a utility library – **`KaggleDatasets`** – which has a utility function **`.get_gcs_path`** that will allow us to access the location of our input datasets within **GCS**.

\n\n
\n πŸ“Œ   TIPS:

- If you have multiple datasets attached to the notebook, you should pass the name of a specific dataset to the `get_gcs_path()` function. In our case, the name of the dataset is the name of the directory the dataset is mounted within.

\n
","metadata":{"papermill":{"duration":0.053551,"end_time":"2021-11-06T21:17:52.999073","exception":false,"start_time":"2021-11-06T21:17:52.945522","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n # Google Cloud Dataset path to training and validation images\n DATA_DIR = KaggleDatasets().get_gcs_path('tensorflow-great-barrier-reef')\n DATASET_DIR = KaggleDatasets().get_gcs_path(\"binary-classifier-cots-dataset\")\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\nelse:\n # Local path to training and validation images\n DATA_DIR = \"/kaggle/input/tensorflow-great-barrier-reef\"\n DATASET_DIR = \"/kaggle/input/binary-classifier-cots-dataset\"\n save_locally = load_locally = None\n \n \nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\n \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.0797,"end_time":"2021-11-06T21:17:53.133972","exception":false,"start_time":"2021-11-06T21:17:53.054272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:06:39.245683Z","iopub.execute_input":"2021-12-13T17:06:39.245963Z","iopub.status.idle":"2021-12-13T17:06:42.608320Z","shell.execute_reply.started":"2021-12-13T17:06:39.245931Z","shell.execute_reply":"2021-12-13T17:06:42.607267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.3 LEVERAGING XLA OPTIMIZATIONS

\n\n---\n\n\n**XLA** (Accelerated Linear Algebra) is a domain-specific compiler for linear algebra that can accelerate TensorFlow models with potentially no source code changes. **The results are improvements in speed and memory usage**.\n\n
\n\nWhen a TensorFlow program is run, all of the operations are executed individually by the TensorFlow executor. Each TensorFlow operation has a precompiled GPU/TPU kernel implementation that the executor dispatches to.\n\nXLA provides us with an alternative mode of running models: it compiles the TensorFlow graph into a sequence of computation kernels generated specifically for the given model. Because these kernels are unique to the model, they can exploit model-specific information for optimization.

\n\n
\n πŸ›‘   WARNING:

- XLA can not currently compile functions where dimensions are not inferrable: that is, if it's not possible to infer the dimensions of all tensors without running the entire computation
\n
\n\n
\n πŸ“Œ   NOTE:

- XLA compilation is only applied to code that is compiled into a graph (in TF2 that's only a code inside tf.function).
- The jit_compile API has must-compile semantics, i.e. either the entire function is compiled with XLA, or an errors.InvalidArgumentError exception is thrown)\n
\n\n
\n πŸ“–   REFERENCE:

- XLA: Optimizing Compiler for Machine Learning
\n
","metadata":{"papermill":{"duration":0.051494,"end_time":"2021-11-06T21:17:53.236017","exception":false,"start_time":"2021-11-06T21:17:53.184523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.128128,"end_time":"2021-11-06T21:17:53.442803","exception":false,"start_time":"2021-11-06T21:17:53.314675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:07:26.842001Z","iopub.execute_input":"2021-12-13T17:07:26.842287Z","iopub.status.idle":"2021-12-13T17:07:26.848853Z","shell.execute_reply.started":"2021-12-13T17:07:26.842258Z","shell.execute_reply":"2021-12-13T17:07:26.847744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.4 BASIC DATA DEFINITIONS & INITIALIZATIONS

\n\n---\n","metadata":{"papermill":{"duration":0.090507,"end_time":"2021-11-06T21:17:53.624612","exception":false,"start_time":"2021-11-06T21:17:53.534105","status":"completed"},"tags":[]}},{"cell_type":"code","source":"####################################################################\n# #################### TEST API CODE #################### #\n####################################################################\n# iter_test = env.iter_test() # an iterator which loops over the test set and sample submission\n# for (pixel_array, sample_prediction_df) in iter_test:\n# sample_prediction_df['annotations'] = 'abc,0.5 0 0 100 100' # make your predictions here\n# env.predict(sample_prediction_df) # register your predictions\n####################################################################\n\nprint(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\nDEFAULT_PLOTLY_COLORS=[(31, 119, 180), (255, 127, 14), \n (44, 160, 44), (214, 39, 40),\n (148, 103, 189), (140, 86, 75),\n (227, 119, 194), (127, 127, 127),\n (188, 189, 34), (23, 190, 207)]\n\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df[\"img_path\"] = os.path.join(DATA_DIR, \"train_images\")+\"/video_\"+train_df.video_id.astype(str)+\"/\"+train_df.video_frame.astype(str)+\".jpg\"\ntrain_df[\"annotations\"] = train_df[\"annotations\"].apply(lambda x: ast.literal_eval(x))\ntrain_df[\"a_count\"] = train_df[\"annotations\"].apply(len)\ntrain_df[\"video_id\"] = train_df[\"video_id\"].astype(str)\ntrain_df[\"sequence\"] = train_df[\"sequence\"].astype(str)\n\nprint(\"\\n... TRAIN DATAFRAME ...\\n\")\ndisplay(train_df)\n\nprint(\"\\n... OBJ ONLY DATAFRAME ...\\n\")\nobj_only_df = train_df[train_df.a_count>0].reset_index(drop=True)\n\nprint(\"\\n... TEST DATAFRAME ...\\n\")\nTEST_CSV = os.path.join(DATA_DIR, \"test.csv\")\ntest_df = pd.read_csv(TEST_CSV)\ndisplay(test_df)\n\nprint(\"\\n... SS DATAFRAME ..\\n\")\nSS_CSV = os.path.join(DATA_DIR, \"example_sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\ndisplay(ss_df)\n\nprint(\"\\n\\n... BASIC DATA SETUP FINISHING ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:27.222497Z","iopub.execute_input":"2021-12-13T17:07:27.222873Z","iopub.status.idle":"2021-12-13T17:07:37.331252Z","shell.execute_reply.started":"2021-12-13T17:07:27.222840Z","shell.execute_reply":"2021-12-13T17:07:37.330254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 3  HELPER FUNCTION & CLASSES    ⤒\n

\n\n---","metadata":{"papermill":{"duration":0.054893,"end_time":"2021-11-06T21:17:54.695576","exception":false,"start_time":"2021-11-06T21:17:54.640683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\ndef tf_load_img(img_path, reshape_to=None):\n if reshape_to is None:\n return tf.image.decode_image(tf.io.read_file(img_path), channels=3)\n else:\n return tf.image.resize(tf.image.decode_image(tf.io.read_file(img_path), channels=3), reshape_to)\n\n# ORIGINAL CREDIT \n# --> https://www.kaggle.com/diegoalejogm/great-barrier-reefs-eda-with-animations\ndef create_animation(img_path_list, annotations=None, fps=12):\n \"\"\" Create an animtation from a list of image paths \"\"\"\n fig = plt.figure(figsize=(20, 11))\n plt.axis('off')\n \n img = plt.imshow(cv2.imread(img_path_list[0])[..., ::-1].astype(np.int32))\n \n def animate_func(i):\n tmp_img = cv2.imread(img_path_list[i])[..., ::-1].astype(np.int32)\n if annotations is not None:\n if len(annotations[i])>0:\n for bbox in annotations[i]:\n tl_box, br_box = get_tl_br(bbox)\n tmp_img = cv2.rectangle(tmp_img, tl_box, br_box, (255,0,0), 8)\n \n img.set_array(tmp_img)\n return [img]\n plt.close()\n return animation.FuncAnimation(fig, animate_func, frames=len(img_path_list), interval=1000//fps)\n\n\ndef get_tl_br(bbox):\n \"\"\" Return the top-left and bottom-right bounding box \"\"\"\n return (bbox['x'], bbox['y']), (bbox['x']+bbox[\"width\"], bbox['y']+bbox[\"height\"])\n\ndef plot_image(img_path, annotations=None, **kwargs):\n \"\"\" Plot an image and bounding boxes \"\"\"\n img = np.array(tf_load_img(img_path))\n \n if annotations:\n plt.figure(figsize=(20,10))\n for i, bbox in enumerate(annotations):\n tl_box, br_box = get_tl_br(bbox)\n img = cv2.rectangle(img, tl_box, br_box, (255-2*i,14*i,0), 4)\n plt.imshow(img)\n plt.axis(False)\n plt.title(f\"Bounding Boxes Shown in Red ({len(annotations)})\", fontweight=\"bold\")\n else:\n plt.figure(figsize=(20,10))\n plt.imshow(img)\n plt.axis(False)\n plt.title(\"No Bounding Boxes\", fontweight=\"bold\")\n plt.tight_layout()\n plt.show()","metadata":{"papermill":{"duration":0.098071,"end_time":"2021-11-06T21:17:54.848036","exception":false,"start_time":"2021-11-06T21:17:54.749965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:07:37.333016Z","iopub.execute_input":"2021-12-13T17:07:37.333250Z","iopub.status.idle":"2021-12-13T17:07:37.351158Z","shell.execute_reply.started":"2021-12-13T17:07:37.333222Z","shell.execute_reply":"2021-12-13T17:07:37.350440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 4  DATASET EXPLORATION    ⤒\n

\n\n---","metadata":{"papermill":{"duration":0.056443,"end_time":"2021-11-06T21:17:54.960518","exception":false,"start_time":"2021-11-06T21:17:54.904075","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

4.1 TRAIN METADATA

\n\n---\n","metadata":{"papermill":{"duration":0.053158,"end_time":"2021-11-06T21:17:55.070372","exception":false,"start_time":"2021-11-06T21:17:55.017214","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... TRAIN METADATA INVESTIGATION STARTING ...\\n\\n\")\n\nprint(\"\\n... TRAIN DATAFRAME HEAD(5) ...\\n\\n\")\ndisplay(train_df.head(5))\n\nprint(\"\\n... TRAIN DATAFRAME TAIL(5) ...\\n\\n\")\ndisplay(train_df.tail(5))\n\nprint(\"\\n... TRAIN DATAFRAME SAMPLE(5) ...\\n\\n\")\ndisplay(train_df.sample(5))\n\nIMG_SHAPE = (720, 1280, 3)\nN_TRAIN = len(train_df)\nall_bboxes = flatten_l_o_l(train_df[train_df.a_count>0].annotations.to_list())\nN_OBJECTS = len(all_bboxes)\nN_VIDEO = train_df.video_id.nunique()\nN_SEQ = train_df.sequence.nunique()\nprint(f\"\\n... NUMBER OF UNIQUE TRAINING IMAGES: {N_TRAIN} ...\")\nprint(f\"... NUMBER OF UNIQUE OBJECTS IN TRAINING DATASET: {N_OBJECTS} ...\")\nprint(f\"... IMAGE SHAPE: {IMG_SHAPE}\")\nprint(\"\\n... TRAIN DATAFRAME PANDAS DESCRIPTION ...\\n\\n\")\ndisplay(train_df.describe().T)\n\nprint(f\"\\n... NUMBER OF UNIQUE VIDEOS: {N_VIDEO} ...\\n\")\n\nfig = px.bar(train_df.groupby(\"video_id\")[\"video_frame\"].max(), color=px.colors.qualitative.Plotly[:3],\n labels={\"sequence\":\"Video IDb>\", \"value\":\"Number Of Frames In Video\", \"variable\":\"Original Column Name\"},\n title=\"Number Of Frames In Each Total Video\")\nfig.update_layout(xaxis=dict(type='category'), showlegend=False)\nfig.show()\n\n\nprint(f\"\\n... NUMBER OF UNIQUE VIDEO SEQUENCES: {N_SEQ} ...\\n\")\n\nfig = px.bar(train_df.groupby(\"sequence\")[[\"sequence_frame\", \"video_id\"]].max().sort_values(by=\"video_id\"), color=\"video_id\",\n labels={\"sequence\":\"Sequence ID\", \"value\":\"Number Of Frames In Sequence\", \"variable\":\"Original Column Name\"},\n title=\"Number Of Frames In Each Sequence (Subset of Video)\")\nfig.update_layout(xaxis=dict(type='category'), showlegend=False)\nfig.show()\n\nprint(f\"\\n... NUMBER OF ANNOTATIONS PER IMAGE ...\\n\")\ndisplay(train_df.a_count.value_counts().to_frame())\nfig = px.histogram(train_df, x=\"sequence\", color=\"a_count\",\n labels={\"sequence\":\"Sequence ID\", \"a_count\":\"# Annot. In Image\"},\n title=\"Number Of Annotations In Each Image\")\nfig.show()\n\nprint(\"\\n\\n... TRAIN METADATA INVESTIGATION FINISHING ...\\n\")","metadata":{"papermill":{"duration":0.313817,"end_time":"2021-11-06T21:17:55.439001","exception":false,"start_time":"2021-11-06T21:17:55.125184","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-13T17:07:37.352218Z","iopub.execute_input":"2021-12-13T17:07:37.352505Z","iopub.status.idle":"2021-12-13T17:07:39.201743Z","shell.execute_reply.started":"2021-12-13T17:07:37.352446Z","shell.execute_reply":"2021-12-13T17:07:39.200749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.2 VISUALIZE THE TRAIN DATA

\n\n---","metadata":{"papermill":{"duration":0.05563,"end_time":"2021-11-06T21:17:55.55248","exception":false,"start_time":"2021-11-06T21:17:55.49685","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for a_count in sorted(train_df.a_count.unique()):\n ex_row = train_df[train_df.a_count==a_count].reset_index(drop=True).iloc[0]\n plot_image(**ex_row)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.3 VISUALIZE & INVESTIGATE INDIVIDUAL OBJECTS

\n\n---\n\n","metadata":{"papermill":{"duration":0.684105,"end_time":"2021-11-06T21:18:30.682923","exception":false,"start_time":"2021-11-06T21:18:29.998818","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.imshow(tf_load_img(train_df.img_path[100])[631:631+88, 276:276+116, :])\nplt.title(\"Individual COTS\", fontweight=\"bold\")\nplt.axis(False)\nplt.tight_layout()\nplt.show()\n\nbbox_train_df = train_df[train_df.a_count>0]\nheatmap_canvas = np.zeros((IMG_SHAPE), dtype=np.int32)\nfor bbox in all_bboxes:\n tl, br = get_tl_br(bbox)\n heatmap_canvas[tl[1]:br[1], tl[0]:br[0], 0]+=2\nplt.figure(figsize=(20,6))\nplt.imshow(heatmap_canvas)\nplt.title(\"Individual Object Box Heatmap\", fontweight=\"bold\")\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":8.127387,"end_time":"2021-11-06T21:18:39.473456","exception":false,"start_time":"2021-11-06T21:18:31.346069","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.4 VISUALIZE & INVESTIGATE SEQUENCES

\n\n---\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 30))\nfor i, (idx, row) in enumerate(train_df[train_df.sequence==\"18048\"].iterrows()):\n plt.subplot(int(np.ceil(len(train_df[train_df.sequence==\"18048\"])/5)), 5, i+1)\n \n img = np.array(tf_load_img(row.img_path))\n if row.annotations:\n for j, bbox in enumerate(row.annotations):\n tl_box, br_box = get_tl_br(bbox)\n img = cv2.rectangle(img, tl_box, br_box, (255-2*j,14*j,0), 4)\n plt.imshow(img)\n plt.axis(False)\n \nplt.tight_layout()\nplt.show()\n ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_animation(img_path_list, annotations=None, _fps=12):\n \"\"\" Create an animtation from a list of image paths \"\"\"\n \n # Animatino Function\n def animate_func(i):\n tmp_img = np.asarray(cv2.imread(img_path_list[i])[..., ::-1]/255)\n if annotations is not None:\n if len(annotations[i])>0:\n for bbox in annotations[i]:\n tl_box, br_box = get_tl_br(bbox)\n tmp_img = cv2.rectangle(tmp_img, tl_box, br_box, (1.0,0.0,0.0), 8)\n \n im.set_array(tmp_img)\n return [im,]\n \n # Instantiate initial framing\n fig = plt.figure(figsize=(7, 4)) # Drops frames... you lose more the bigger you go...\n plt.axis('off')\n im = plt.imshow(np.asarray(cv2.imread(img_path_list[0])[..., ::-1]/255.))\n plt.close()\n \n # Animate that video!\n return animation.FuncAnimation(fig, animate_func, frames=len(img_path_list), interval=1000/_fps)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**I need to change the below to use tf not cv2 otherwise it will break w/ TPU**","metadata":{}},{"cell_type":"code","source":"# # See a random video sequence\n# max_n_frames = 720 # Limit the max video length to 1 minute\n# random_seq_id = random.sample(list(train_df.sequence.unique()), 1)[0]\n# seq_img_path_list = train_df[train_df.sequence==random_seq_id].img_path.to_list()[:max_n_frames]\n# seq_annotation_list = train_df[train_df.sequence==random_seq_id].annotations.to_list()[:max_n_frames]\n\n# print(\"\\n... VIDEO SEQUENCE INFORMATION ...\\n\")\n# print(f\"# OF IMAGES : {len(seq_img_path_list)}\")\n# print(f\"# OF ANNOTATIONS TOTAL : {len(seq_annotation_list)}\")\n# print(f\"# OF ANNOTATIONS W/ BBOXES : {sum([1 if x!=[] else 0 for x in seq_annotation_list])}\\n\\n\")\n\n# print(f\"\\n... ANIMATION FOR VIDEO SEQUENCE ID {random_seq_id} ...\\n\")\n# create_animation(seq_img_path_list, seq_annotation_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.5 VISUALIZE & INVESTIGATE WHOLE VIDEO

\n\n---\n\nNote that the rectangle colour on the outside of the frame represents which individual sequence within the video we're seeing...","metadata":{}},{"cell_type":"code","source":"color_map = {k:i for i, k in enumerate(train_df[train_df.video_id==\"2\"].sequence.unique())}\n\nplt.figure(figsize=(20, 96))\nfor i, (idx, row) in tqdm(enumerate(train_df[train_df.video_id==\"2\"][::16].iterrows())):\n plt.subplot(int(np.ceil(len(train_df[train_df.video_id==\"2\"][::16])/8)), 8, i+1)\n img = np.array(tf_load_img(row.img_path))\n if row.annotations:\n for j, bbox in enumerate(row.annotations):\n tl_box, br_box = get_tl_br(bbox)\n img = cv2.rectangle(img, tl_box, br_box, (255-2*j,14*j,0), 8)\n img = cv2.rectangle(img, (0,0), (IMG_SHAPE[1], IMG_SHAPE[0]), DEFAULT_PLOTLY_COLORS[1+color_map[row.sequence]], 50)\n plt.imshow(img)\n plt.axis(False)\n \nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.6 STATS REGARDING BOUNDING BOXES

\n\n---\n\nTBD","metadata":{"papermill":{"duration":0.769828,"end_time":"2021-11-06T21:20:15.680491","exception":false,"start_time":"2021-11-06T21:20:14.910663","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true}},{"cell_type":"code","source":"widths = [x[\"width\"] for x in all_bboxes]\nheights = [x[\"height\"] for x in all_bboxes]\nareas = [x[\"width\"]*x[\"height\"] for x in all_bboxes]\n\n# Plot\nfig = px.histogram(pd.DataFrame({\"widths\":widths, \"heights\":heights}), nbins=250,\n labels={\"value\":\"Number of Pixels\"},\n title=\"Distribution of Bounding Box Heights and Widths\")\nfig.show()\n\nfig = px.histogram(pd.DataFrame({\"areas\":areas}), nbins=500,\n labels={\"value\":\"Number of Pixels\"},\n title=\"Distribution of Bounding Box Areas\")\nfig.show()","metadata":{"papermill":{"duration":2.917754,"end_time":"2021-11-06T21:20:19.48286","exception":false,"start_time":"2021-11-06T21:20:16.565106","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 5  BINARY CLASSIFIER AS BASELINE⤒\n

\n\n---\n\nFor my baseline model I plan to do the following:\n- Take only images that have COTS present\n- I will create a 10x10 grid of sub-images (72x128) for each image\n - sub-images will be referred to as grid images\n - original images will be referred to as global images\n- Grid images will be placed in two folders (background/foreground)\n - A grid image is considered foreground if it has a portion of box with more than 250px area\n - I will subsample the background images (1 for every 10)\n- These folders will be used to train a simple binary classifier using **EfficientNetV2**\n- We will then use GradCAM/PuzzleCAM to analyse the predictions\n- If there is a decent signal, we will use the grid images and the respective expalinable heatmaps to box the detections","metadata":{}},{"cell_type":"markdown","source":"

5.1 HELPFUL MODELLING FUNCTIONS

\n\n---","metadata":{}},{"cell_type":"code","source":"def compute_intersection(box_1, box_2):\n \"\"\"Computes pairwise IOU matrix for given two sets of boxes\n \n REFERENCE: https://keras.io/examples/vision/retinanet/\n \n Args:\n box_1 (tf.Tensor): An array with shape `(4,)` representing bounding boxes\n where each box is of the format `[xmin, ymin, xmax, ymax]`\n box_2: An array with shape `(4,)` representing bounding boxes\n where each box is of the format `[xmin, ymin, xmax, ymax]`\n\n Returns:\n intersection area between two boxes\n \"\"\"\n \n # determine the (x, y)-coordinates of the intersection rectangle\n tl_x_max = max(box_1[0], box_2[0])\n tl_y_max = max(box_1[1], box_2[1])\n br_x_min = min(box_1[2], box_2[2])\n br_y_min = min(box_1[3], box_2[3])\n \n inter_area = max(0, br_x_min-tl_x_max+1) * max(0, br_y_min-tl_y_max+1)\n inter_width, inter_height = br_x_min-tl_x_max, br_y_min-tl_y_max\n return inter_area, inter_width, inter_height\n\ndef get_grid_images(img, annotation, img_dim=(720,1280,3), grid_dim=(160,160,3), stride=(80,80), min_inter_area=250, min_inter_width=16, min_inter_height=16, return_as_ordered_dict=False):\n \n x_points = np.linspace(0, img_dim[1], img_dim[1]//stride[1]+1, dtype=np.int32)[:-grid_dim[1]//stride[1]]\n y_points = np.linspace(0, img_dim[0], img_dim[0]//stride[0]+1, dtype=np.int32)[:-grid_dim[0]//stride[0]]\n\n if return_as_ordered_dict:\n # top left corners\n ordered_grid_image_map = {i:{j:{\"annotation\":[], \"image\":None, \"label\":0} for j, y_pt in enumerate(y_points)} for i, x_pt in enumerate(x_points)}\n for ii, i in enumerate(x_points):\n for jj, j in enumerate(y_points):\n grid_img=img[j:j+grid_dim[1], i:i+grid_dim[1], :]\n ordered_grid_image_map[ii][jj][\"image\"]=grid_img\n ordered_grid_image_map[ii][jj][\"tl\"]=(i,j)\n ordered_grid_image_map[ii][jj][\"br\"]=(i+grid_dim[0],j+grid_dim[1])\n \n is_bg=True\n for box in annotation:\n inter_area, inter_width, inter_height = compute_intersection(\n box_1=(box[\"x\"], box[\"y\"], box[\"x\"]+box[\"width\"], box[\"y\"]+box[\"height\"]),\n box_2=(i, j, i+grid_dim[0], j+grid_dim[1]),\n )\n if inter_area>=min_inter_area and inter_width>=min_inter_width and inter_height>=min_inter_height:\n is_bg=False\n ordered_grid_image_map[ii][jj][\"annotation\"].append(box)\n ordered_grid_image_map[ii][jj][\"label\"]=1 \n return ordered_grid_image_map\n \n else:\n fg_grid_images = []\n bg_grid_images = []\n for i in x_points:\n for j in y_points:\n grid_img=img[j:j+grid_dim[1], i:i+grid_dim[1], :]\n is_bg=True\n for box in annotation:\n inter_area, inter_width, inter_height = compute_intersection(\n box_1=(box[\"x\"], box[\"y\"], box[\"x\"]+box[\"width\"], box[\"y\"]+box[\"height\"]),\n box_2=(i, j, i+grid_dim[0], j+grid_dim[1]),\n )\n if inter_area>=min_inter_area and inter_width>=min_inter_width and inter_height>=min_inter_height:\n is_bg=False\n fg_grid_images.append(grid_img)\n break\n if is_bg:\n bg_grid_images.append(grid_img)\n\n return fg_grid_images, bg_grid_images\n\n\ndef draw_od_grid(img, annotation=None, img_dim=(720,1280,3), grid_dim=(160,160,3), stride=(80,80), ):\n \n x_points = np.linspace(0, img_dim[1], img_dim[1]//stride[1]+1, dtype=np.int32)\n y_points = np.linspace(0, img_dim[0], img_dim[0]//stride[0]+1, dtype=np.int32)\n \n for x_pt in x_points[1:-1:2]:\n img = cv2.line(img, (x_pt, 0), (x_pt, img_dim[0]), (255,200,0), 2)\n \n for y_pt in y_points[1:-1:2]:\n img = cv2.line(img, (0, y_pt), (img_dim[1], y_pt), (255,200,0), 2)\n \n for x_pt in x_points[:-1:2]:\n img = cv2.line(img, (x_pt, 0), (x_pt, img_dim[0]), (0,200,255), 2)\n \n for y_pt in y_points[:-1:2]:\n img = cv2.line(img, (0, y_pt), (img_dim[1], y_pt), (0,200,255), 2)\n\n if annotation is not None:\n for box in annotation:\n img = cv2.rectangle(img, (box[\"x\"], box[\"y\"]), (box[\"x\"]+box[\"width\"], box[\"y\"]+box[\"height\"]), (255, 64, 64), 2)\n \n return img","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:39.204112Z","iopub.execute_input":"2021-12-13T17:07:39.204369Z","iopub.status.idle":"2021-12-13T17:07:39.237396Z","shell.execute_reply.started":"2021-12-13T17:07:39.204340Z","shell.execute_reply":"2021-12-13T17:07:39.236444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.2 DEMO GRID DISPLAY FUNCTIONALITY AND RETURNED GRID IMAGES

\n\n---","metadata":{}},{"cell_type":"code","source":"DEMO_ID = 3954 # 12 objects\nIMG_SHAPE = (720, 1280, 3)\nGRID_IMG_SHAPE = (160, 160, 3)\nSTRIDE = (40, 40)\n\nDEMO_ROW = obj_only_df.iloc[DEMO_ID]\nDEMO_IMG = np.asarray(tf_load_img(DEMO_ROW.img_path))\n\nplt.figure(figsize=(18,10))\nplt.imshow(draw_od_grid(DEMO_IMG, DEMO_ROW.annotations, grid_dim=GRID_IMG_SHAPE, stride=STRIDE))\nplt.show()\n\nDEMO_GRID_IMAGES, _ = get_grid_images(DEMO_IMG, DEMO_ROW.annotations,\n img_dim=IMG_SHAPE, grid_dim=GRID_IMG_SHAPE, stride=STRIDE, \n min_inter_area=400, min_inter_width=20, min_inter_height=20,)\n\n# PLOT\nN_TO_PLOT = 40\nn_rows = int(np.ceil(min(N_TO_PLOT, len(DEMO_GRID_IMAGES))/8))\nplt.figure(figsize=(20,n_rows*3))\n\nfor i, demo_grid_img in enumerate(DEMO_GRID_IMAGES[:N_TO_PLOT]):\n plt.subplot(n_rows, 8, i+1)\n plt.title(f\"DEMO IMAGE #{i+1}\", fontweight=\"bold\")\n plt.imshow(demo_grid_img)\n plt.axis(False)\n\nplt.tight_layout()\nplt.show()\n\n# For later usage\nDEMO_IMG = np.asarray(tf_load_img(DEMO_ROW.img_path))\nFG_DEMO_GRID_IMAGES, BG_DEMO_GRID_IMAGES = get_grid_images(DEMO_IMG, DEMO_ROW.annotations,\n img_dim=IMG_SHAPE, grid_dim=GRID_IMG_SHAPE, \n stride=STRIDE, min_inter_area=400, \n min_inter_width=20, min_inter_height=20,)\nORDERED_DEMO_GRID_IMAGES = get_grid_images(DEMO_IMG, DEMO_ROW.annotations,\n img_dim=IMG_SHAPE, grid_dim=GRID_IMG_SHAPE, stride=STRIDE, \n min_inter_area=400, min_inter_width=20, min_inter_height=20, \n return_as_ordered_dict=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:39.238576Z","iopub.execute_input":"2021-12-13T17:07:39.238789Z","iopub.status.idle":"2021-12-13T17:07:44.443295Z","shell.execute_reply.started":"2021-12-13T17:07:39.238763Z","shell.execute_reply":"2021-12-13T17:07:44.442348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.3 CREATE & SAVE GRID IMAGES

\n\n---\n\nI was doing this with raw images... but now I am creating tfrecords and saving them as a dataset so that I can leverage TPU in the future.","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value, is_list=False):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\ndef _float_feature(value, is_list=False):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value, is_list=False):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\ndef serialize_raw(img, lbl):\n \"\"\"\n Creates a tf.Example message ready to be written to a file from 4 features.\n \"\"\"\n \n img = tf.io.encode_png(tf.constant(img, dtype=tf.uint8))\n \n # Create a dictionary mapping the feature name to the \n # tf.Example-compatible data type.\n feature_dict = {\n 'img': _bytes_feature(img),\n 'lbl': _int64_feature(lbl),\n }\n \n # Create a Features message using tf.train.Example.\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature_dict))\n return example_proto.SerializeToString()\n\ndef img_lbl_generator(subsample_neg_at=15):\n for idx, row in obj_only_df.sample(len(obj_only_df)).iterrows():\n fg_images, bg_images = get_grid_images(cv2.imread(row.img_path)[..., ::-1], row.annotations,\n img_dim=IMG_SHAPE, grid_dim=GRID_IMG_SHAPE, stride=STRIDE, \n min_inter_area=300, min_inter_width=16, min_inter_height=16,)\n for image in fg_images:\n yield image, 1\n \n for image in bg_images[::subsample_neg_at]:\n yield image, 0\n\n\ndef write_tfrecords(n_ex, n_ex_per_rec=250, serialize_fn=serialize_raw, out_dir=\"/kaggle/working/tfrecords\", ds_type=\"all\"):\n \"\"\" Function to create our tfrecords \n \n Args:\n TBD\n \n Returns:\n TBD\n \"\"\"\n \n n_recs = int(np.ceil(n_ex/n_ex_per_rec))\n \n # Make dataframe iterable\n iter_il_gen = img_lbl_generator()\n \n out_dir = os.path.join(out_dir, ds_type)\n \n # Create folder\n if not os.path.isdir(out_dir):\n os.makedirs(out_dir, exist_ok=True)\n \n # Create tfrecords\n for i in tqdm(range(n_recs), total=n_recs):\n print(f\"\\n... Writing {ds_type.title()} TFRecord {i+1} of {n_recs} ...\\n\")\n tfrec_path = os.path.join(out_dir, f\"{ds_type}__{(i+1):02}_{n_recs:02}.tfrec\")\n \n # This makes the tfrecord\n with tf.io.TFRecordWriter(tfrec_path) as writer:\n for ex in tqdm(range(n_ex_per_rec), total=n_ex_per_rec):\n try:\n example = serialize_fn(*next(iter_il_gen))\n writer.write(example)\n except:\n break\n \nTFRECORD_FILES = [x for x in tf.io.gfile.glob(os.path.join(DATASET_DIR, \"**/**/*.tfrec\")) if tf.io.gfile.GFile(x).size()>0]\nif len(TFRECORD_FILES)>10:\n print(\"\\n... LOADING THE TFRECORD FILES ...\\n\")\nelse:\n print(\"\\n... WRITE THE TFRECORD FILES ...\\n\")\n write_tfrecords(n_ex=360000, n_ex_per_rec=1800, serialize_fn=serialize_raw, out_dir=\"/kaggle/working/tfrecords\")","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:44.444841Z","iopub.execute_input":"2021-12-13T17:07:44.445103Z","iopub.status.idle":"2021-12-13T17:07:57.742352Z","shell.execute_reply.started":"2021-12-13T17:07:44.445072Z","shell.execute_reply":"2021-12-13T17:07:57.741455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.4 INSTANTIATE THE EFFICIENTNET MODEL

\n\n---\n\n**NOTE: In some runs of this notebook we don't use EfficientNetV2 due to an error when leveraging TPU. In those cases we swap out the EfficientNetV1 version of the same model.**\n\n
\n πŸ›‘   WARNING:

Each tf.keras application expects a specific kind of input preprocessing. For EfficientNetV2, by default input preprocessing is included as a part of the model (as a Rescaling layer), and thus tf.keras.applications.efficientnet_v2.preprocess_input is actually a pass-through function. In this use case, EfficientNetV2 models expect their inputs to be float tensors of pixels with values in the [0-255] range. At the same time, preprocessing as a part of the model (i.e. Rescaling layer) can be disabled by setting include_preprocessing argument to False. With preprocessing disabled EfficientNetV2 models expect their inputs to be float tensors of pixels with values in the [-1, 1] range
\n
\n\n
","metadata":{}},{"cell_type":"code","source":"def add_model_head(_model, fc_units=512, dropouts=[0.45, 0.45], freeze_backbone=True, freeze_up_to=\"block6h\"):\n \n # Set trainability of model\n if freeze_backbone:\n for _l in _model.layers:\n if _l.name.startswith(freeze_up_to): \n break\n _l.trainable=False\n \n x = tf.keras.layers.Dropout(dropouts[0])(_model.output)\n x = tf.keras.layers.Dense(fc_units, activation=\"relu\")(x)\n x = tf.keras.layers.Dropout(dropouts[1])(x)\n \n _outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n \n return tf.keras.Model(inputs=_model.inputs, outputs=_outputs)\n\nLOAD_MODEL_PATH=\"\"\nwith strategy.scope():\n if LOAD_MODEL_PATH!=\"\":\n model = tf.keras.models.load_model(LOAD_MODEL_PATH, \n options=load_locally, compile=False)\n else:\n # tf.keras.applications.EfficientNetB0\n model = tf.keras.applications.EfficientNetV2B0(\n include_top=False, weights='imagenet', \n input_shape=GRID_IMG_SHAPE, pooling=\"avg\",\n )\n\n # RANGER Optimizer With Default HyperParameters\n radam = tfa.optimizers.RectifiedAdam()\n ranger = tfa.optimizers.Lookahead(radam, sync_period=6, slow_step_size=0.5)\n\n model = add_model_head(model, fc_units=1024)\n model.compile(optimizer=ranger, loss=\"binary_crossentropy\", metrics=[\"acc\", \n tf.keras.metrics.TrueNegatives(name=\"tn\"), \n tf.keras.metrics.TruePositives(name=\"tp\"),\n tf.keras.metrics.FalseNegatives(name=\"fn\"), \n tf.keras.metrics.FalsePositives(name=\"fp\"),\n tfa.metrics.FBetaScore(\n num_classes=1, average=\"micro\", beta=2.0,\n )])\n model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:14:11.599722Z","iopub.execute_input":"2021-12-13T17:14:11.600028Z","iopub.status.idle":"2021-12-13T17:14:12.123618Z","shell.execute_reply.started":"2021-12-13T17:14:11.599993Z","shell.execute_reply":"2021-12-13T17:14:12.122385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.5 CREATE DATASETS FOR TRAINING AND VALIDATION

\n\n---","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data, n_channels=3, reshape_to=GRID_IMG_SHAPE, cast_to=tf.float32):\n image = tf.image.decode_png(image_data, channels=n_channels) \n image = tf.reshape(image, reshape_to) \n return tf.cast(image, cast_to)\n\ndef decode(serialized_example, n_channels=3, reshape_to=GRID_IMG_SHAPE):\n \"\"\" Parses a set of features and label from the given `serialized_example`.\n \n It is used as a map function for `dataset.map`\n\n Args:\n serialized_example (tf.Example): A serialized example\n \n Returns:\n A decoded tf.data.Dataset object representing the tfrecord dataset\n \"\"\"\n \n # Defaults are not specified since both keys are required.\n feature_dict = {\n 'img': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n 'lbl': tf.io.FixedLenFeature(shape=(), dtype=tf.int64),\n }\n \n # Define a parser\n features = tf.io.parse_single_example(serialized_example, features=feature_dict) \n img = decode_image(features['img'], n_channels, reshape_to)\n lbl = tf.cast(features[\"lbl\"], tf.uint8)\n \n return img, lbl\n\ndef tf_load_grid_img(img_path, _shape=GRID_IMG_SHAPE):\n return tf.cast(tf.reshape(tf.image.decode_image(tf.io.read_file(img_path)), _shape), tf.float32)\n\ndef augment_fn(img_batch, lbl_batch):\n \"\"\" super simple augmentation fn (for now) \"\"\"\n \n img_batch = tf.image.random_flip_left_right(img_batch)\n img_batch = tf.image.random_flip_up_down(img_batch)\n \n return img_batch, lbl_batch","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:58.785171Z","iopub.status.idle":"2021-12-13T17:07:58.786116Z","shell.execute_reply.started":"2021-12-13T17:07:58.785810Z","shell.execute_reply":"2021-12-13T17:07:58.785840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.6 CONVERT DATASETS INTO TF.DATA.DATASETS

\n\n---\n\n* **We only perform augmentation on the training dataset**","metadata":{}},{"cell_type":"code","source":"MODEL_CKPT_DIR = \"/kaggle/working/model_wts\"\nif not os.path.isdir(MODEL_CKPT_DIR): os.makedirs(MODEL_CKPT_DIR, exist_ok=True)\n\ncallback_list = []\nwith strategy.scope(): \n callback_list.append(tf.keras.callbacks.ModelCheckpoint(\n filepath=os.path.join(MODEL_CKPT_DIR, \"epoch_{epoch:02d}__fbeta_{val_fbeta_score:.3f}\"),\n save_weights_only=False, monitor='val_fbeta_score', mode='max', save_best_only=True,\n options=save_locally,\n ))\n callback_list.append(tf.keras.callbacks.ReduceLROnPlateau(\n monitor='val_fbeta_score', factor=0.9, patience=2, verbose=1, mode='max',\n ))\n\n\nN_EPOCHS = 100\nBATCH_SIZE = 192*N_REPLICAS\nSHUFFLE_BUFFER = 10*BATCH_SIZE\n \nTRAIN_TFREC_PATHS = TFRECORD_FILES[5:]\nVAL_TFREC_PATHS = TFRECORD_FILES[:5]\n\ntrain_ds = tf.data.TFRecordDataset(TRAIN_TFREC_PATHS, num_parallel_reads=tf.data.AUTOTUNE)\nval_ds = tf.data.TFRecordDataset(VAL_TFREC_PATHS, num_parallel_reads=tf.data.AUTOTUNE)\n\ntrain_ds = train_ds.map(decode)\nval_ds = val_ds.map(decode)\n\nprint(\"\\n\\n\\n... DATASET STATISTICS ...\\n\")\nclass_weighting = {0:0.625, 1:1.125}\nprint(\"WEIGHT MAPPING --> \", class_weighting)\n \ntrain_ds = train_ds.shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE, drop_remainder=True)\ntrain_ds = train_ds.map(augment_fn, num_parallel_calls=tf.data.AUTOTUNE)\ntrain_ds = train_ds.prefetch(tf.data.AUTOTUNE)\n\nval_ds = val_ds.shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE, drop_remainder=True)\nval_ds = val_ds.prefetch(tf.data.AUTOTUNE)\n\nprint(\"\\nTRAIN : \", train_ds)\nprint(\"\\nVAL : \", val_ds)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:58.787331Z","iopub.status.idle":"2021-12-13T17:07:58.787838Z","shell.execute_reply.started":"2021-12-13T17:07:58.787580Z","shell.execute_reply":"2021-12-13T17:07:58.787605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.7 FIT OUR BASELINE MODEL

\n\n---\n\n* We must pass in class weighting (which has been overweighted towards positive samples)\n* We specify the datasets and number of epochs as well\n","metadata":{}},{"cell_type":"code","source":"model.fit(train_ds, validation_data=val_ds, epochs=N_EPOCHS, class_weight=class_weighting, callbacks=callback_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.7 SEE HOW WE DID ON A MINIBATCH OF VALIDATION EXAMPLES

\n\n---","metadata":{}},{"cell_type":"code","source":"MINIBATCH_SIZE = 16\nN_COLS = 4\nN_ROWS = int(np.ceil(MINIBATCH_SIZE/N_COLS))\nN_MINIBATCHES = 1\nfor x_batch, y_batch in val_ds.unbatch().batch(MINIBATCH_SIZE).take(N_MINIBATCHES):\n y_preds = model.predict(x_batch)[:, 0] \n plt.figure(figsize=(20, 5*N_ROWS))\n for i in range(MINIBATCH_SIZE):\n plt.subplot(N_ROWS, N_COLS, i+1)\n plt.imshow(tf.cast(x_batch[i], tf.uint8))\n plt.axis(False)\n plt.title(f\"GROUND TRUTH: {int(np.round(y_batch[i].numpy()))}\\n\" \\\n f\"MODEL PREDICTION: {int(np.round(y_preds[i]))}\", \n fontweight=\"bold\")\n plt.tight_layout()\n plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.8 SEE HOW THIS WORKS ON A WHOLE IMAGE

\n\n---\n\nRemember the demo image (**`DEMO_IMAGE`**)? We will leverage that and the subsequent tiles (**`DEMO_GRID_IMAGES`**) to see how we do on a full image.","metadata":{}},{"cell_type":"code","source":"print(\"\\n... SEE THE FULL IMAGE ...\\n\")\nplt.figure(figsize=(20, 14))\nplt.title(\"Full Image\", fontweight=\"bold\")\nplt.axis(False)\nplt.imshow(DEMO_IMG)\nplt.show()\n\nprint(\"\\n... SEE THE FIRST FIVE TILES ...\\n\")\nplt.figure(figsize=(20, 5))\nfor i in range(5):\n plt.subplot(1,5,i+1)\n plt.imshow(ORDERED_DEMO_GRID_IMAGES[0][i][\"image\"])\n plt.title(f\"Grid Tile #{i+1}\\n\" \\\n f\"[Label = {ORDERED_DEMO_GRID_IMAGES[0][i]['label']}]\\n\" \\\n f\"[TL Corner = {ORDERED_DEMO_GRID_IMAGES[0][i]['tl']}]\\n\" \\\n f\"[BR Corner = {ORDERED_DEMO_GRID_IMAGES[0][i]['br']}]\", fontweight=\"bold\")\n plt.axis(False)\n \nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEMO_IMG = np.asarray(tf_load_img(DEMO_ROW.img_path))\nplt.figure(figsize=(18,10))\nplt.title(\"Original Drawing Of Grid and Boxes\", fontweight=\"bold\")\nplt.imshow(draw_od_grid(DEMO_IMG, DEMO_ROW.annotations, grid_dim=GRID_IMG_SHAPE, stride=STRIDE))\nplt.axis(False)\nplt.show()\n\nDEMO_IMG_VIS = np.asarray(tf_load_img(DEMO_ROW.img_path)).copy()\nCOLOR_LBL_VIS = np.zeros_like(DEMO_IMG_VIS)\n\nfor i, row_image_dicts in ORDERED_DEMO_GRID_IMAGES.items():\n for j, tile_image in row_image_dicts.items():\n if tile_image[\"label\"]:\n COLOR_LBL_VIS[tile_image[\"tl\"][1]:tile_image[\"br\"][1], tile_image[\"tl\"][0]:tile_image[\"br\"][0], 0] = 255\n else:\n COLOR_LBL_VIS[tile_image[\"tl\"][1]:tile_image[\"br\"][1], tile_image[\"tl\"][0]:tile_image[\"br\"][0], 1:] += 15\n\nCOLOR_LBL_VIS[..., 0] = np.clip(COLOR_LBL_VIS[..., 0], 0, 255)\nCOLOR_LBL_VIS[..., 1] = np.clip(COLOR_LBL_VIS[..., 1], 0, 85)\nCOLOR_LBL_VIS[..., 2] = np.clip(COLOR_LBL_VIS[..., 2], 0, 85)\n\nDEMO_IMG_VIS= cv2.addWeighted(DEMO_IMG_VIS, 0.575, COLOR_LBL_VIS, 0.75, 0.0)\n \nplt.figure(figsize=(18, 10))\nplt.title(\"Color Map Showing Ground Truth Label Intensities From Overlap\", fontweight=\"bold\")\nplt.imshow(DEMO_IMG_VIS)\nplt.axis(False)\nplt.show()\n\nDEMO_IMG_VIS= cv2.addWeighted(DEMO_IMG.copy(), 0.55, COLOR_LBL_VIS, 0.825, 0.0)\nplt.figure(figsize=(18, 10))\nplt.title(\"Color Map Showing Ground Truth Label Intensities From Overlap w/ Bounding Box and Grid Overlay\", fontweight=\"bold\")\nplt.imshow(DEMO_IMG_VIS)\nplt.axis(False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_tile_arr = np.array([[tile[\"image\"] for tile in row.values()] for row in ORDERED_DEMO_GRID_IMAGES.values()], dtype=np.float32)\n\n# (cols, rows, tile_h, tile_w, channels)\nprint(\"ORIGINAL: \", ord_tile_arr.shape)\n\nord_tile_arr_v2 = ord_tile_arr.reshape(-1, *GRID_IMG_SHAPE)\n\n# (cols*rows, tile_h, tile_w, channels)\nprint(\"RESHAPED FOR BATCHING: \", ord_tile_arr_v2.shape)\n\nord_tile_arr_v3 = ord_tile_arr_v2.reshape(29, 15, *GRID_IMG_SHAPE)\nprint(\"RESHAPED BACK TO ORIGINAL: \", ord_tile_arr_v3.shape)\n\nassert np.sum(ord_tile_arr_v3!=ord_tile_arr)==0\nprint(\"\\n ASSERTION PASSED... WE CAN RESHAPE AS SHOWN TO GET A BATCH SIZE OF 435 \\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_pred_arr = model.predict(ord_tile_arr_v2).reshape(ord_tile_arr.shape[:2])\n\nprint(ord_pred_arr.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEMO_IMG = np.asarray(tf_load_img(DEMO_ROW.img_path))\nplt.figure(figsize=(18,10))\nplt.title(\"Original Drawing Of Grid and Boxes\", fontweight=\"bold\")\nplt.imshow(draw_od_grid(DEMO_IMG, DEMO_ROW.annotations, grid_dim=GRID_IMG_SHAPE, stride=STRIDE))\nplt.axis(False)\nplt.show()\n\nDEMO_IMG_VIS = np.asarray(tf_load_img(DEMO_ROW.img_path)).copy()\nCOLOR_LBL_VIS = np.zeros_like(DEMO_IMG_VIS)\n\nfor i, row_image_dicts in ORDERED_DEMO_GRID_IMAGES.items():\n for j, tile_image in row_image_dicts.items():\n if ord_pred_arr[i, j]>0.5:\n COLOR_LBL_VIS[tile_image[\"tl\"][1]:tile_image[\"br\"][1], tile_image[\"tl\"][0]:tile_image[\"br\"][0], 0] = 255\n else:\n COLOR_LBL_VIS[tile_image[\"tl\"][1]:tile_image[\"br\"][1], tile_image[\"tl\"][0]:tile_image[\"br\"][0], 1:] += 15\n\nCOLOR_LBL_VIS[..., 0] = np.clip(COLOR_LBL_VIS[..., 0], 0, 255)\nCOLOR_LBL_VIS[..., 1] = np.clip(COLOR_LBL_VIS[..., 1], 0, 85)\nCOLOR_LBL_VIS[..., 2] = np.clip(COLOR_LBL_VIS[..., 2], 0, 85)\n\nDEMO_IMG_VIS= cv2.addWeighted(DEMO_IMG_VIS, 0.575, COLOR_LBL_VIS, 0.75, 0.0)\n \nplt.figure(figsize=(18, 10))\nplt.title(\"Color Map Showing Ground Truth Label Intensities From Overlap\", fontweight=\"bold\")\nplt.imshow(DEMO_IMG_VIS)\nplt.axis(False)\nplt.show()\n\nDEMO_IMG_VIS= cv2.addWeighted(DEMO_IMG.copy(), 0.55, COLOR_LBL_VIS, 0.825, 0.0)\nplt.figure(figsize=(18, 10))\nplt.title(\"Color Map Showing Ground Truth Label Intensities From Overlap w/ Bounding Box and Grid Overlay\", fontweight=\"bold\")\nplt.imshow(DEMO_IMG_VIS)\nplt.axis(False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# def do_grid_inferece(model, img, annotation=None, n_rows=9, n_cols=16, img_w=1280, img_h=720, min_inter_area=250, min_inter_width=16, min_inter_height=16, do_plot=False):\n \n# grid_image_arr = np.zeros((n_rows, n_cols, GRID_IMG_SHAPE[0], GRID_IMG_SHAPE[1], 3), dtype=np.uint8)\n# grid_image_gt = np.zeros((n_rows, n_cols, 1), dtype=np.uint8)\n \n# x_points = np.linspace(0, img_w, n_cols+1, dtype=np.int32)\n# y_points = np.linspace(0, img_h, n_rows+1, dtype=np.int32)\n \n# for i in range(1, len(x_points)):\n# for j in range(1, len(y_points)):\n# grid_image_arr[j, i]=img[y_points[j-1]:y_points[j], x_points[i-1]:x_points[i], :]\n# bg_grid_images.append(grid_img)\n# for box in annotation:\n# inter_area, inter_width, inter_height = compute_intersection(\n# box_1=(box[\"x\"], box[\"y\"], box[\"x\"]+box[\"width\"], box[\"y\"]+box[\"height\"]),\n# box_2=(x_points[i-1], y_points[j-1], x_points[i], x_points[j]),\n# )\n# if inter_area>=min_inter_area and inter_width>=min_inter_width and inter_height>=min_inter_height:\n# fg_grid_images.append(grid_img)\n# if do_plot:\n# plt.imshow(grid_img)\n# plt.title(f\"{i-1}, {j-1}, {i}, {j} - {inter_area} - {inter_width} - {inter_height}\")\n# plt.show()\n \n# return fg_grid_images, bg_grid_images","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}