{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 0 . IMPORT AND INSTALL MODULES","metadata":{}},{"cell_type":"code","source":"DATASET_PATH = '/kaggle/input/tensorflow-great-barrier-reef/train_images/'\nCKPT_PATH = '../input/cots-slice256-overlap02-ratio03-fold0/best.pt'\n\n#CUSTOM_YOLO5_CLASS const (we can execute using standard SAHI predict or custom one implemented in this notebook). \nCUSTOM_YOLO5_CLASS = True\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport sys\nimport cv2\nimport torch\nfrom PIL import Image as Img\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2022-01-27T16:15:06.362040Z","iopub.execute_input":"2022-01-27T16:15:06.362356Z","iopub.status.idle":"2022-01-27T16:15:07.929541Z","shell.execute_reply.started":"2022-01-27T16:15:06.362270Z","shell.execute_reply":"2022-01-27T16:15:07.928805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 0.A - CUTOM MAGIC FUNCTION\n\nI implemented magic function to skip execution of notebook cell - it depends on CUSTOM_YOLO5_CLASS const (we can execute using standard SAHI predict or custom one). ","metadata":{}},{"cell_type":"code","source":"from IPython.core.magic import (register_line_cell_magic)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T16:15:07.931083Z","iopub.execute_input":"2022-01-27T16:15:07.931307Z","iopub.status.idle":"2022-01-27T16:15:07.934823Z","shell.execute_reply.started":"2022-01-27T16:15:07.931275Z","shell.execute_reply":"2022-01-27T16:15:07.934191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@register_line_cell_magic\ndef custom_yolo5(line, cell=None):\n    if eval(line):\n        print(\"Cell skipped - not executed\")\n        return\n    get_ipython().ex(cell)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T16:15:07.936592Z","iopub.execute_input":"2022-01-27T16:15:07.936976Z","iopub.status.idle":"2022-01-27T16:15:07.946530Z","shell.execute_reply.started":"2022-01-27T16:15:07.936942Z","shell.execute_reply":"2022-01-27T16:15:07.945824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 0.B - INSTALL MODULES","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/input/sahihub/s-lib\n!pip -q install ./fire-0.4.0/fire-0.4.0.tar -f ./ --no-index\n!pip -q install terminaltables-3.1.10-py2.py3-none-any.whl -f ./ --no-index\n!pip -q install sahi-0.8.22-py3-none-any.whl -f ./ --no-index\n!pip -q install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n!pip -q install yolov5-6.0.6-py36.py37.py38-none-any.whl -f ./ --no-index\n!pip -q install yolo5-0.0.1-py36.py37.py38-none-any.whl -f ./ --no-index\n\n!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/sahihub/Arial.ttf /root/.config/Ultralytics/\n\n%cd /kaggle/working","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-27T16:15:07.947883Z","iopub.execute_input":"2022-01-27T16:15:07.948158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. IMPORT SAHI MODULES","metadata":{}},{"cell_type":"code","source":"from sahi.model import Yolov5DetectionModel\nfrom sahi.utils.cv import read_image\nfrom sahi.predict import get_prediction, get_sliced_prediction, predict\nfrom IPython.display import Image\nfrom sahi.utils.yolov5 import (\n    download_yolov5s6_model,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. HELPER FUNCTIONS","metadata":{}},{"cell_type":"code","source":"def show_prediction(img, bboxes, scores, show = True):\n    colors = [(0, 0, 255)]\n\n    obj_names = [\"s\"]\n\n    for box, score in zip(bboxes, scores):\n        cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[0] + box[2]), int(box[1] + box[3])), (255,0,0), 2)\n        cv2.putText(img, f'{score}', (int(box[0]), int(box[1])-3), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,0,0), 1, cv2.LINE_AA)\n    \n    if show:\n        img = Img.fromarray(img).resize((1280, 720))\n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. MODELS","metadata":{}},{"cell_type":"markdown","source":"### A. YOLOv5 - get_sliced_prediction\n\n* **image**: str or np.ndarray - Location of image or numpy image matrix to slice\n* **detection_model**: model.DetectionModel\n* **image_size**: int: Input image size for each inference (image is scaled by preserving asp. rat.).\n* **slice_height**: int: Height of each slice.  Defaults to ``512``.\n* **slice_width**: int: Width of each slice.  Defaults to ``512``.\n* **overlap_height_ratio**: float: Fractional overlap in height of each window (e.g. an overlap of 0.2 for a window of size 512 yields an overlap of 102 pixels). Default to ``0.2``.\n* **overlap_width_ratio**: float: Fractional overlap in width of each window (e.g. an overlap of 0.2 for a window of size 512 yields an overlap of 102 pixels). Default to ``0.2``.\n* **perform_standard_pred**: bool: Perform a standard prediction on top of sliced predictions to increase large object detection accuracy. Default: True.\n* **postprocess_type**: str: Type of the postprocess to be used after sliced inference while merging/eliminating predictions. Options are 'NMM', 'GRREDYNMM' or 'NMS'. Default is 'GRREDYNMM'.\n* **postprocess_match_metric**: str: Metric to be used during object prediction matching after sliced prediction. 'IOU' for intersection over union, 'IOS' for intersection over smaller area.\n* **postprocess_match_threshold**: float: Sliced predictions having higher iou than postprocess_match_threshold will be postprocessed after sliced prediction.\n* **postprocess_class_agnostic**: bool: If True, postprocess will ignore category ids.\n* **verbose**: int: 0: no print, 1: print number of slices (default), 2: print number of slices and slice/prediction durations","metadata":{}},{"cell_type":"markdown","source":"### A1. CUSTOM Yolo5 PREDICTION CLASS\nThis is not obligatory but I decided to write this to have more control over prediction.\nIdea provided by Dewei Chen @dwchen in this discussion: https://www.kaggle.com/c/tensorflow-great-barrier-reef/discussion/302761","metadata":{}},{"cell_type":"code","source":"from sahi.prediction import ObjectPrediction\nfrom sahi.model import DetectionModel\nfrom typing import Dict, List, Optional, Union\nfrom sahi.utils.compatibility import fix_full_shape_list, fix_shift_amount_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class COTSYolov5DetectionModel(DetectionModel):\n\n    \n    def load_model(self):\n        model = torch.hub.load('/kaggle/input/yolov5-lib-ds', \n                               'custom', \n                               path=self.model_path,\n                               source='local',\n                               force_reload=True)\n        \n        model.conf = self.confidence_threshold\n        self.model = model\n        \n        if not self.category_mapping:\n            category_mapping = {str(ind): category_name for ind, category_name in enumerate(self.category_names)}\n            self.category_mapping = category_mapping\n\n    def perform_inference(self, image: np.ndarray, image_size: int = None):\n        if image_size is not None:\n            warnings.warn(\"Set 'image_size' at DetectionModel init.\", DeprecationWarning)\n            prediction_result = self.model(image, size=image_size, augment=True)\n            if debug_mode:\n                display(Img.fromarray(image).resize((320, 200)))\n        elif self.image_size is not None:\n            prediction_result = self.model(image, size=self.image_size, augment=True)\n        else:\n            prediction_result = self.model(image)\n\n        self._original_predictions = prediction_result\n\n    @property\n    def num_categories(self):\n        \"\"\"\n        Returns number of categories\n        \"\"\"\n        return len(self.model.names)\n\n    @property\n    def has_mask(self):\n        \"\"\"\n        Returns if model output contains segmentation mask\n        \"\"\"\n        has_mask = self.model.with_mask\n        return has_mask\n\n    @property\n    def category_names(self):\n        return self.model.names\n\n    def _create_object_prediction_list_from_original_predictions(\n        self,\n        shift_amount_list: Optional[List[List[int]]] = [[0, 0]],\n        full_shape_list: Optional[List[List[int]]] = None,):\n\n        original_predictions = self._original_predictions\n        shift_amount_list = fix_shift_amount_list(shift_amount_list)\n        full_shape_list = fix_full_shape_list(full_shape_list)\n\n        # handle all predictions\n        object_prediction_list_per_image = []\n        for image_ind, image_predictions_in_xyxy_format in enumerate(original_predictions.xyxy):\n            shift_amount = shift_amount_list[image_ind]\n            full_shape = None if full_shape_list is None else full_shape_list[image_ind]\n            object_prediction_list = []\n\n            # process predictions\n            for prediction in image_predictions_in_xyxy_format.cpu().detach().numpy():\n                x1 = int(prediction[0])\n                y1 = int(prediction[1])\n                x2 = int(prediction[2])\n                y2 = int(prediction[3])\n                bbox = [x1, y1, x2, y2]\n                score = prediction[4]\n                category_id = int(prediction[5])\n                category_name = self.category_mapping[str(category_id)]\n\n                # ignore invalid predictions\n                if bbox[0] > bbox[2] or bbox[1] > bbox[3] or bbox[0] < 0 or bbox[1] < 0 or bbox[2] < 0 or bbox[3] < 0:\n                    logger.warning(f\"ignoring invalid prediction with bbox: {bbox}\")\n                    continue\n                if full_shape is not None and (\n                    bbox[1] > full_shape[0]\n                    or bbox[3] > full_shape[0]\n                    or bbox[0] > full_shape[1]\n                    or bbox[2] > full_shape[1]\n                ):\n                    logger.warning(f\"ignoring invalid prediction with bbox: {bbox}\")\n                    continue\n\n                object_prediction = ObjectPrediction(\n                    bbox=bbox,\n                    category_id=category_id,\n                    score=score,\n                    bool_mask=None,\n                    category_name=category_name,\n                    shift_amount=shift_amount,\n                    full_shape=full_shape,\n                )\n                object_prediction_list.append(object_prediction)\n            object_prediction_list_per_image.append(object_prediction_list)\n\n        self._object_prediction_list_per_image = object_prediction_list_per_image ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### A2. HELPER FUNCTION","metadata":{}},{"cell_type":"code","source":"def predict(img, model, sw, sh, ohr, owr, pmt, img_size, verb):\n    result = get_sliced_prediction(img,\n                                   model,\n                                   slice_width = sw,\n                                   slice_height = sh,\n                                   overlap_height_ratio = ohr,\n                                   overlap_width_ratio = owr,\n                                   postprocess_match_threshold = pmt,\n                                   image_size = img_size,\n                                   verbose = verb,\n                                   perform_standard_pred = True)\n    \n    \n    bboxes = []\n    scores = []\n    result_len = result.to_coco_annotations()\n    for pred in result_len:\n        bboxes.append(pred['bbox'])\n        scores.append(pred['score'])\n    \n    return bboxes, scores ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detection_model = COTSYolov5DetectionModel(\n   model_path = CKPT_PATH,\n   confidence_threshold = 0.2,\n   device=\"cuda\",\n)\n\ndetection_model.model.iou = 0.45","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%custom_yolo5 $CUSTOM_YOLO5_CLASS\n\ndetection_model = Yolov5DetectionModel(\n   model_path = CKPT_PATH,\n   confidence_threshold = 0.2,\n   device=\"cuda\",\n)\n\ndetection_model.model.iou = 0.45","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### A3. PREDICTION","metadata":{}},{"cell_type":"code","source":"# I intruduced DEBUG_MODE so you can understand how SAHI make a slices for predition. \n# If True then it shows slices and ... oryginal image when:\n# perform_standard_pred is set to True; if False then only slices are presented) \n\ndebug_mode = True #show slices and oryginal image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir = f'{DATASET_PATH}'\n\nimgs = [dir + f for f in ('video_2/5748.jpg', 'video_2/5748.jpg', 'video_2/5772.jpg')]\n\nfor img in(imgs):\n    im = cv2.imread(img)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    if debug_mode:\n        print(\"\\n>>>> DEBUG MODE - SHOW SLICES AND FULL FRAME <<<<\")\n    bboxes, scores = predict(img, detection_model, 256, 256, 0.2, 0.2, 0.45, 640, 2) #\n    display(show_prediction(im, bboxes, scores))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. MAKE VIDEO FROM PREDS","metadata":{}},{"cell_type":"code","source":"import ast\nimport os\nimport pandas as pd\nimport subprocess\n\nfrom ast import literal_eval\nfrom tqdm.auto import tqdm\n\nfrom IPython.display import HTML\nfrom base64 import b64encode","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_path(row):\n    return f\"{DATASET_PATH}/video_{row.video_id}/{row.video_frame}.jpg\"\n\ndf['path'] = df.apply(lambda row: add_path(row), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(video_id, video_frame, image_dir):\n    assert os.path.exists(image_dir), f'{image_dir} does not exist.'\n    img = cv2.imread(image_dir)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\n\n\ndef decode_annotations(annotaitons_str):\n    return literal_eval(annotaitons_str)\n\ndef load_image_with_annotations(img, annotaitons_str):\n    annotations = decode_annotations(annotaitons_str)\n    if len(annotations) > 0:\n        for ann in annotations:\n            cv2.rectangle(img, (ann['x'], ann['y']),\n                (ann['x'] + ann['width'], ann['y'] + ann['height']),\n                (0, 255, 255), thickness=2,)\n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.query('video_id == 2 and sequence == 22643 and video_frame > 5700 ').head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## This code I found in: https://www.kaggle.com/bamps53/create-annotated-video Thank you for sharing.\n\ndef make_sahi_video(df, video_id, sequence_id, out_dir):\n    fps = 15 \n    width = 1280\n    height = 720\n\n    save_path = f'{out_dir}/video-{video_id}.mp4'\n    tmp_path =  f'{out_dir}/tmp-video-{video_id}.mp4'\n    output_video = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*\"MP4V\"), fps, (width, height))\n    \n    # I just generate ony part of video\n    video_df = df.query('video_id == @video_id and sequence == @sequence_id and video_frame > 5700 and video_frame < 6000')\n    for _, row in tqdm(video_df.iterrows(), total=len(video_df)):\n        video_id = row.video_id\n        video_frame = row.video_frame\n        annotations_str = row.annotations\n        img_file = row.path\n        img = load_image(video_id, video_frame, img_file)\n        bboxes, scores = predict(img, detection_model, 256, 256, 0.2, 0.2, 0.45, 640, 0)\n        img = show_prediction(img, bboxes, scores, False)\n        img = load_image_with_annotations(img, annotations_str)\n        cv2.putText(img, f'{video_id}-{video_frame}', (10,70), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,0,0), 1, cv2.LINE_AA)\n        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n        output_video.write(img)\n\n    \n    output_video.release()\n\n    if os.path.exists(save_path):\n        os.remove(save_path)\n    subprocess.run(\n        [\"ffmpeg\", \"-i\", tmp_path, \"-crf\", \"18\", \"-preset\", \"veryfast\", \"-vcodec\", \"libx264\", save_path]\n    )\n    os.remove(tmp_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To speed up I just generate ony part of video\n# This prediction is for sure overfitted but it is for demo only (I can see it on prediction)\n\ndebug_mode = False\n\nmake_sahi_video(df, 2, 22643, '/kaggle/working/')","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def play(filename):\n    html = ''\n    video = open(filename,'rb').read()\n    src = 'data:video/mp4;base64,' + b64encode(video).decode()\n    html += '<video width=800 controls autoplay loop><source src=\"%s\" type=\"video/mp4\"></video>' % src \n    return HTML(html)\n\nplay('/kaggle/working/video-2.mp4')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. SUBMIT","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug_mode = False\n\nfor (image_np, sample_prediction_df) in iter_test:\n    \n    bboxes, scores = predict(image_np, detection_model, 256, 256, 0.2, 0.2, 0.45, 640, 0)\n    \n    predictions = []\n    detects = []\n    \n    for i in range(len(bboxes)):\n        box = bboxes[i]\n        score = scores[i]\n        x_min = int(box[0])\n        y_min = int(box[1])\n        x_max = int(box[0]) + int(box[2])\n        y_max = int(box[1]) + int(box[3])\n        \n        bbox_width = x_max - x_min\n        bbox_height = y_max - y_min\n        detects.append([x_min, y_min, x_max, y_max, score])\n        \n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n    \n    \n    prediction_str = ' '.join(predictions)\n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}