{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Yolov5 is all you need - Make compact videos\n\n* This notebook is based on https://www.kaggle.com/steamedsheep/yolov5-is-all-you-need from steamedsheep (version 2)\n\n# Why this notebook?\n\n* Besides measuring the usual metrics (mAP, F1, F2, P, R, etc), creating compact videos is essential for understanding how your model is performing.\n* Each model has its strengths and weaknesses. It is difficult to appreciate these by just looking at the metrics; you need to see it in action, comparing against ground truth\n* As this competition is based on object detection of video frames, watching the video of the inference helps us understand model weaknesses, including:\n  * Whether the model can detect large or small objects, or both\n  * Which COTS are consistently not detected, and why\n  * Why are some COTS detected on some video frames and not others (i.e. the importance of object tracking algorithms like DeepSort and Norfair)\n  \nOnly from understanding your model can you improve it by adding augmentation, modifying train / inference resolution, etc\n  \n# What's new in this notebook?\n\n* Plotting videos with both ground-truth and inference bounding boxes in the same video (ground truth in green, inference in red)\n* AVI to mp4 compression for easy download ","metadata":{}},{"cell_type":"code","source":"# CONTROL PANEL (change your settings here instead)\n\n# The pato to your yolov5 model\nMODEL_PATH = '../input/reef-baseline-fold12/l6_3600_uflip_vm5_f12_up/f1/best.pt'\n\n# Confidence cutoff\nINFER_CONF = 0.15\n\n# Inference size\nINFER_SIZE = 7200\n\n# Whether to use yolov5 TTA\nINFER_TTA = True","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:00:21.264535Z","iopub.execute_input":"2022-01-18T02:00:21.264791Z","iopub.status.idle":"2022-01-18T02:00:21.268536Z","shell.execute_reply.started":"2022-01-18T02:00:21.264764Z","shell.execute_reply":"2022-01-18T02:00:21.267812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nfrom tqdm import tqdm\nimport sys\n\nsys.path.append('../input/tensorflow-great-barrier-reef')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-18T01:32:44.015764Z","iopub.execute_input":"2022-01-18T01:32:44.016232Z","iopub.status.idle":"2022-01-18T01:32:45.357996Z","shell.execute_reply.started":"2022-01-18T01:32:44.016156Z","shell.execute_reply":"2022-01-18T01:32:45.357270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\n# import torch\nimport importlib\nimport cv2 \n# import pandas as pd\n# import numpy as np\n\nimport ast\nimport shutil\nimport sys\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\nfrom PIL import Image\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:32:47.121680Z","iopub.execute_input":"2022-01-18T01:32:47.121976Z","iopub.status.idle":"2022-01-18T01:32:47.327427Z","shell.execute_reply.started":"2022-01-18T01:32:47.121943Z","shell.execute_reply":"2022-01-18T01:32:47.326328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:32:49.094445Z","iopub.execute_input":"2022-01-18T01:32:49.094919Z","iopub.status.idle":"2022-01-18T01:32:50.426288Z","shell.execute_reply.started":"2022-01-18T01:32:49.094881Z","shell.execute_reply":"2022-01-18T01:32:50.425320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define the Model Here","metadata":{}},{"cell_type":"code","source":"model = torch.hub.load('../input/yolov5-lib-ds', \n                       'custom', \n                       path=MODEL_PATH,\n                       source='local',\n                       force_reload=True)  # local repo\nmodel.conf = 0.01","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:34:36.244061Z","iopub.execute_input":"2022-01-18T01:34:36.244356Z","iopub.status.idle":"2022-01-18T01:34:42.077256Z","shell.execute_reply.started":"2022-01-18T01:34:36.244304Z","shell.execute_reply":"2022-01-18T01:34:42.076500Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Videos","metadata":{}},{"cell_type":"markdown","source":"## Install ffmpeg for Kaggle\n\n* I'm not sure if this is necessary?","metadata":{}},{"cell_type":"code","source":"# Install ffmpeg for video compression\n%cd /kaggle/working\n\n! tar xvf ../input/ffmpeg-static-build/ffmpeg-git-amd64-static.tar.xz\n\nimport subprocess\n\nFFMPEG_BIN = \"/kaggle/working/ffmpeg-git-20191209-amd64-static/ffmpeg\"","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:32:53.541779Z","iopub.execute_input":"2022-01-18T01:32:53.542469Z","iopub.status.idle":"2022-01-18T01:32:57.723740Z","shell.execute_reply.started":"2022-01-18T01:32:53.542429Z","shell.execute_reply":"2022-01-18T01:32:57.722853Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utility Functions","metadata":{}},{"cell_type":"code","source":"# Modified from https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507\n\ndef draw_yolox_predictions(img, bboxes, scores, bbclasses, classes_dict, boxcolor = (0,0,255)):\n    outimg = img.copy()\n    for i in range(len(bboxes)):\n        box = bboxes[i]\n        cls_id = int(bbclasses[i])\n        score = scores[i]\n        x0 = int(box[0])\n        y0 = int(box[1])\n        x1 = x0 + int(box[2])\n        y1 = y0 + int(box[3])\n\n        cv2.rectangle(outimg, (x0, y0), (x1, y1), boxcolor, 2)\n        cv2.putText(outimg, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, boxcolor, thickness = 1)\n    return outimg","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:33:00.519908Z","iopub.execute_input":"2022-01-18T01:33:00.520185Z","iopub.status.idle":"2022-01-18T01:33:00.527410Z","shell.execute_reply.started":"2022-01-18T01:33:00.520152Z","shell.execute_reply":"2022-01-18T01:33:00.526748Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing the Training Dataset and selecting videos\n\n* Note that it is important to also include videos with NO COTS, so you can understand where false positive detections may arise","metadata":{}},{"cell_type":"code","source":"# Modified from https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507\n\n%cd /kaggle/working\n\nfrom sklearn.model_selection import GroupKFold\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\n\ndf = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\n\n\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df\n\n#Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\ndf_train = df_train.progress_apply(get_path, axis=1)\n\nkf = GroupKFold(n_splits = 5) \ndf_train = df_train.reset_index(drop=True)\ndf_train['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n    df_train.loc[val_idx, 'fold'] = fold\n\ndf_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:33:02.591618Z","iopub.execute_input":"2022-01-18T01:33:02.594036Z","iopub.status.idle":"2022-01-18T01:33:19.094694Z","shell.execute_reply.started":"2022-01-18T01:33:02.593991Z","shell.execute_reply":"2022-01-18T01:33:19.093875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select the validation dataset (fold-4 in my private models, probably not the same as steamedsheep's YoloV5 model)\n\n* Take care not to shuffle the videos! Keep them in the original order by not sorting the data frame","metadata":{}},{"cell_type":"code","source":"df_test = df_train[df_train.fold == 4]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:33:21.517326Z","iopub.execute_input":"2022-01-18T01:33:21.518011Z","iopub.status.idle":"2022-01-18T01:33:21.525752Z","shell.execute_reply.started":"2022-01-18T01:33:21.517974Z","shell.execute_reply":"2022-01-18T01:33:21.524985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define image paths and ground truth bounding boxes","metadata":{}},{"cell_type":"code","source":"image_paths = df_test.image_path.tolist()\ngt = df_test.bboxes.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:33:37.879004Z","iopub.execute_input":"2022-01-18T01:33:37.879268Z","iopub.status.idle":"2022-01-18T01:33:37.884016Z","shell.execute_reply.started":"2022-01-18T01:33:37.879239Z","shell.execute_reply":"2022-01-18T01:33:37.883344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference on Validation Videos and Recording the Video","metadata":{}},{"cell_type":"markdown","source":"### Test inference","metadata":{}},{"cell_type":"code","source":"i = 1350\nTEST_IMAGE_PATH = image_paths[i]\nimg = cv2.imread(TEST_IMAGE_PATH)\n\nr = model(img, size=3600, augment=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:50:04.700641Z","iopub.execute_input":"2022-01-18T01:50:04.700928Z","iopub.status.idle":"2022-01-18T01:50:04.926783Z","shell.execute_reply.started":"2022-01-18T01:50:04.700870Z","shell.execute_reply":"2022-01-18T01:50:04.926089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r.pandas().xyxy[0]\nres = np.array(r.pandas().xyxy[0])\nbboxes = res[..., 0:4]\nbboxes[:,2:4] -= bboxes[:,0:2]\nbboxes","metadata":{"execution":{"iopub.status.busy":"2022-01-18T01:51:00.216987Z","iopub.execute_input":"2022-01-18T01:51:00.217243Z","iopub.status.idle":"2022-01-18T01:51:00.232700Z","shell.execute_reply.started":"2022-01-18T01:51:00.217213Z","shell.execute_reply":"2022-01-18T01:51:00.231969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working\n\nvideo_size = (1280, 720)\nCOCO_CLASSES = (\"starfish\")\n\nout1 = cv2.VideoWriter('Video.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, video_size)\n\nfor i in tqdm(range(1250, 2200)):\n    # Test a small video first. For the full video, substitute \"tqdm(range(start, finish))\" with \"tqdm(range(len(image_paths)))\"\n    TEST_IMAGE_PATH = image_paths[i]\n    img = cv2.imread(TEST_IMAGE_PATH)\n\n    # Draw GT\n    out_image0 = draw_yolox_predictions(img, gt[i], [1.0] * len(gt[i]), [0] * len(gt[i]), COCO_CLASSES, (0,255,0)) # Green ground truth box\n\n    # Insert your inference code here:\n    r = model(img, size=INFER_SIZE, augment=INFER_TTA)\n    \n    bboxes = []\n    bbclasses = []\n    scores = []\n    if r.pandas().xyxy[0].shape[0] > 0:\n        res = np.array(r.pandas().xyxy[0])\n        for r in res:\n            # Filter by INFER_CONF\n            if r[4] > INFER_CONF:\n                bb = r[0:4]\n                bb[2:4] -= bb[0:2] # Convert to xywh format\n                bboxes.append(bb)\n                scores.append(r[4])\n                bbclasses.append(r[5])\n        \n    out_image = draw_yolox_predictions(out_image0, bboxes, scores, bbclasses, COCO_CLASSES, (0,0,255)) # Red ground truth box (as image is BGR, not RGB)\n    out1.write(out_image)\n    \n# Finalize AVI\nout1.release()","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:00:27.051568Z","iopub.execute_input":"2022-01-18T02:00:27.052076Z","iopub.status.idle":"2022-01-18T02:01:37.020706Z","shell.execute_reply.started":"2022-01-18T02:00:27.052037Z","shell.execute_reply":"2022-01-18T02:01:37.019989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert AVI to compressed mp4 for more convenient downloading\n\n* The created AVI is a large file. Compress the video file so you can download it and watch it locally","metadata":{}},{"cell_type":"code","source":"# I'm open to suggestions as to what compression settings might be better?\nAVI2MP4 = \"-ac 2 -b:v 2000k -c:a aac -c:v libx264 -b:a 160k -vprofile high -bf 0 -strict experimental -f mp4\"\n\ncommand = f\"{FFMPEG_BIN} -i Video.avi {AVI2MP4} Video.mp4\"\nsubprocess.call(command, shell=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-18T02:03:19.584501Z","iopub.execute_input":"2022-01-18T02:03:19.585067Z","iopub.status.idle":"2022-01-18T02:03:38.434007Z","shell.execute_reply.started":"2022-01-18T02:03:19.585028Z","shell.execute_reply":"2022-01-18T02:03:38.433242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show off your video!\n\n* Green boxes are ground truth\n* Red boxes are model inference","metadata":{}},{"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\ndef play(filename):\n    html = ''\n    video = open(filename,'rb').read()\n    src = 'data:video/mp4;base64,' + b64encode(video).decode()\n    html += '<video width=800 controls autoplay loop><source src=\"%s\" type=\"video/mp4\"></video>' % src \n    return HTML(html)\n\nplay('Video.mp4')","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:03:45.175771Z","iopub.execute_input":"2022-01-18T02:03:45.176212Z","iopub.status.idle":"2022-01-18T02:03:45.298008Z","shell.execute_reply.started":"2022-01-18T02:03:45.176179Z","shell.execute_reply":"2022-01-18T02:03:45.297221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleanup\n\n!rm *.avi\n!rm -r ffmpeg*","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition Inference (left unchanged)","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-01-12T06:17:41.194729Z","iopub.execute_input":"2022-01-12T06:17:41.195011Z","iopub.status.idle":"2022-01-12T06:17:41.220687Z","shell.execute_reply.started":"2022-01-12T06:17:41.194973Z","shell.execute_reply":"2022-01-12T06:17:41.21992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    anno = ''\n    r = model(img, size=INFER_SIZE, augment=INFER_TTA)\n    if r.pandas().xyxy[0].shape[0] == 0:\n        anno = ''\n    else:\n        for idx, row in r.pandas().xyxy[0].iterrows():\n            if row.confidence > INFER_CONF:\n                anno += '{} {} {} {} {} '.format(row.confidence, int(row.xmin), int(row.ymin), int(row.xmax-row.xmin), int(row.ymax-row.ymin))\n#                 pred.append([row.confidence, row.xmin, row.ymin, row.xmax-row.xmin, row.ymax-row.ymin])\n    pred_df['annotations'] = anno.strip(' ')\n    env.predict(pred_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T06:18:41.248178Z","iopub.execute_input":"2022-01-12T06:18:41.249643Z","iopub.status.idle":"2022-01-12T06:18:46.826251Z","shell.execute_reply.started":"2022-01-12T06:18:41.249595Z","shell.execute_reply":"2022-01-12T06:18:46.825522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Credits:\n\nhttps://www.kaggle.com/steamedsheep/yolov5-is-all-you-need for example model inference\n\nhttps://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507 for utility functions\n\nhttps://www.kaggle.com/bamps53/create-annotated-video for inspiration on video creation and compression","metadata":{}}]}