{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Detectron2 -Keypoint Detection Model-","metadata":{}},{"cell_type":"markdown","source":"## Import dependencies","metadata":{}},{"cell_type":"code","source":"!python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:31:46.940505Z","iopub.execute_input":"2023-02-09T09:31:46.940951Z","iopub.status.idle":"2023-02-09T09:34:53.495865Z","shell.execute_reply.started":"2023-02-09T09:31:46.940864Z","shell.execute_reply":"2023-02-09T09:34:53.494645Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport matplotlib.pyplot as plt\n\nimport torch\nimport cv2\nimport torch\nfrom glob import glob\nfrom tqdm import tqdm\n\n# Detectron2\nimport detectron2\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\n\nimport IPython\nfrom IPython.display import Video, display\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:34:53.498489Z","iopub.execute_input":"2023-02-09T09:34:53.498945Z","iopub.status.idle":"2023-02-09T09:34:54.744408Z","shell.execute_reply.started":"2023-02-09T09:34:53.498894Z","shell.execute_reply":"2023-02-09T09:34:54.743388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def play_video(video_path: str):\n    frac = 0.65 # scaling factor for display \n    display(\n        Video(data=video_path, embed=True, height=int(720*frac), width=int(1280*frac))\n    )","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:34:54.745981Z","iopub.execute_input":"2023-02-09T09:34:54.746657Z","iopub.status.idle":"2023-02-09T09:34:54.753695Z","shell.execute_reply.started":"2023-02-09T09:34:54.746616Z","shell.execute_reply":"2023-02-09T09:34:54.752057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\n\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\"))\n\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7\n\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\")\n\ncfg.MODEL.DEVICE = \"cuda:0\"","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:34:54.756550Z","iopub.execute_input":"2023-02-09T09:34:54.756953Z","iopub.status.idle":"2023-02-09T09:34:54.784046Z","shell.execute_reply.started":"2023-02-09T09:34:54.756915Z","shell.execute_reply":"2023-02-09T09:34:54.783093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# video -> frame\n!mkdir -p frames\n!ffmpeg -i /kaggle/input/nfl-player-contact-detection/train/58173_003606_Endzone.mp4 -q:v 2 -f image2 /kaggle/working/frames/frame_%04d.jpg -hide_banner -loglevel error","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:38:14.655789Z","iopub.execute_input":"2023-02-09T09:38:14.656951Z","iopub.status.idle":"2023-02-09T09:38:32.065724Z","shell.execute_reply.started":"2023-02-09T09:38:14.656892Z","shell.execute_reply":"2023-02-09T09:38:32.064345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" frames_paths = sorted(glob('/kaggle/working/frames/*'))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:38:49.299800Z","iopub.execute_input":"2023-02-09T09:38:49.300194Z","iopub.status.idle":"2023-02-09T09:38:49.310710Z","shell.execute_reply.started":"2023-02-09T09:38:49.300161Z","shell.execute_reply":"2023-02-09T09:38:49.309666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:38:53.593783Z","iopub.execute_input":"2023-02-09T09:38:53.594410Z","iopub.status.idle":"2023-02-09T09:39:00.605641Z","shell.execute_reply.started":"2023-02-09T09:38:53.594373Z","shell.execute_reply":"2023-02-09T09:39:00.604671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example","metadata":{}},{"cell_type":"code","source":"frame = cv2.imread(frames_paths[0])\nframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\noutputs  = predictor(frame)\nv = Visualizer(frame[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\nv = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n\nplt.figure(figsize=(15,15))\nplt.imshow(v.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:39:17.199611Z","iopub.execute_input":"2023-02-09T09:39:17.200008Z","iopub.status.idle":"2023-02-09T09:39:24.558713Z","shell.execute_reply.started":"2023-02-09T09:39:17.199974Z","shell.execute_reply":"2023-02-09T09:39:24.557892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction (Whole video)","metadata":{}},{"cell_type":"code","source":"FPS = 15\nheight, width, c = v.get_image()[:, :, ::-1].shape\nframes_paths = frames_paths[::4]","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:39:41.903880Z","iopub.execute_input":"2023-02-09T09:39:41.904267Z","iopub.status.idle":"2023-02-09T09:39:42.034373Z","shell.execute_reply.started":"2023-02-09T09:39:41.904235Z","shell.execute_reply":"2023-02-09T09:39:42.033348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_path = \"/kaggle/working/output2.mp4\"\n\nout = cv2.VideoWriter(\n    output_path, \n    cv2.VideoWriter_fourcc(*'VP90'), \n    FPS, \n    (width, height)\n)\n\n\nfor paths in tqdm(frames_paths):\n    \n    frame = cv2.imread(paths)\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    outputs  = predictor(frame)\n    v = Visualizer(frame[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    \n    out.write(v.get_image()[:, :, ::-1])\n    \nout.release()","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:39:50.657653Z","iopub.execute_input":"2023-02-09T09:39:50.658469Z","iopub.status.idle":"2023-02-09T09:48:32.853089Z","shell.execute_reply.started":"2023-02-09T09:39:50.658432Z","shell.execute_reply":"2023-02-09T09:48:32.851769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"play_video(\"/kaggle/working/output2.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-02-09T09:48:32.856009Z","iopub.execute_input":"2023-02-09T09:48:32.856515Z","iopub.status.idle":"2023-02-09T09:48:34.505605Z","shell.execute_reply.started":"2023-02-09T09:48:32.856468Z","shell.execute_reply":"2023-02-09T09:48:34.502563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defaulut visualizer is not fast. If you would like to run inference process faster, please refer to following [notebook](https://www.kaggle.com/code/takuyasukegawa/detectron2-faster-visualizer#Defalut-visualizer). (For panoptic segmentation.)","metadata":{}},{"cell_type":"markdown","source":"## Reference\n\n* [NFL - YOLOv8 Object Detection and Segmentation](https://www.kaggle.com/code/dariussingh/nfl-yolov8-object-detection-and-segmentation)\n* [NFL player segementation](https://www.kaggle.com/code/vedantgoswami/nfl-player-segementation)","metadata":{}}]}