{"cells":[{"metadata":{},"cell_type":"markdown","source":"- In this kernel I am exploring https://github.com/biubug6/Pytorch_Retinaface for face detection\n- Replaced video loader with Decord"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from fastai.vision import *\nimport cv2 as cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T.reset_index()\ntrain_sample_metadata.columns = ['fname','label','split','original']\ntrain_sample_metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_sample_df = train_sample_metadata[train_sample_metadata.label == 'FAKE']\nreal_sample_df = train_sample_metadata[train_sample_metadata.label == 'REAL']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = Path('/kaggle/input/deepfake-detection-challenge/train_sample_videos/')\ntest_dir = Path('/kaggle/input/deepfake-detection-challenge/test_videos/')\ntrain_video_files = get_files(train_dir, extensions=['.mp4'])\ntest_video_files = get_files(test_dir, extensions=['.mp4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_video_files), len(test_video_files)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### read frames"},{"metadata":{"trusted":true},"cell_type":"code","source":"def frame_img_generator(path, freq=None):\n    \"frame image generator for a given video file\"\n    vidcap = cv.VideoCapture(str(path))\n    n_frames = 0\n    while True:\n        success = vidcap.grab()\n        if not success: \n            vidcap.release()\n            break   \n            \n        if (freq is None) or (n_frames % freq == 0):\n            _, image = vidcap.retrieve()\n            image = cv.cvtColor(image, cv.COLOR_BGR2RGB)\n            yield image    \n        \n        n_frames += 1\n        \n    vidcap.release()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nframes = list(frame_img_generator(train_video_files[0],10)); len(frames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_video_batch(fname, sz, freq=10):\n    \"get batch tensor for inference, original for cropping and H,W of video\"\n    orig_frames = array(frame_img_generator(fname, freq))\n    H,W = orig_frames[0].shape[:-1]\n    resized_frames = array([cv2.resize(o,(sz,sz)) for o in orig_frames])\n    t = (resized_frames - array([123,117,104]))\n    t = (torch.from_numpy(t).to(torch.float32)).permute(0,3,1,2).to(device)\n    return (t, orig_frames, (H, W))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Decord Reader GPU/CPU\n\nThanks to: https://www.kaggle.com/leighplt/decord-videoreader/data"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ipyexperiments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp /kaggle/input/decord/install.sh . && chmod  +x install.sh && ./install.sh ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,'/kaggle/working/reader/python')\n\nfrom decord import VideoReader\nfrom decord import cpu, gpu\nfrom decord.bridge import set_bridge\nset_bridge('torch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"retinaface_stats = tensor([123,117,104])[...,None,None].cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### MEMORY LEAK - LET ME KNOW HOW TO MAKE THIS WORK :)\nfrom torch.utils.dlpack import to_dlpack, from_dlpack\n\ndef get_decord_video_batch(fname, sz, freq=10):\n    \"get batch tensor for inference, original for cropping and H,W of video\"\n    video = VideoReader(str(fname), ctx=gpu())\n#     data = video.get_batch(range(0, len(video), 10))\n    data = video.get_batch(range(0, len(video), freq))\n    H,W = data.shape[2:]\n    data = F.interpolate(data.to(torch.float32), (sz,sz))\n    data -= retinaface_stats\n    del video; gc.collect()\n    return (data, None, (H, W))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ipyexperiments import IPyExperimentsPytorch\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# sz = 640\n# for fname in tqdm(train_video_files[:10]):\n#     with IPyExperimentsPytorch() as exp: \n#         t, _, (H, W) = get_decord_video_batch(fname, sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.dlpack import to_dlpack, from_dlpack\n\ndef get_decord_video_batch_cpu(fname, sz, freq=10):\n    \"get batch tensor for inference, original for cropping and H,W of video\"\n    video = VideoReader(str(fname), ctx=cpu())\n    data = video.get_batch(range(0, len(video), 10)).cuda()\n    H,W = data.shape[2:]\n    data = F.interpolate(data.to(torch.float32), (sz,sz))\n    data -= retinaface_stats\n    return (data, None, (H, W))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# data, _, (H, W) = get_decord_video_batch(train_video_files[0], 640)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del data; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### NVIDIA Dali"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/cuda/10.0 nvidia-dali","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nvidia.dali.pipeline import Pipeline\nfrom nvidia.dali import ops","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=1\nsequence_length=30\ninitial_prefetch_size=16\n\nclass VideoPipe(Pipeline):\n    \"video pipeline for a single video with 30 frames\"\n    def __init__(self, batch_size, num_threads, device_id, data, shuffle):\n        super(VideoPipe, self).__init__(batch_size, num_threads, device_id, seed=16)\n        self.input = ops.VideoReader(device=\"gpu\", filenames=data, sequence_length=sequence_length,\n                                     shard_id=0, num_shards=1,\n                                     random_shuffle=shuffle, initial_fill=initial_prefetch_size)\n    def define_graph(self):\n        output = self.input(name=\"Reader\")\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = train_video_files[0]\npipe = VideoPipe(batch_size=batch_size, num_threads=defaults.cpus, device_id=0, data=[fname], shuffle=False)\npipe.build()\npipe_out = pipe.run()\nsequences_out = pipe_out[0].as_cpu().as_array()\ndata = torch.from_numpy(sequences_out[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape[1:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dali_batch(fname, sz=640):\n    pipe = VideoPipe(batch_size=batch_size, num_threads=defaults.cpus, device_id=0, data=[fname], shuffle=False)\n    pipe.build()\n    pipe_out = pipe.run()\n    sequences_out = pipe_out[0].as_cpu().as_array()\n    data = torch.from_numpy(sequences_out[0])\n    H,W = data.shape[1:3]\n    data = data.permute(0,3,1,2).cuda()\n    data = F.interpolate(data.to(torch.float32), (sz,sz))\n    data -= retinaface_stats\n    return data, _, (H,W)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# WARM UP\ndata = dali_batch(train_video_files[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndata = dali_batch(train_video_files[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### retinaface"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,\"/kaggle/input/retina-face/Pytorch_Retinaface/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nfrom data import cfg_mnet, cfg_re50\nfrom layers.functions.prior_box import PriorBox\nfrom utils.nms.py_cpu_nms import py_cpu_nms\nimport cv2\nfrom models.retinaface import RetinaFace\nfrom utils.box_utils import decode, decode_landm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_keys(model, pretrained_state_dict):\n    ckpt_keys = set(pretrained_state_dict.keys())\n    model_keys = set(model.state_dict().keys())\n    used_pretrained_keys = model_keys & ckpt_keys\n    unused_pretrained_keys = ckpt_keys - model_keys\n    missing_keys = model_keys - ckpt_keys\n    print('Missing keys:{}'.format(len(missing_keys)))\n    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))\n    print('Used keys:{}'.format(len(used_pretrained_keys)))\n    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'\n    return True\n\n\ndef remove_prefix(state_dict, prefix):\n    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''\n    print('remove prefix \\'{}\\''.format(prefix))\n    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x\n    return {f(key): value for key, value in state_dict.items()}\n\n\ndef load_model(model, pretrained_path, load_to_cpu):\n    print('Loading pretrained model from {}'.format(pretrained_path))\n    if load_to_cpu:\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)\n    else:\n        device = torch.cuda.current_device()\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))\n    if \"state_dict\" in pretrained_dict.keys():\n        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')\n    else:\n        pretrained_dict = remove_prefix(pretrained_dict, 'module.')\n    check_keys(model, pretrained_dict)\n    model.load_state_dict(pretrained_dict, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg_re50['image_size'], cfg_mnet['image_size']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cudnn.benchmark = True # keep input size constant for better runtime\ndevice = torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(modelname=\"mobilenet\"):\n    torch.set_grad_enabled(False)\n    cfg = None\n    cfg_mnet['pretrain'] = False\n    cfg_re50['pretrain'] = False\n    \n    if modelname == \"mobilenet\":\n        cfg = cfg_mnet\n        pretrained_path = \"/kaggle/input/retina-face/Pytorch_Retinaface/weights/mobilenet0.25_Final.pth\"\n    else:\n        cfg = cfg_re50\n        pretrained_path = \"/kaggle/input/retina-face/Pytorch_Retinaface/weights/Resnet50_Final.pth\"\n    \n    # net and model\n    net = RetinaFace(cfg=cfg, phase='test')\n    net = load_model(net, pretrained_path, False)\n    net.eval().to(device)\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resize = 1\nscale_rate = 1\n\nsz = 640\nim_height, im_width = sz, sz \nscale = torch.Tensor([sz, sz, sz, sz])\nscale = scale.to(device)\n\n\nconfidence_threshold = 0.5\ntop_k = 5\nnms_threshold = 0.5\nkeep_top_k = 5\n\n\n\n\ndef predict(model:torch.nn.Module, t:tensor, sz:int, cfg):\n    \"get prediction for a batch t by model with image sz\"\n    locs, confs, landmss = torch.Tensor([]), torch.Tensor([]), torch.Tensor([])\n    locs = locs.to(device)\n    confs = confs.to(device)\n    landmss = landmss.to(device)\n    \n    # forward pass\n    locs_, confs_, landmss_ = model(t)  \n    locs = torch.cat((locs, locs_), 0)\n    confs = torch.cat((confs, confs_), 0)\n    landmss = torch.cat((landmss, landmss_), 0)\n    \n\n\n    result = []\n    priorbox = PriorBox(cfg, image_size=(im_height, im_width))\n    priors = priorbox.forward()\n    priors = priors.to(device)\n    prior_data = priors.data\n    for idx in range(t.size(0)):\n        loc = locs[idx]\n        conf = confs[idx]\n        landms = landmss[idx]\n\n        boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])\n        boxes = boxes * scale / resize\n\n        boxes = boxes.cpu().numpy()\n        scores = conf.squeeze(0).data.cpu().numpy()[:, 1]\n        landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])\n        scale1 = torch.Tensor([t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2]])\n        scale1 = scale1.to(device)\n        landms = landms * scale1 / resize\n        landms = landms.cpu().numpy()\n\n        # ignore low scores\n        inds = np.where(scores > confidence_threshold)[0]\n        boxes = boxes[inds]\n        landms = landms[inds]\n        scores = scores[inds]\n\n        # keep top-K before NMS\n        order = scores.argsort()[::-1][:top_k]\n        boxes = boxes[order]\n        landms = landms[order]\n        scores = scores[order]\n\n        # do NMS\n        dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)\n        keep = py_cpu_nms(dets, nms_threshold)\n\n        # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)\n        dets = dets[keep, :]\n        landms = landms[keep]\n\n        # keep top-K faster NMS\n        dets = dets[:keep_top_k, :]\n        landms = landms[:keep_top_k, :]\n\n    #     dets = np.concatenate((dets, landms), axis=1)\n    #     dets = np.concatenate((dets, landms), axis=1)\n        result.append(dets[:,:-1])\n\n\n    for idx in range(len(result)):\n        result[idx][:, :4]=result[idx][:, :4]/scale_rate\n#         result[idx][:, 5:]=result[idx][:, 5:]/scale_rate\n    \n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel = get_model(\"mobilenet\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time \n# sz=640\n# (t, _, (H, W)) = get_decord_video_batch(train_video_files[0], sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# out = predict(model, t, sz, cfg_mnet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_bboxes(bboxes, H, W, sz):\n    \"rescale to original image sz\"\n    res = []\n    for bb in bboxes:\n        h_scale, w_scale = H/sz, W/sz\n        orig_bboxes = (bb*array([w_scale, h_scale, w_scale, h_scale])[None, ...]).astype(int)\n        res.append(orig_bboxes)\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# _= convert_bboxes(out, H, W, sz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del t, out\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsz = 640\nall_orig_bboxes = []\nfor fname in tqdm(train_video_files):\n#     t, _, (H, W) = get_video_batch(fname, sz) # ~30 s\n#     t, _, (H, W) = get_decord_video_batch_cpu(fname, sz)\n#     t, _, (H, W) = get_decord_video_batch(fname, sz)\n    t, _, (H, W) = dali_batch(fname, sz)\n    bboxes = predict(model, t, sz, cfg_mnet)\n    orig_bboxes = convert_bboxes(bboxes, H, W, sz)\n    all_orig_bboxes.append(orig_bboxes)\n    del t; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(all_orig_bboxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_orig_bboxes[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize\n\n- FIXME: DALI fails at detection: /kaggle/input/deepfake-detection-challenge/train_sample_videos/atxvxouljq.mp4"},{"metadata":{"trusted":true},"cell_type":"code","source":"i = np.random.choice(400)\norig_frames = list(frame_img_generator(train_video_files[i],10))\norig_bboxes = all_orig_bboxes[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_video_files[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orig_frames[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i,orig_bboxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_metadata[train_sample_metadata.fname == train_video_files[i].name]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"axes = subplots(5,6, figsize=(3*6,3*5)).flatten()\nfor idx, (ax, _frame, _bb) in enumerate(zip(axes, orig_frames, orig_bboxes)):\n    try:\n        left, top, right, bottom = _bb[0] # pick first detection for the given frame\n        ax.imshow(_frame[top:bottom, left:right, :])\n    except: continue # false negatives","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Play Video"},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\ndef video_url(fname):\n    vid1 = open(fname,'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(vid1).decode()\n    return data_url\n\ndef play_video(fname1, fname2=None): \n    url1 = video_url(fname1)\n    url2 = video_url(fname2) if fname2 else None\n    if url1 and url2:\n        html = HTML(\n            \"\"\"\n        <video width=900 controls>\n              <source src=\"%s\" type=\"video/mp4\">\n        </video>\n        <video width=900 controls>\n              <source src=\"%s\" type=\"video/mp4\">\n        </video>\n\n        \"\"\" % (url1, url2))\n    else:\n        html = HTML(\n            \"\"\"\n        <video width=900 controls>\n              <source src=\"%s\" type=\"video/mp4\">\n        </video>\n        \"\"\" % (url1))\n    return html","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# play_video(train_video_files[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Conclusion\n\n**Previously**\n\n- I/O seems to be the real bottleneck ~3 sec / 30 frames\n- RetinaNet with mobilenet is pretty fast ~ 120 ms / 30 frames\n- Total processing 400 videos: ~20 mins\n- Thresholds might need to be tuned for multi face detection\n- Results are pretty accurate in overall\n\n**After Decord**\n\n- Face detection of 30 frames per video can be made in ~ 6mins / 400 videos\n- It's not feasible because of memory leaks - let me know if there is a solution\n\n**After DALI**\n\n- Seems to be fast but there may be bugs which causes detection errors due to my missing knowledge about DALI\n- No memory leak"},{"metadata":{},"cell_type":"markdown","source":"### Feedback for improvement is welcome!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}