{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this kernel we will explore FaceForensics++ model from https://github.com/ondyari/FaceForensics\n\nSteps:\n\n- Detect faces with Retinaface\n- Check inference results of FaceForensics++\n- Do transfer learning "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T.reset_index()\ntrain_sample_metadata.columns = ['fname','label','split','original']\ntrain_sample_metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_sample_df = train_sample_metadata[train_sample_metadata.label == 'FAKE']\nreal_sample_df = train_sample_metadata[train_sample_metadata.label == 'REAL']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = Path('/kaggle/input/deepfake-detection-challenge/train_sample_videos/')\ntest_dir = Path('/kaggle/input/deepfake-detection-challenge/test_videos/')\ntrain_video_files = get_files(train_dir, extensions=['.mp4'])\ntest_video_files = get_files(test_dir, extensions=['.mp4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_video_files), len(test_video_files)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### FaceForensics"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q /kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,'/kaggle/input/faceforensics-pretrained/FaceForensics/classification/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_model = torch.load(\"/kaggle/input/faceforensics-pretrained/faceforensics_models_subset/xception/full_raw.p\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Decord Reader GPU/CPU\n\nThanks to: https://www.kaggle.com/leighplt/decord-videoreader/data"},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp /kaggle/input/decord/install.sh . && chmod  +x install.sh && ./install.sh ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,'/kaggle/working/reader/python')\n\nfrom decord import VideoReader\nfrom decord import cpu, gpu\nfrom decord.bridge import set_bridge\nset_bridge('torch')\ndevice = torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"retinaface_stats = tensor([123,117,104])[...,None,None].to(device) # RGB stats for mean\n\ndef get_decord_video_batch_cpu(fname, sz, freq=10, stats=None):\n    \"get batch tensor for inference, original for cropping and H,W of video\"\n    video = VideoReader(str(fname), ctx=cpu())\n    t = video.get_batch(range(0, len(video), freq))\n    H,W = t.shape[2:]\n    if sz: t = F.interpolate(t.to(torch.float32), (sz,sz)).to(device)\n    if stats is not None: t -= stats\n    return (t, (H, W))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### retinaface"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,\"/kaggle/input/retina-face/Pytorch_Retinaface/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nfrom data import cfg_mnet, cfg_re50\nfrom layers.functions.prior_box import PriorBox\nfrom utils.nms.py_cpu_nms import py_cpu_nms\nimport cv2\nfrom models.retinaface import RetinaFace\nfrom utils.box_utils import decode, decode_landm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_keys(model, pretrained_state_dict):\n    ckpt_keys = set(pretrained_state_dict.keys())\n    model_keys = set(model.state_dict().keys())\n    used_pretrained_keys = model_keys & ckpt_keys\n    unused_pretrained_keys = ckpt_keys - model_keys\n    missing_keys = model_keys - ckpt_keys\n    print('Missing keys:{}'.format(len(missing_keys)))\n    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))\n    print('Used keys:{}'.format(len(used_pretrained_keys)))\n    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'\n    return True\n\n\ndef remove_prefix(state_dict, prefix):\n    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''\n    print('remove prefix \\'{}\\''.format(prefix))\n    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x\n    return {f(key): value for key, value in state_dict.items()}\n\n\ndef load_model(model, pretrained_path, load_to_cpu):\n    print('Loading pretrained model from {}'.format(pretrained_path))\n    if load_to_cpu:\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)\n    else:\n        device = torch.cuda.current_device()\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))\n    if \"state_dict\" in pretrained_dict.keys():\n        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')\n    else:\n        pretrained_dict = remove_prefix(pretrained_dict, 'module.')\n    check_keys(model, pretrained_dict)\n    model.load_state_dict(pretrained_dict, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg_re50['image_size'], cfg_mnet['image_size']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cudnn.benchmark = True # keep input size constant for better runtime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(modelname=\"mobilenet\"):\n    torch.set_grad_enabled(False)\n    cfg = None\n    cfg_mnet['pretrain'] = False\n    cfg_re50['pretrain'] = False\n    \n    if modelname == \"mobilenet\":\n        cfg = cfg_mnet\n        pretrained_path = \"/kaggle/input/retina-face/Pytorch_Retinaface/weights/mobilenet0.25_Final.pth\"\n    else:\n        cfg = cfg_re50\n        pretrained_path = \"/kaggle/input/retina-face/Pytorch_Retinaface/weights/Resnet50_Final.pth\"\n    \n    # net and model\n    net = RetinaFace(cfg=cfg, phase='test')\n    net = load_model(net, pretrained_path, False)\n    net.eval().to(device)\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resize = 1\nscale_rate = 1\n\nsz = 640\nim_height, im_width = sz, sz \nscale = torch.Tensor([sz, sz, sz, sz])\nscale = scale.to(device)\n\n\nconfidence_threshold = 0.5\ntop_k = 5\nnms_threshold = 0.5\nkeep_top_k = 5\n\n\n\n\ndef predict(model:torch.nn.Module, t:tensor, sz:int, cfg):\n    \"get prediction for a batch t by model with image sz\"\n    locs, confs, landmss = torch.Tensor([]), torch.Tensor([]), torch.Tensor([])\n    locs = locs.to(device)\n    confs = confs.to(device)\n    landmss = landmss.to(device)\n    \n    # forward pass\n    locs_, confs_, landmss_ = model(t)  \n    locs = torch.cat((locs, locs_), 0)\n    confs = torch.cat((confs, confs_), 0)\n    landmss = torch.cat((landmss, landmss_), 0)\n    \n\n\n    result = []\n    priorbox = PriorBox(cfg, image_size=(im_height, im_width))\n    priors = priorbox.forward()\n    priors = priors.to(device)\n    prior_data = priors.data\n    for idx in range(t.size(0)):\n        loc = locs[idx]\n        conf = confs[idx]\n        landms = landmss[idx]\n\n        boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])\n        boxes = boxes * scale / resize\n\n        boxes = boxes.cpu().numpy()\n        scores = conf.squeeze(0).data.cpu().numpy()[:, 1]\n        landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])\n        scale1 = torch.Tensor([t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2]])\n        scale1 = scale1.to(device)\n        landms = landms * scale1 / resize\n        landms = landms.cpu().numpy()\n\n        # ignore low scores\n        inds = np.where(scores > confidence_threshold)[0]\n        boxes = boxes[inds]\n        landms = landms[inds]\n        scores = scores[inds]\n\n        # keep top-K before NMS\n        order = scores.argsort()[::-1][:top_k]\n        boxes = boxes[order]\n        landms = landms[order]\n        scores = scores[order]\n\n        # do NMS\n        dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)\n        keep = py_cpu_nms(dets, nms_threshold)\n\n        # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)\n        dets = dets[keep, :]\n        landms = landms[keep]\n\n        # keep top-K faster NMS\n        dets = dets[:keep_top_k, :]\n        landms = landms[:keep_top_k, :]\n\n    #     dets = np.concatenate((dets, landms), axis=1)\n    #     dets = np.concatenate((dets, landms), axis=1)\n        result.append(dets[:,:-1])\n\n\n    for idx in range(len(result)):\n        result[idx][:, :4]=result[idx][:, :4]/scale_rate\n#         result[idx][:, 5:]=result[idx][:, 5:]/scale_rate\n    \n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel = get_model(\"mobilenet\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_bboxes(bboxes, H, W, sz):\n    \"rescale to original image sz\"\n    res = []\n    for bb in bboxes:\n        h_scale, w_scale = H/sz, W/sz\n        orig_bboxes = (bb*array([w_scale, h_scale, w_scale, h_scale])[None, ...]).astype(int)\n        res.append(orig_bboxes)\n    return res","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rescale_bbox(bb, bb_scale, H,W):\n    \"rescale a bbox: (left, top, right, bottom) with a given scale parameter\"\n    left, top, right, bottom = bb\n    \n    cx,cy = (top + bottom)//2, (left + right)//2 \n    h,w = (bottom - top), (right - left)\n    sh, sw = int(h*bb_scale), int(w*bb_scale)\n\n    stop, sbottom = cx - sh//2, cx + sh//2\n    sleft, sright = cy - sw//2, cy + sw//2\n    stop, sleft, sbottom, sright = max(0, stop), max(0, sleft), min(H, sbottom), min(W, sright)    \n    return (sleft, stop, sright, sbottom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_model = xception_model.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_state_dict = torch.load(\"/kaggle/input/deepfake-trained-models/part-49-xception-single-face.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_model.load_state_dict(xception_state_dict['model']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_stats = (tensor([0.5,0.5,0.5], tensor([0.5,0.5,0.5])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"retinaface_stats = tensor([123,117,104])[...,None,None].to(device) # RGB stats for mean\n\ndef get_decord_video_batch_cpu_inference(fname, sz, freq=10, stats=None):\n    \"get batch tensor for inference, original for cropping and H,W of video\"\n    video = VideoReader(str(fname), ctx=cpu())\n    t_raw = video.get_batch(range(0, len(video), freq))\n    H,W = t_raw.shape[2:]\n    if sz: t = F.interpolate(t_raw.to(torch.float32), (sz,sz)).to(device)\n    if stats is not None: t -= stats\n    return (t, t_raw, (H, W))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsz = 640\nfreq = 10\nfor fname in tqdm(test_video_files):\n    try:\n        t, t_raw, (H, W) = get_decord_video_batch_cpu_inference(fname, sz, freq, retinaface_stats)\n        bboxes = predict(model, t, sz, cfg_mnet)\n        orig_bboxes = convert_bboxes(bboxes, H, W, sz)\n        del t; gc.collect()\n\n        # collect crops    \n        face_crops = []\n        for frame_no, (_frame, _bb) in enumerate(zip(t_raw, orig_bboxes)):\n            # don't try cropping if no detection is available for the frame\n            try: _bb[0] \n            except: continue\n            # naive: get first bbox, optionally rescale\n            left, top, right, bottom  = rescale_bbox(_bb[0], 1.3, H, W) \n            # crop and save\n            face_crop = F.interpolate(_frame[:, top:bottom, left:right][None].float(), (299,299))[0]\n            face_crops.append(face_crop)\n\n        # predict\n        xb = normalize(torch.stack(face_crops)/255, *xception_stats)\n        score = to_cpu(xception_model(xb.cuda()).softmax(1))[:,1].mean()\n        predictions.append(score.item())\n    except:\n        predictions.append(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_fnames = [o.name for o in test_video_files]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/deepfake-detection-challenge/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.label = submission_df.filename.map(dict(zip(test_fnames, predictions)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}