{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T.reset_index()\ntrain_sample_metadata.columns = ['fname','label','split','original']\ntrain_sample_metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_sample_df = train_sample_metadata[train_sample_metadata.label == 'FAKE']\nreal_sample_df = train_sample_metadata[train_sample_metadata.label == 'REAL']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = Path('/kaggle/input/deepfake-detection-challenge/train_sample_videos/')\ntest_dir = Path('/kaggle/input/deepfake-detection-challenge/test_videos/')\ntrain_video_files = get_files(train_dir, extensions=['.mp4'])\ntest_video_files = get_files(test_dir, extensions=['.mp4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_video_files), len(test_video_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dummy_video_file = train_video_files[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Decord Reader GPU/CPU\n\nThanks to: https://www.kaggle.com/leighplt/decord-videoreader/data"},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp /kaggle/input/decord/install.sh . && chmod  +x install.sh && ./install.sh ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,'/kaggle/working/reader/python')\n\nfrom decord import VideoReader\nfrom decord import cpu\nfrom decord.bridge import set_bridge\nset_bridge('torch')\ndevice = torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"retinaface_stats = tensor([123,117,104]).to(device) # RGB stats for mean\n\ndef decord_cpu_video_reader(path, freq=None):\n    video = VideoReader(str(path), ctx=cpu())\n    len_video = len(video)\n    if freq: t = video.get_batch(range(0, len(video), freq)).permute(0,3,1,2)\n    else: t = video.get_batch(range(len_video))\n    return t, len_video\n\n#export\ndef get_decord_video_batch_cpu(path, freq=10, sz=640, stats:Tensor=None, device=defaults.device):\n    \"get resized and mean substracted batch tensor of a sampled video (scale of 255)\"\n    t_raw, len_video = decord_cpu_video_reader(path, freq)\n    H,W = t_raw.size(2), t_raw.size(3) \n    t = F.interpolate(t_raw.to(device).to(torch.float32), (sz,sz))\n    if stats is not None: t -= stats[...,None,None]\n    return t, t_raw, (H, W)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"retinaface_stats = tensor([123,117,104]).to(device) # RGB stats for mean\nt, t_raw, (H,W) = get_decord_video_batch_cpu(dummy_video_file, 10, 640, retinaface_stats)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### retinaface"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0,\"/kaggle/input/retina-face-2/Pytorch_Retinaface_2/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nfrom data import cfg_mnet, cfg_re50\nfrom layers.functions.prior_box import PriorBox\nfrom utils.nms.py_cpu_nms import py_cpu_nms\nimport cv2\nfrom models.retinaface import RetinaFace\nfrom utils.box_utils import decode, decode_landm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_keys(model, pretrained_state_dict):\n    ckpt_keys = set(pretrained_state_dict.keys())\n    model_keys = set(model.state_dict().keys())\n    used_pretrained_keys = model_keys & ckpt_keys\n    unused_pretrained_keys = ckpt_keys - model_keys\n    missing_keys = model_keys - ckpt_keys\n    print('Missing keys:{}'.format(len(missing_keys)))\n    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))\n    print('Used keys:{}'.format(len(used_pretrained_keys)))\n    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'\n    return True\n\n\ndef remove_prefix(state_dict, prefix):\n    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''\n    print('remove prefix \\'{}\\''.format(prefix))\n    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x\n    return {f(key): value for key, value in state_dict.items()}\n\n\ndef load_model(model, pretrained_path, load_to_cpu):\n    print('Loading pretrained model from {}'.format(pretrained_path))\n    if load_to_cpu:\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)\n    else:\n        device = torch.cuda.current_device()\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))\n    if \"state_dict\" in pretrained_dict.keys():\n        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')\n    else:\n        pretrained_dict = remove_prefix(pretrained_dict, 'module.')\n    check_keys(model, pretrained_dict)\n    model.load_state_dict(pretrained_dict, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg_re50['image_size'], cfg_mnet['image_size']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cudnn.benchmark = True # keep input size constant for better runtime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(modelname=\"mobilenet\"):\n    torch.set_grad_enabled(False)\n    cfg = None\n    cfg_mnet['pretrain'] = False\n    cfg_re50['pretrain'] = False\n    \n#     if modelname == \"mobilenet\": \n#         pretrained_path = \"../input/retina-face-2/Pytorch_Retinaface_2/weights/mobilenet0.25_Final.pth\"\n#         cfg = cfg_mnet\n#     else: raise Exception(f\"only options are: 'mobilenet' or 'resnet50'\")\n\n    if modelname == \"mobilenet\": \n        pretrained_path = \"../input/retina-face-2/Pytorch_Retinaface_2/weights/mobilenet0.25_Final.pth\"\n        cfg = cfg_mnet\n        \n    if modelname == \"resnet50\": \n        pretrained_path = \"../input/retina-face/Pytorch_Retinaface/weights/Resnet50_Final.pth\"\n        cfg = cfg_re50\n    \n    # net and model\n    net = RetinaFace(cfg=cfg, phase='test')\n    net = load_model(net, pretrained_path, False)\n    net.eval().to(device)\n    return net, cfg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, t, sz, cfg, \n            confidence_threshold = 0.5, top_k = 5, nms_threshold = 0.5, keep_top_k = 5):\n    \"get prediction for a batch t by model with image sz\"\n\n    resize = 1\n    scale_rate = 1\n\n    im_height, im_width = sz, sz \n    scale = torch.Tensor([sz, sz, sz, sz])\n    scale = scale.to(device)\n    \n    \n    locs, confs, landmss = torch.Tensor([]), torch.Tensor([]), torch.Tensor([])\n    locs = locs.to(device)\n    confs = confs.to(device)\n    landmss = landmss.to(device)\n    \n    # forward pass\n    locs_, confs_, landmss_ = model(t)  \n    locs = torch.cat((locs, locs_), 0)\n    confs = torch.cat((confs, confs_), 0)\n    landmss = torch.cat((landmss, landmss_), 0)\n    \n    \n    bbox_result, landms_result = [], []\n    priorbox = PriorBox(cfg, image_size=(im_height, im_width))\n    priors = priorbox.forward()\n    priors = priors.to(device)\n    prior_data = priors.data\n    for idx in range(t.size(0)):\n        loc = locs[idx]\n        conf = confs[idx]\n        landms = landmss[idx]\n\n        boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])\n        boxes = boxes * scale / resize\n\n        boxes = boxes.cpu().numpy()\n        scores = conf.squeeze(0).data.cpu().numpy()[:, 1]\n        landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])\n        scale1 = torch.Tensor([t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2], t.shape[3], t.shape[2],\n                            t.shape[3], t.shape[2]])\n        scale1 = scale1.to(device)\n        landms = landms * scale1 / resize\n        landms = landms.cpu().numpy()\n\n        # ignore low scores\n        inds = np.where(scores > confidence_threshold)[0]\n        boxes = boxes[inds]\n        landms = landms[inds]\n        scores = scores[inds]\n\n        # keep top-K before NMS\n        order = scores.argsort()[::-1][:top_k]\n        boxes = boxes[order]\n        landms = landms[order]\n        scores = scores[order]\n\n        # do NMS\n        dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)\n        keep = py_cpu_nms(dets, nms_threshold)\n\n        # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)\n        dets = dets[keep, :]\n        landms = landms[keep]\n    \n        # keep top-K faster NMS\n        dets = dets[:keep_top_k, :]\n        landms = landms[:keep_top_k, :]\n\n    #     dets = np.concatenate((dets, landms), axis=1)\n    #     dets = np.concatenate((dets, landms), axis=1)\n        bbox_result.append(dets[:,:-1]) # don't keep confidence score\n        landms_result.append(landms)\n    \n    return  bbox_result, landms_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel, cfg = get_model(\"mobilenet\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Bbox Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def bboxes_to_original_scale(bboxes, H, W, sz):\n    \"\"\"\n    convert bbox points to original image scale\n    \n    bboxes: List of numpy arrays with shape (M, 4) M: # of bbox coordinates\n    \"\"\"\n    res = []\n    for bb in bboxes:\n        h_scale, w_scale = H/sz, W/sz\n        orig_bboxes = (bb*array([w_scale, h_scale, w_scale, h_scale])[None, ...]).astype(int)\n        res.append(orig_bboxes)\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_bbox_by_scale(bb, bb_scale, H, W):\n    \"\"\"\n    resize a bbox with a given scale parameter\n    \n    bb: a bounding box with (left, top, right, bottom) values\n    \"\"\"\n    left, top, right, bottom = bb\n    \n    cx,cy = (top + bottom)//2, (left + right)//2 \n    h,w = (bottom - top), (right - left)\n    sh, sw = int(h*bb_scale), int(w*bb_scale)\n\n    stop, sbottom = cx - sh//2, cx + sh//2\n    sleft, sright = cy - sw//2, cy + sw//2\n    stop, sleft, sbottom, sright = max(0, stop), max(0, sleft), min(H, sbottom), min(W, sright)    \n    return (sleft, stop, sright, sbottom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def landmarks_to_original_scale(landmarks, H, W, sz):\n    \"\"\"\n    convert landmarks to original image scale\n    \n    landmarks: List of numpy arrays with shape (M, 10) M: # landmark coordinates\n    \"\"\"\n    res = []\n    for landms in landmarks:\n        h_scale, w_scale = H/sz, W/sz\n        orig_landms = (landms*array([w_scale, h_scale]*5)[None, ...]).astype(int)\n        res.append(orig_landms)\n    return res","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# face detection config\nfreq = 5\nmodel_args = dict(confidence_threshold = 0.5, top_k = 5, nms_threshold = 0.5, keep_top_k = 5)\nsz = cfg['image_size']\nimgnet_stats = [tensor(o) for o in imagenet_stats]\nrescale_param = 1.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install -q ../input/pretrainedmodels/pretrainedmodels-0.7.4/\n!pip install -q ../input/efficientnetpytorchpip/efficientnet_pytorch-0.6.3/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from fastai.vision.models.cadene_models import *\nfrom fastai.vision.models.efficientnet import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load detection model\nclass DummyDatabunch:\n    c = 2\n    path = '.'\n    device = defaults.device\n    loss_func = None\n\ndata = DummyDatabunch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"effnet_model = EfficientNet.from_name(\"efficientnet-b5\", override_params={'num_classes': 2})\nlearner = Learner(data, effnet_model); learner.model_dir = '.'\nlearner.load('../input/deepfakerandmergeaugmodels/single_frame_effnetb5_randmerge')\neffnetb5_inference_model = learner.model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"effnet_model = EfficientNet.from_name(\"efficientnet-b7\", override_params={'num_classes': 2})\nlearner = Learner(data, effnet_model); learner.model_dir = '.'\nlearner.load('../input/deepfakerandmergeaugmodels/single_frame_effnetb7_randmerge_fp16')\neffnetb7_inference_model = learner.model.float().eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner = cnn_learner(data, models.resnet34, pretrained=False); learner.model_dir = '.'\nlearner.load('../input/deepfakerandmergeaugmodels/single_frame_resnet34_randmerge')\nresnet_inference_model = learner.model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []\nvideo_fnames = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fname in tqdm(test_video_files):\n    try:\n        # append video filename\n        video_fnames.append(fname.name)\n        \n        # read video frames\n        t, t_raw, (H,W) = get_decord_video_batch_cpu(fname, freq, sz, retinaface_stats)\n        \n        # detect bboxes\n        bboxes, landmarks = predict(model, t, sz, cfg, **model_args)\n        orig_bboxes = bboxes_to_original_scale(bboxes, H, W, sz)\n        orig_bboxes = [o.tolist() for o in orig_bboxes]\n        \n        # create face crop batch\n        video_crop_batch, video_crop_batch_tta = [], []\n        for frame_no, (_frame, _bb) in enumerate(zip(t_raw, orig_bboxes)):\n            # don't try cropping if no detection is available for the frame\n            try: _bb[0] \n            except: continue\n            # naive: get first bbox, optionally rescale\n            left, top, right, bottom  = resize_bbox_by_scale(_bb[0], rescale_param, H, W) \n            \n            # crop and save\n            face_crop = Image(_frame[:, top:bottom, left:right].to(torch.float32).div(255))\n            # resize\n            x = face_crop.resize(224).data\n            # normalize\n            x = normalize(x, *imgnet_stats)\n            video_crop_batch.append(x)\n            \n            # crop and save\n            face_crop = Image(_frame[:, top:bottom, left:right].to(torch.float32).div(255)).flip_lr()\n            # resize\n            x = face_crop.resize(224).data\n            # normalize\n            x = normalize(x, *imgnet_stats)\n            video_crop_batch_tta.append(x)\n        \n        # batches\n        video_crop_batch = torch.stack(video_crop_batch)\n        video_crop_batch_tta = torch.stack(video_crop_batch_tta)\n        \n        \n        # do inference\n        resnet_out = to_cpu(resnet_inference_model(video_crop_batch.to(device))).softmax(1)\n        effnetb5_out = to_cpu(effnetb5_inference_model(video_crop_batch.to(device))).softmax(1)\n        effnetb7_out = to_cpu(effnetb7_inference_model(video_crop_batch.to(device))).softmax(1)     \n        # tta\n        resnet_out_tta = to_cpu(resnet_inference_model(video_crop_batch_tta.to(device))).softmax(1)\n        effnetb5_out_tta = to_cpu(effnetb5_inference_model(video_crop_batch_tta.to(device))).softmax(1)\n        effnetb7_out_tta = to_cpu(effnetb7_inference_model(video_crop_batch_tta.to(device))).softmax(1)     \n        \n        \n        mean_pred = torch.stack([resnet_out[:,1],\n                                 effnetb5_out[:,1],\n                                 effnetb7_out[:,1],\n                                 resnet_out_tta[:,1],\n                                 effnetb5_out_tta[:,1],\n                                 effnetb7_out_tta[:,1]], 1).mean(1).mean().item()                \n        predictions.append(mean_pred)\n\n\n    except:\n        predictions.append(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname2pred = dict(zip(video_fnames, predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/deepfake-detection-challenge/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.label = submission_df.filename.map(fname2pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['label'] = np.clip(submission_df['label'], 0.01, 0.99)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}