{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a based on using Capsule networks to identify Deep Forged Images & Videos  [first paper](https://arxiv.org/abs/1810.11215) and [second paper](https://arxiv.org/abs/1910.12467)\n\nThe notebook is modified from the following [github rep](https://github.com/nii-yamagishilab/Capsule-Forensics-v2)\n\nThe models are used as is w/o any training.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install facenet-pytorch\n!pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-2.0.0-py3-none-any.whl\n\nfrom facenet_pytorch.models.inception_resnet_v1 import get_torch_home\ntorch_home = get_torch_home()\n\n# Copy model checkpoints to torch cache so they are loaded automatically by the package\n!mkdir -p $torch_home/checkpoints/\n!cp /kaggle/input/facenet-pytorch-vggface2/20180402-114759-vggface2-logits.pth $torch_home/checkpoints/vggface2_DG3kwML46X.pt\n!cp /kaggle/input/facenet-pytorch-vggface2/20180402-114759-vggface2-features.pth $torch_home/checkpoints/vggface2_G5aNV2VSMn.pt\n!cp /kaggle/input/superresolution-pets/vgg19-dcbb9e9d.pth $torch_home/checkpoints/vgg19-dcbb9e9d.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# See github.com/timesler/facenet-pytorch:\nfrom facenet_pytorch import MTCNN, InceptionResnetV1, extract_face\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport pathlib\nfrom pathlib import Path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sys.path.append('/kaggle/input/superresolution-pets')\nsys.path.append('/kaggle/input/facent-pytorch-vggface2')\nsys.path.append('/kaggle/input/resnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import model_big\nimport utilDeepFake","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.setrecursionlimit(15000)\nimport os\nimport torch\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nfrom torch.autograd import Variable\nimport torch.utils.data\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nfrom tqdm import tqdm\nimport argparse\nfrom sklearn import metrics\nfrom scipy.optimize import brentq\nfrom scipy.interpolate import interp1d\nfrom sklearn.metrics import roc_curve","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# setup a dict with the required command line default parameters \n\nopt = dict()\n\nopt['dataset'] =''\nopt['imageSize'] = 160\nopt['gpu_id'] = 0 if torch.cuda.is_available() else 'cpu'\nprint(f'Running on device: {opt[\"gpu_id\"]}')\nopt['outf'] = '/kaggle/input/superresolution-pets/checkpoints/binary_faceforensicspp'\nopt['id'] = 21\nopt['random'] = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform_fwd = transforms.Compose([\n        transforms.Resize((opt['imageSize'], opt['imageSize'])),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#vgg_ext = torch.load('/kaggle/input/superresolution-pets/vgg19-dcbb9e9d.pth')\nvgg_ext = model_big.VggExtractor()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"capnet = model_big.CapsuleNet(2, opt['gpu_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"capnet.load_state_dict(torch.load(os.path.join(opt['outf'],'capsule_' + str(opt['id']) + '.pt')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"capnet.eval();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if opt['gpu_id'] >= 0:\n    vgg_ext.cuda(opt['gpu_id'])\n    capnet.cuda(opt['gpu_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tol_label = np.array([], dtype=np.float)\ntol_pred = np.array([], dtype=np.float)\ntol_pred_prob = np.array([], dtype=np.float)\n\ncount = 0\nloss_test = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testPath = Path('/kaggle/input/deepfake-detection-challenge/test_videos')\ntest_videos = [f for f in testPath.glob(\"**/*\") if f.is_file() and '.mp4' in str(f)]\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nfrom fastai.vision import *\nfrom PIL import Image\nimport torchvision.transforms.functional as TF","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_face(fileList):\n    frameSample = len(fileList)\n    predsF = list()\n    \n    for faceFile in fileList:\n        img = Image.open(faceFile)\n        x = TF.to_tensor(img).cuda(opt['gpu_id'])\n        x.unsqueeze_(0)\n        input_v = Variable(x)\n        x = vgg_ext(input_v)\n        classes, class_ = capnet(x, random=opt['random'])\n        outputs = class_.data.cpu()\n        predsF.append(float(outputs[0][0]))\n        #print(outputs)\n        os.remove(faceFile)\n    print(sum(predsF)/frameSample) \n    return sum(predsF)/frameSample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vName = str(test_videos[0]).split('/')[-1]\nframes = utilDeepFake.extractFrames(str(test_videos[0]), frameSample = 10)\nfaceCrops = utilDeepFake.detect_facenet_pytorch(frames, 16)\nfileList = utilDeepFake.saveFaces(faceCrops, vName, '2', faceDir = '/kaggle/working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"faceFile = fileList[0]\n\nimg = Image.open(faceFile)\nx = TF.to_tensor(img).cuda(opt['gpu_id'])\nx.unsqueeze_(0)\ninput_v = Variable(x)\nx = vgg_ext(input_v)\nclasses, class_ = capnet(x, random=opt['random'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"        outputs = class_.data.cpu()\n        predsF.append(float(outputs[0][0]))\n        #print(outputs)\n        os.remove(faceFile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time.time()\n# Sample run \nvName = str(test_videos[0]).split('/')[-1]\nframes = utilDeepFake.extractFrames(str(test_videos[0]), frameSample = 10)\nfaceCrops = utilDeepFake.detect_facenet_pytorch(frames, 16)\nfileList = utilDeepFake.saveFaces(faceCrops, vName, '2', faceDir = '/kaggle/working/')\npreds = predict_face(fileList)\nend = time.time()\nprint(\"Prediction: \", preds)\nprint(\"Time for processing single video \", (end - start))\nprint(\"Is it good to go? \", (end - start)< 8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time \n\nclassP =list()\nprediction = list()\nfilename = list()\n\nstart = time.time()\n\nfor video in test_videos:\n    vName = str(video).split('/')[-1]\n    \n    try:\n\n        frames = utilDeepFake.extractFrames(str(video), frameSample = 10)\n        faceCrops = utilDeepFake.detect_facenet_pytorch(frames, 16)\n        fileList = utilDeepFake.saveFaces(faceCrops, vName, '2', faceDir = '/kaggle/working/')\n        preds = predict_face(fileList)\n        #classP.append(pred_class)\n        prediction.append(preds)\n        filename.append(vName)\n        \n    except:\n        print(\"Error in file: {}, appending 0.5 prob\".format(vName) )\n        #classP.append('NA')\n        prediction.append(0.5)\n        filename.append(vName)\n    #break\n    \nend = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total time elapsed: \", (end-start) )\nprint(\"Average time on each video: \", (end-start)/len(test_videos))\nprint(\"Cool, so we have a GO!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({\"filename\": filename, \"label\": prediction})\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(submission_df.shape)\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.label.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}