{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":846606,"sourceType":"datasetVersion","datasetId":446574}],"dockerImageVersionId":29845,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Running FaceForensics++ in a Kaggle Notebook\n\n![](https://github.com/ondyari/FaceForensics/raw/master/images/teaser.png)\n\nIn this notebook I test out the state of the art FaceForensics++ package on the provided dataset. Later in this notebook I modify the code slightly to predict for our dataset.\n\nThis notebook imports from a dataset for:\n- dlib package wheel (needs to be installed for code to work)\n- other required packages not in the default kernel\n- pretrained models by FaceForensics++\n\n- The paper that was published can be found here: https://arxiv.org/pdf/1901.08971.pdf\n- The github repo is here: https://github.com/ondyari/FaceForensics\n\nReference:\n```\n@inproceedings{roessler2019faceforensicspp,\n\tauthor = {Andreas R\\\"ossler and Davide Cozzolino and Luisa Verdoliva and Christian Riess and Justus Thies and Matthias Nie{\\ss}ner},\n\ttitle = {Face{F}orensics++: Learning to Detect Manipulated Facial Images},\n\tbooktitle= {International Conference on Computer Vision (ICCV)},\n\tyear = {2019}\n}\n```","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pylab as plt\nimport cv2\nimport seaborn as sns\nfrom sklearn.metrics import log_loss\nXCEPTION_MODEL = '../input/deepfakemodelspackages/xception-b5690688.pth'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-08T14:26:52.025986Z","iopub.execute_input":"2024-03-08T14:26:52.026369Z","iopub.status.idle":"2024-03-08T14:26:54.537210Z","shell.execute_reply.started":"2024-03-08T14:26:52.026304Z","shell.execute_reply":"2024-03-08T14:26:54.536537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Install packages\n!pip install ../input/deepfakemodelspackages/Pillow-6.2.1-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/munch-2.5.0-py2.py3-none-any.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/numpy-1.17.4-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/six-1.13.0-py2.py3-none-any.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/torchvision-0.4.2-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/tqdm-4.40.2-py2.py3-none-any.whl -f ./ --no-index","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:27:03.745221Z","iopub.execute_input":"2024-03-08T14:27:03.745579Z","iopub.status.idle":"2024-03-08T14:27:52.598771Z","shell.execute_reply.started":"2024-03-08T14:27:03.745519Z","shell.execute_reply":"2024-03-08T14:27:52.597682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install dlib\nBuilding takes roughly ~6 minutes","metadata":{}},{"cell_type":"code","source":"%%time\n!pip install ../input/deepfakemodelspackages/dlib-19.19.0/dlib-19.19.0/ -f ./ --no-index","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:27:52.601301Z","iopub.execute_input":"2024-03-08T14:27:52.601659Z","iopub.status.idle":"2024-03-08T14:33:38.638927Z","shell.execute_reply.started":"2024-03-08T14:27:52.601591Z","shell.execute_reply":"2024-03-08T14:33:38.637805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copy in FaceForensics Code and Modify to run in kernel\n\n- Change the `models.py` file to reference our `xception-b5690688.pth` file.\n- Change the code to be provided the pytorch model and not the model path. This works better for our purposes.\nChange the line here:\n```\n        state_dict = torch.load(\n            #'/home/ondyari/.torch/models/xception-b5690688.pth')\n            XCEPTION_MODEL)\n```","metadata":{}},{"cell_type":"code","source":"## xception.py\n\"\"\"\nPorted to pytorch thanks to [tstandley](https://github.com/tstandley/Xception-PyTorch)\n@author: tstandley\nAdapted by cadene\nCreates an Xception Model as defined in:\nFrancois Chollet\nXception: Deep Learning with Depthwise Separable Convolutions\nhttps://arxiv.org/pdf/1610.02357.pdf\nThis weights ported from the Keras implementation. Achieves the following performance on the validation set:\nLoss:0.9173 Prec@1:78.892 Prec@5:94.292\nREMEMBER to set your image size to 3x299x299 for both test and validation\nnormalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],\n                                  std=[0.5, 0.5, 0.5])\nThe resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n\"\"\"\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom torch.nn import init\n\npretrained_settings = {\n    'xception': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1000,\n            'scale': 0.8975 # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n        }\n    }\n}\n\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):\n        super(SeparableConv2d,self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)\n        self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):\n        super(Block, self).__init__()\n\n        if out_filters != in_filters or strides!=1:\n            self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)\n            self.skipbn = nn.BatchNorm2d(out_filters)\n        else:\n            self.skip=None\n\n        self.relu = nn.ReLU(inplace=True)\n        rep=[]\n\n        filters=in_filters\n        if grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n            filters = out_filters\n\n        for i in range(reps-1):\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n\n        if not start_with_relu:\n            rep = rep[1:]\n        else:\n            rep[0] = nn.ReLU(inplace=False)\n\n        if strides != 1:\n            rep.append(nn.MaxPool2d(3,strides,1))\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self,inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x+=skip\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Xception optimized for the ImageNet dataset, as specified in\n    https://arxiv.org/pdf/1610.02357.pdf\n    \"\"\"\n    def __init__(self, num_classes=1000):\n        \"\"\" Constructor\n        Args:\n            num_classes: number of classes\n        \"\"\"\n        super(Xception, self).__init__()\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.relu = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32,64,3,bias=False)\n        self.bn2 = nn.BatchNorm2d(64)\n        #do relu here\n\n        self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)\n        self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)\n        self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)\n\n        self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)\n\n        self.conv3 = SeparableConv2d(1024,1536,3,1,1)\n        self.bn3 = nn.BatchNorm2d(1536)\n\n        #do relu here\n        self.conv4 = SeparableConv2d(1536,2048,3,1,1)\n        self.bn4 = nn.BatchNorm2d(2048)\n\n        self.fc = nn.Linear(2048, num_classes)\n\n        # #------- init weights --------\n        # for m in self.modules():\n        #     if isinstance(m, nn.Conv2d):\n        #         n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n        #         m.weight.data.normal_(0, math.sqrt(2. / n))\n        #     elif isinstance(m, nn.BatchNorm2d):\n        #         m.weight.data.fill_(1)\n        #         m.bias.data.zero_()\n        # #-----------------------------\n\n    def features(self, input):\n        x = self.conv1(input)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        return x\n\n    def logits(self, features):\n        x = self.relu(features)\n\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef xception(num_classes=1000, pretrained='imagenet'):\n    model = Xception(num_classes=num_classes)\n    if pretrained:\n        settings = pretrained_settings['xception'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        model = Xception(num_classes=num_classes)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n\n    # TODO: ugly\n    model.last_linear = model.fc\n    del model.fc\n    return model\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:09.384504Z","iopub.execute_input":"2024-03-08T14:34:09.384822Z","iopub.status.idle":"2024-03-08T14:34:10.894265Z","shell.execute_reply.started":"2024-03-08T14:34:09.384773Z","shell.execute_reply":"2024-03-08T14:34:10.893492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## models.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\n\n\nimport torch\n# import pretrainedmodels\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from network.xception import xception\nimport math\nimport torchvision\n\n\ndef return_pytorch04_xception(pretrained=True):\n    # Raises warning \"src not broadcastable to dst\" but thats fine\n    model = xception(pretrained=False)\n    if pretrained:\n        # Load model in torch 0.4+\n        model.fc = model.last_linear\n        del model.last_linear\n        state_dict = torch.load(\n            #'/home/ondyari/.torch/models/xception-b5690688.pth')\n            XCEPTION_MODEL)\n        for name, weights in state_dict.items():\n            if 'pointwise' in name:\n                state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)\n        model.load_state_dict(state_dict)\n        model.last_linear = model.fc\n        del model.fc\n    return model\n\n\nclass TransferModel(nn.Module):\n    \"\"\"\n    Simple transfer learning model that takes an imagenet pretrained model with\n    a fc layer as base model and retrains a new fc layer for num_out_classes\n    \"\"\"\n    def __init__(self, modelchoice, num_out_classes=2, dropout=0.0):\n        super(TransferModel, self).__init__()\n        self.modelchoice = modelchoice\n        if modelchoice == 'xception':\n            self.model = return_pytorch04_xception()\n            # Replace fc\n            num_ftrs = self.model.last_linear.in_features\n            if not dropout:\n                self.model.last_linear = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                print('Using dropout', dropout)\n                self.model.last_linear = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        elif modelchoice == 'resnet50' or modelchoice == 'resnet18':\n            if modelchoice == 'resnet50':\n                self.model = torchvision.models.resnet50(pretrained=True)\n            if modelchoice == 'resnet18':\n                self.model = torchvision.models.resnet18(pretrained=True)\n            # Replace fc\n            num_ftrs = self.model.fc.in_features\n            if not dropout:\n                self.model.fc = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                self.model.fc = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        else:\n            raise Exception('Choose valid model, e.g. resnet50')\n\n    def set_trainable_up_to(self, boolean, layername=\"Conv2d_4a_3x3\"):\n        \"\"\"\n        Freezes all layers below a specific layer and sets the following layers\n        to true if boolean else only the fully connected final layer\n        :param boolean:\n        :param layername: depends on network, for inception e.g. Conv2d_4a_3x3\n        :return:\n        \"\"\"\n        # Stage-1: freeze all the layers\n        if layername is None:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = True\n                return\n        else:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = False\n        if boolean:\n            # Make all layers following the layername layer trainable\n            ct = []\n            found = False\n            for name, child in self.model.named_children():\n                if layername in ct:\n                    found = True\n                    for params in child.parameters():\n                        params.requires_grad = True\n                ct.append(name)\n            if not found:\n                raise Exception('Layer not found, cant finetune!'.format(\n                    layername))\n        else:\n            if self.modelchoice == 'xception':\n                # Make fc trainable\n                for param in self.model.last_linear.parameters():\n                    param.requires_grad = True\n\n            else:\n                # Make fc trainable\n                for param in self.model.fc.parameters():\n                    param.requires_grad = True\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n\ndef model_selection(modelname, num_out_classes,\n                    dropout=None):\n    \"\"\"\n    :param modelname:\n    :return: model, image size, pretraining<yes/no>, input_list\n    \"\"\"\n    if modelname == 'xception':\n        return TransferModel(modelchoice='xception',\n                             num_out_classes=num_out_classes), 299, \\\n               True, ['image'], None\n    elif modelname == 'resnet18':\n        return TransferModel(modelchoice='resnet18', dropout=dropout,\n                             num_out_classes=num_out_classes), \\\n               224, True, ['image'], None\n    else:\n        raise NotImplementedError(modelname)\n\n# if __name__ == '__main__':\n#     model, image_size, *_ = model_selection('resnet18', num_out_classes=2)\n#     print(model)\n#     model = model.cuda()\n#     from torchsummary import summary\n#     input_s = (3, image_size, image_size)\n#     print(summary(model, input_s))\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:15.203037Z","iopub.execute_input":"2024-03-08T14:34:15.203503Z","iopub.status.idle":"2024-03-08T14:34:15.348023Z","shell.execute_reply.started":"2024-03-08T14:34:15.203302Z","shell.execute_reply":"2024-03-08T14:34:15.347372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## transform.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nfrom torchvision import transforms\n\nxception_default_data_transforms = {\n    'train': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ]),\n    'val': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n}","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:18.955746Z","iopub.execute_input":"2024-03-08T14:34:18.956046Z","iopub.status.idle":"2024-03-08T14:34:18.964024Z","shell.execute_reply.started":"2024-03-08T14:34:18.955992Z","shell.execute_reply":"2024-03-08T14:34:18.963310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## detect_from_video.py\n\"\"\"\nEvaluates a folder of video files or a single file with a xception binary\nclassification network.\nUsage:\npython detect_from_video.py\n    -i <folder with video files or path to video file>\n    -m <path to model file>\n    -o <path to output folder, will write one or multiple output videos there>\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\nfrom os.path import join\nimport cv2\nimport dlib\nimport torch\nimport torch.nn as nn\nfrom PIL import Image as pil_image\nfrom tqdm.notebook import tqdm\n\n# from network.models import model_selection\n# from dataset.transform import xception_default_data_transforms\n\ndef get_boundingbox(face, width, height, scale=1.3, minsize=None):\n    \"\"\"\n    Expects a dlib face to generate a quadratic bounding box.\n    :param face: dlib face class\n    :param width: frame width\n    :param height: frame height\n    :param scale: bounding box size multiplier to get a bigger face region\n    :param minsize: set minimum bounding box size\n    :return: x, y, bounding_box_size in opencv form\n    \"\"\"\n    x1 = face.left()\n    y1 = face.top()\n    x2 = face.right()\n    y2 = face.bottom()\n    size_bb = int(max(x2 - x1, y2 - y1) * scale)\n    if minsize:\n        if size_bb < minsize:\n            size_bb = minsize\n    center_x, center_y = (x1 + x2) // 2, (y1 + y2) // 2\n\n    # Check for out of bounds, x-y top left corner\n    x1 = max(int(center_x - size_bb // 2), 0)\n    y1 = max(int(center_y - size_bb // 2), 0)\n    # Check for too big bb size for given x, y\n    size_bb = min(width - x1, size_bb)\n    size_bb = min(height - y1, size_bb)\n\n    return x1, y1, size_bb\n\n\ndef preprocess_image(image, cuda=True):\n    \"\"\"\n    Preprocesses the image such that it can be fed into our network.\n    During this process we envoke PIL to cast it into a PIL image.\n    :param image: numpy image in opencv form (i.e., BGR and of shape\n    :return: pytorch tensor of shape [1, 3, image_size, image_size], not\n    necessarily casted to cuda\n    \"\"\"\n    # Revert from BGR\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    # Preprocess using the preprocessing function used during training and\n    # casting it to PIL image\n    preprocess = xception_default_data_transforms['test']\n    preprocessed_image = preprocess(pil_image.fromarray(image))\n    # Add first dimension as the network expects a batch\n    preprocessed_image = preprocessed_image.unsqueeze(0)\n    if cuda:\n        preprocessed_image = preprocessed_image.cuda()\n    return preprocessed_image\n\n\ndef predict_with_model(image, model, post_function=nn.Softmax(dim=1),\n                       cuda=True):\n    \"\"\"\n    Predicts the label of an input image. Preprocesses the input image and\n    casts it to cuda if required\n    :param image: numpy image\n    :param model: torch model with linear layer at the end\n    :param post_function: e.g., softmax\n    :param cuda: enables cuda, must be the same parameter as the model\n    :return: prediction (1 = fake, 0 = real)\n    \"\"\"\n    # Preprocess\n    preprocessed_image = preprocess_image(image, cuda)\n\n    # Model prediction\n    output = model(preprocessed_image)\n    output = post_function(output)\n\n    # Cast to desired\n    _, prediction = torch.max(output, 1)    # argmax\n    prediction = float(prediction.cpu().numpy())\n\n    return int(prediction), output\n\ndef test_full_image_network(video_path, model, output_path,\n                            start_frame=0, end_frame=None, cuda=True):\n    \"\"\"\n    Reads a video and evaluates a subset of frames with the a detection network\n    that takes in a full frame. Outputs are only given if a face is present\n    and the face is highlighted using dlib.\n    :param video_path: path to video file\n    :param model_path: path to model file (should expect the full sized image)\n    :param output_path: path where the output video is stored\n    :param start_frame: first frame to evaluate\n    :param end_frame: last frame to evaluate\n    :param cuda: enable cuda\n    :return:\n    \n    # Modified to take in the model file instead of model\n    \"\"\"\n    #print('Starting: {}'.format(video_path))\n\n    # Read and write\n    reader = cv2.VideoCapture(video_path)\n\n    video_fn = video_path.split('/')[-1].split('.')[0]+'.avi'\n    os.makedirs(output_path, exist_ok=True)\n    fourcc = cv2.VideoWriter_fourcc(*'MJPG')\n    fps = reader.get(cv2.CAP_PROP_FPS)\n    num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT))\n    writer = None\n\n    # Face detector\n    face_detector = dlib.get_frontal_face_detector()\n\n    # Load model\n#     model, *_ = model_selection(modelname='xception', num_out_classes=2)\n#     if model_path is not None:\n#         model = torch.load(model_path)\n#         print('Model found in {}'.format(model_path))\n#     else:\n#         print('No model found, initializing random model.')\n#     if cuda:\n#         model = model.cuda()\n\n    # Text variables\n    font_face = cv2.FONT_HERSHEY_SIMPLEX\n    thickness = 2\n    font_scale = 1\n\n    # Frame numbers and length of output video\n    frame_num = 0\n    assert start_frame < num_frames - 1\n    end_frame = end_frame if end_frame else num_frames\n    pbar = tqdm(total=end_frame-start_frame)\n\n    while reader.isOpened():\n        _, image = reader.read()\n        if image is None:\n            break\n        frame_num += 1\n\n        if frame_num < start_frame:\n            continue\n        pbar.update(1)\n\n        # Image size\n#         print('getting image size')\n        height, width = image.shape[:2]\n\n        # Init output writer\n#         print('init output writer')\n        if writer is None:\n            writer = cv2.VideoWriter(join(output_path, video_fn), fourcc, fps,\n                                     (height, width)[::-1])\n\n        # 2. Detect with dlib\n#         print('detect with dlib')\n        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        faces = face_detector(gray, 1)\n        if len(faces):\n            # For now only take biggest face\n            face = faces[0]\n\n            # --- Prediction ---------------------------------------------------\n            # Face crop with dlib and bounding box scale enlargement\n            x, y, size = get_boundingbox(face, width, height)\n            cropped_face = image[y:y+size, x:x+size]\n\n            # Actual prediction using our model\n            prediction, output = predict_with_model(cropped_face, model,\n                                                    cuda=cuda)\n            # ------------------------------------------------------------------\n\n            # Text and bb\n            x = face.left()\n            y = face.top()\n            w = face.right() - x\n            h = face.bottom() - y\n            label = 'fake' if prediction == 1 else 'real'\n            color = (0, 255, 0) if prediction == 0 else (0, 0, 255)\n            output_list = ['{0:.2f}'.format(float(x)) for x in\n                           output.detach().cpu().numpy()[0]]\n            cv2.putText(image, str(output_list)+'=>'+label, (x, y+h+30),\n                        font_face, font_scale,\n                        color, thickness, 2)\n            # draw box over face\n            cv2.rectangle(image, (x, y), (x + w, y + h), color, 2)\n\n        if frame_num >= end_frame:\n            break\n\n        # Show\n#         print('show result')\n        # cv2.imshow('test', image)\n#         cv2.waitKey(33)     # About 30 fps\n        writer.write(image)\n    pbar.close()\n    if writer is not None:\n        writer.release()\n        #print('Finished! Output saved under {}'.format(output_path))\n    else:\n        pass\n        #print('Input video file was empty')\n    return","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:21.793339Z","iopub.execute_input":"2024-03-08T14:34:21.793714Z","iopub.status.idle":"2024-03-08T14:34:22.038315Z","shell.execute_reply.started":"2024-03-08T14:34:21.793629Z","shell.execute_reply":"2024-03-08T14:34:22.037561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Use the `classification` package per the github instructions\nReference: https://github.com/ondyari/FaceForensics/tree/master/classification#classification\n\nNow that we have the code available to us in the kernel, we will use the instructions as provided on the the github page.","metadata":{}},{"cell_type":"markdown","source":"# Write Files to disk for importing model\n- Since we are importing pretrained models, the code expects a `network` package with some files from the github repo\n- We will write these files to disk so the model import will run correctly.","metadata":{}},{"cell_type":"code","source":"!mkdir network","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:30.646549Z","iopub.execute_input":"2024-03-08T14:34:30.646885Z","iopub.status.idle":"2024-03-08T14:34:31.594392Z","shell.execute_reply.started":"2024-03-08T14:34:30.646823Z","shell.execute_reply":"2024-03-08T14:34:31.593397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/__init__.py\n# init","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:33.311267Z","iopub.execute_input":"2024-03-08T14:34:33.311596Z","iopub.status.idle":"2024-03-08T14:34:33.318198Z","shell.execute_reply.started":"2024-03-08T14:34:33.311535Z","shell.execute_reply":"2024-03-08T14:34:33.317281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/xception.py\n\"\"\"\nPorted to pytorch thanks to [tstandley](https://github.com/tstandley/Xception-PyTorch)\n@author: tstandley\nAdapted by cadene\nCreates an Xception Model as defined in:\nFrancois Chollet\nXception: Deep Learning with Depthwise Separable Convolutions\nhttps://arxiv.org/pdf/1610.02357.pdf\nThis weights ported from the Keras implementation. Achieves the following performance on the validation set:\nLoss:0.9173 Prec@1:78.892 Prec@5:94.292\nREMEMBER to set your image size to 3x299x299 for both test and validation\nnormalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],\n                                  std=[0.5, 0.5, 0.5])\nThe resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n\"\"\"\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom torch.nn import init\n\npretrained_settings = {\n    'xception': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1000,\n            'scale': 0.8975 # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n        }\n    }\n}\n\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):\n        super(SeparableConv2d,self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)\n        self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):\n        super(Block, self).__init__()\n\n        if out_filters != in_filters or strides!=1:\n            self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)\n            self.skipbn = nn.BatchNorm2d(out_filters)\n        else:\n            self.skip=None\n\n        self.relu = nn.ReLU(inplace=True)\n        rep=[]\n\n        filters=in_filters\n        if grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n            filters = out_filters\n\n        for i in range(reps-1):\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n\n        if not start_with_relu:\n            rep = rep[1:]\n        else:\n            rep[0] = nn.ReLU(inplace=False)\n\n        if strides != 1:\n            rep.append(nn.MaxPool2d(3,strides,1))\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self,inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x+=skip\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Xception optimized for the ImageNet dataset, as specified in\n    https://arxiv.org/pdf/1610.02357.pdf\n    \"\"\"\n    def __init__(self, num_classes=1000):\n        \"\"\" Constructor\n        Args:\n            num_classes: number of classes\n        \"\"\"\n        super(Xception, self).__init__()\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.relu = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32,64,3,bias=False)\n        self.bn2 = nn.BatchNorm2d(64)\n        #do relu here\n\n        self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)\n        self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)\n        self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)\n\n        self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)\n\n        self.conv3 = SeparableConv2d(1024,1536,3,1,1)\n        self.bn3 = nn.BatchNorm2d(1536)\n\n        #do relu here\n        self.conv4 = SeparableConv2d(1536,2048,3,1,1)\n        self.bn4 = nn.BatchNorm2d(2048)\n\n        self.fc = nn.Linear(2048, num_classes)\n\n        # #------- init weights --------\n        # for m in self.modules():\n        #     if isinstance(m, nn.Conv2d):\n        #         n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n        #         m.weight.data.normal_(0, math.sqrt(2. / n))\n        #     elif isinstance(m, nn.BatchNorm2d):\n        #         m.weight.data.fill_(1)\n        #         m.bias.data.zero_()\n        # #-----------------------------\n\n    def features(self, input):\n        x = self.conv1(input)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        return x\n\n    def logits(self, features):\n        x = self.relu(features)\n\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef xception(num_classes=1000, pretrained='imagenet'):\n    model = Xception(num_classes=num_classes)\n    if pretrained:\n        settings = pretrained_settings['xception'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        model = Xception(num_classes=num_classes)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n\n    # TODO: ugly\n    model.last_linear = model.fc\n    del model.fc\n    return model","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:35.882039Z","iopub.execute_input":"2024-03-08T14:34:35.882364Z","iopub.status.idle":"2024-03-08T14:34:35.889367Z","shell.execute_reply.started":"2024-03-08T14:34:35.882315Z","shell.execute_reply":"2024-03-08T14:34:35.888644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/models.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\n\n\nimport torch\n#import pretrainedmodels\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom network.xception import xception\nimport math\nimport torchvision\n\n\ndef return_pytorch04_xception(pretrained=True):\n    # Raises warning \"src not broadcastable to dst\" but thats fine\n    model = xception(pretrained=False)\n    if pretrained:\n        # Load model in torch 0.4+\n        model.fc = model.last_linear\n        del model.last_linear\n        state_dict = torch.load(\n            '/home/ondyari/.torch/models/xception-b5690688.pth')\n        for name, weights in state_dict.items():\n            if 'pointwise' in name:\n                state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)\n        model.load_state_dict(state_dict)\n        model.last_linear = model.fc\n        del model.fc\n    return model\n\n\nclass TransferModel(nn.Module):\n    \"\"\"\n    Simple transfer learning model that takes an imagenet pretrained model with\n    a fc layer as base model and retrains a new fc layer for num_out_classes\n    \"\"\"\n    def __init__(self, modelchoice, num_out_classes=2, dropout=0.0):\n        super(TransferModel, self).__init__()\n        self.modelchoice = modelchoice\n        if modelchoice == 'xception':\n            self.model = return_pytorch04_xception()\n            # Replace fc\n            num_ftrs = self.model.last_linear.in_features\n            if not dropout:\n                self.model.last_linear = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                print('Using dropout', dropout)\n                self.model.last_linear = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        elif modelchoice == 'resnet50' or modelchoice == 'resnet18':\n            if modelchoice == 'resnet50':\n                self.model = torchvision.models.resnet50(pretrained=True)\n            if modelchoice == 'resnet18':\n                self.model = torchvision.models.resnet18(pretrained=True)\n            # Replace fc\n            num_ftrs = self.model.fc.in_features\n            if not dropout:\n                self.model.fc = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                self.model.fc = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        else:\n            raise Exception('Choose valid model, e.g. resnet50')\n\n    def set_trainable_up_to(self, boolean, layername=\"Conv2d_4a_3x3\"):\n        \"\"\"\n        Freezes all layers below a specific layer and sets the following layers\n        to true if boolean else only the fully connected final layer\n        :param boolean:\n        :param layername: depends on network, for inception e.g. Conv2d_4a_3x3\n        :return:\n        \"\"\"\n        # Stage-1: freeze all the layers\n        if layername is None:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = True\n                return\n        else:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = False\n        if boolean:\n            # Make all layers following the layername layer trainable\n            ct = []\n            found = False\n            for name, child in self.model.named_children():\n                if layername in ct:\n                    found = True\n                    for params in child.parameters():\n                        params.requires_grad = True\n                ct.append(name)\n            if not found:\n                raise Exception('Layer not found, cant finetune!'.format(\n                    layername))\n        else:\n            if self.modelchoice == 'xception':\n                # Make fc trainable\n                for param in self.model.last_linear.parameters():\n                    param.requires_grad = True\n\n            else:\n                # Make fc trainable\n                for param in self.model.fc.parameters():\n                    param.requires_grad = True\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n\ndef model_selection(modelname, num_out_classes,\n                    dropout=None):\n    \"\"\"\n    :param modelname:\n    :return: model, image size, pretraining<yes/no>, input_list\n    \"\"\"\n    if modelname == 'xception':\n        return TransferModel(modelchoice='xception',\n                             num_out_classes=num_out_classes), 299, \\\n               True, ['image'], None\n    elif modelname == 'resnet18':\n        return TransferModel(modelchoice='resnet18', dropout=dropout,\n                             num_out_classes=num_out_classes), \\\n               224, True, ['image'], None\n    else:\n        raise NotImplementedError(modelname)\n\n\nif __name__ == '__main__':\n    model, image_size, *_ = model_selection('resnet18', num_out_classes=2)\n    print(model)\n    model = model.cuda()\n    from torchsummary import summary\n    input_s = (3, image_size, image_size)\n    print(summary(model, input_s))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:38.207897Z","iopub.execute_input":"2024-03-08T14:34:38.208257Z","iopub.status.idle":"2024-03-08T14:34:38.217607Z","shell.execute_reply.started":"2024-03-08T14:34:38.208203Z","shell.execute_reply":"2024-03-08T14:34:38.216679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained Models from FaceForensics++\nPretrained models were downloaded from the referenced website here:\nhttp://kaldir.vc.in.tum.de/FaceForensics/models/faceforensics++_models.zip\n\nWe can load the model files using torch like this on cpu:\n```\nmodel = torch.load(model_path, map_location=torch.device('cpu'))\n```\nor with gpu/cuda:\n```\nmodel = torch.load(model_path)\nmodel = model.cuda()\n```\n\n## The authors provide different types of models:\nFrom Appendix 4 of the [paper](https://arxiv.org/pdf/1901.08971.pdf) we see the published accuracy:\n![](https://i.imgur.com/ebVvY9A.png)\n\nWe have:\n- Full image models\n- Face_detected models\n- RAW, compressed 23, and compressed 40","metadata":{}},{"cell_type":"markdown","source":"## Create a prediction function.\n- This function is provided a video file, and runs the `test_full_image_network`\n- Takes in the model file.\n- Save the output avi file\n- Display some of the frames of the output video","metadata":{}},{"cell_type":"code","source":"metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\n\ndef predict_model(video_fn, model,\n                  start_frame=0, end_frame=30,\n                  plot_every_x_frames = 5):\n    \"\"\"\n    Given a video and model, starting frame and end frame.\n    Predict on all frames.\n    \n    \"\"\"\n    fn = video_fn.split('.')[0]\n    label = metadata.loc[video_fn]['label']\n    original = metadata.loc[video_fn]['original']\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    output_path = './'\n    test_full_image_network(video_path, model, output_path, start_frame=0, end_frame=30, cuda=False)\n    # Read output\n    vidcap = cv2.VideoCapture(f'{fn}.avi')\n    success,image = vidcap.read()\n    count = 0\n    fig, axes = plt.subplots(3, 2, figsize=(20, 15))\n    axes = axes = axes.flatten()\n    i = 0\n    while success:\n        # Show every xth frame\n        if count % plot_every_x_frames == 0:\n\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            axes[i].imshow(image)\n            axes[i].set_title(f'{fn} - frame {count} - true label: {label}')\n            axes[i].xaxis.set_visible(False)\n            axes[i].yaxis.set_visible(False)\n            i += 1\n        success,image = vidcap.read()\n        count += 1\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-08T14:34:42.560432Z","iopub.execute_input":"2024-03-08T14:34:42.560775Z","iopub.status.idle":"2024-03-08T14:34:43.058380Z","shell.execute_reply.started":"2024-03-08T14:34:42.560712Z","shell.execute_reply":"2024-03-08T14:34:43.057602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing Full Image Models\n## full_raw.p (full resolution)","metadata":{}},{"cell_type":"code","source":"model_path = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_raw.p'\nmodel = torch.load(model_path, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T14:34:46.621368Z","iopub.execute_input":"2024-03-08T14:34:46.621702Z","iopub.status.idle":"2024-03-08T14:34:48.847688Z","shell.execute_reply.started":"2024-03-08T14:34:46.621644Z","shell.execute_reply":"2024-03-08T14:34:48.846932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model)\npredict_model('crezycjqyk.mp4', model)\npredict_model('ebchwmwayp.mp4', model)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T14:44:21.805744Z","iopub.execute_input":"2024-03-08T14:44:21.806090Z","iopub.status.idle":"2024-03-08T14:46:22.761391Z","shell.execute_reply.started":"2024-03-08T14:44:21.806039Z","shell.execute_reply":"2024-03-08T14:46:22.760431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## full_c40.p model","metadata":{}},{"cell_type":"code","source":"model_path_full_c40 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_c40.p'\nmodel_full_c40 = torch.load(model_path_full_c40, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:06:38.245183Z","iopub.execute_input":"2024-03-08T15:06:38.245507Z","iopub.status.idle":"2024-03-08T15:06:39.760522Z","shell.execute_reply.started":"2024-03-08T15:06:38.245461Z","shell.execute_reply":"2024-03-08T15:06:39.759542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_full_c40)\npredict_model('crezycjqyk.mp4', model_full_c40)\npredict_model('ebchwmwayp.mp4', model_full_c40)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:06:42.317773Z","iopub.execute_input":"2024-03-08T15:06:42.318086Z","iopub.status.idle":"2024-03-08T15:08:42.030432Z","shell.execute_reply.started":"2024-03-08T15:06:42.318040Z","shell.execute_reply":"2024-03-08T15:08:42.029540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## full_c23.p model","metadata":{}},{"cell_type":"code","source":"model_path_full23 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_c23.p'\nmodel_full_c23 = torch.load(model_path_full23, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:09:18.829240Z","iopub.execute_input":"2024-03-08T15:09:18.829579Z","iopub.status.idle":"2024-03-08T15:09:20.260099Z","shell.execute_reply.started":"2024-03-08T15:09:18.829518Z","shell.execute_reply":"2024-03-08T15:09:20.259242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_full_c23)\npredict_model('crezycjqyk.mp4', model_full_c23)\npredict_model('ebchwmwayp.mp4', model_full_c23)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:09:21.809949Z","iopub.execute_input":"2024-03-08T15:09:21.810267Z","iopub.status.idle":"2024-03-08T15:11:22.260285Z","shell.execute_reply.started":"2024-03-08T15:09:21.810221Z","shell.execute_reply":"2024-03-08T15:11:22.259262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing face_detection/xcepion models\nThese models are specific to detecting just the face.\n## face_detection/xcepion all_raw.p model\n","metadata":{}},{"cell_type":"code","source":"model_path_face_allraw = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_raw.p'\nmodel_face_allraw = torch.load(model_path_face_allraw, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:12:28.045326Z","iopub.execute_input":"2024-03-08T15:12:28.045659Z","iopub.status.idle":"2024-03-08T15:12:29.683813Z","shell.execute_reply.started":"2024-03-08T15:12:28.045606Z","shell.execute_reply":"2024-03-08T15:12:29.682874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_face_allraw)\npredict_model('crezycjqyk.mp4', model_face_allraw)\npredict_model('ebchwmwayp.mp4', model_face_allraw)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:12:32.745644Z","iopub.execute_input":"2024-03-08T15:12:32.745982Z","iopub.status.idle":"2024-03-08T15:14:34.844968Z","shell.execute_reply.started":"2024-03-08T15:12:32.745923Z","shell.execute_reply":"2024-03-08T15:14:34.843880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  face_detection/xcepion all_c40.p model","metadata":{}},{"cell_type":"code","source":"model_path_face_all_c40 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_c40.p'\nmodel_face_all_c40 = torch.load(model_path_face_all_c40, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:15:47.401964Z","iopub.execute_input":"2024-03-08T15:15:47.402291Z","iopub.status.idle":"2024-03-08T15:15:48.836819Z","shell.execute_reply.started":"2024-03-08T15:15:47.402245Z","shell.execute_reply":"2024-03-08T15:15:48.836143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_face_all_c40)\npredict_model('crezycjqyk.mp4', model_face_all_c40)\npredict_model('ebchwmwayp.mp4', model_face_all_c40)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:15:50.425582Z","iopub.execute_input":"2024-03-08T15:15:50.425886Z","iopub.status.idle":"2024-03-08T15:17:51.619769Z","shell.execute_reply.started":"2024-03-08T15:15:50.425841Z","shell.execute_reply":"2024-03-08T15:17:51.618708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  face_detection/xcepion all_c23.p model","metadata":{}},{"cell_type":"code","source":"model, *_ = model_selection(modelname='xception', num_out_classes=2)\nmodel_path_face_allc23 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_c23.p'\nmodel_face_all_c23 = torch.load(model_path_face_allc23, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:17:56.833653Z","iopub.execute_input":"2024-03-08T15:17:56.833950Z","iopub.status.idle":"2024-03-08T15:18:00.946415Z","shell.execute_reply.started":"2024-03-08T15:17:56.833906Z","shell.execute_reply":"2024-03-08T15:18:00.945479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_face_all_c23)\npredict_model('crezycjqyk.mp4', model_face_all_c23)\npredict_model('ebchwmwayp.mp4', model_face_all_c23)","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:18:07.351347Z","iopub.execute_input":"2024-03-08T15:18:07.351782Z","iopub.status.idle":"2024-03-08T15:20:09.111018Z","shell.execute_reply.started":"2024-03-08T15:18:07.351712Z","shell.execute_reply":"2024-03-08T15:20:09.109918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validate Predictions on train set\n- Predict for 50 frames\n- Average the prediction of these frames to make a single prediction\n- Test for some FAKE and REAL\n- Return the maximum, minimum and average \"fake\" prediction of all the fames sampled\n- If unable to detect a face, predict 0.5","metadata":{}},{"cell_type":"code","source":"def video_file_frame_pred(video_path, model,\n                          start_frame=0, end_frame=300,\n                          cuda=True, n_frames=5):\n    \"\"\"\n    Predict and give result as numpy array\n    \"\"\"\n    pred_frames = [int(round(x)) for x in np.linspace(start_frame, end_frame, n_frames)]\n    predictions = []\n    outputs = []\n    # print('Starting: {}'.format(video_path))\n\n    # Read and write\n    reader = cv2.VideoCapture(video_path)\n\n    video_fn = video_path.split('/')[-1].split('.')[0]+'.avi'\n    fourcc = cv2.VideoWriter_fourcc(*'MJPG')\n    fps = reader.get(cv2.CAP_PROP_FPS)\n    num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT))\n    writer = None\n\n    # Face detector\n    face_detector = dlib.get_frontal_face_detector()\n\n    # Text variables\n    font_face = cv2.FONT_HERSHEY_SIMPLEX\n    thickness = 2\n    font_scale = 1\n\n    # Frame numbers and length of output video\n    frame_num = 0\n    assert start_frame < num_frames - 1\n    end_frame = end_frame if end_frame else num_frames\n    while reader.isOpened():\n        _, image = reader.read()\n        if image is None:\n            break\n        frame_num += 1\n        if frame_num in pred_frames:\n            height, width = image.shape[:2]\n            # 2. Detect with dlib\n            gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n            faces = face_detector(gray, 1)\n            if len(faces):\n                # For now only take biggest face\n                face = faces[0]\n                # --- Prediction ---------------------------------------------------\n                # Face crop with dlib and bounding box scale enlargement\n                x, y, size = get_boundingbox(face, width, height)\n                cropped_face = image[y:y+size, x:x+size]\n\n                # Actual prediction using our model\n                prediction, output = predict_with_model(cropped_face, model,\n                                                        cuda=cuda)\n                predictions.append(prediction)\n                outputs.append(output)\n                # ------------------------------------------------------------------\n        if frame_num >= end_frame:\n            break\n    # Figure out how to do this with torch\n    preds_np = [x.detach().cpu().numpy()[0][1] for x in outputs]\n    if len(preds_np) == 0:\n        return predictions, outputs, 0.5, 0.5, 0.5\n    try:\n        mean_pred = np.mean(preds_np)\n    except:\n        # couldnt find faces\n        mean_pred = 0.5\n    min_pred = np.min(preds_np)\n    max_pred = np.max(preds_np)\n    return predictions, outputs, mean_pred, min_pred, max_pred","metadata":{"execution":{"iopub.status.busy":"2024-03-08T16:21:01.092307Z","iopub.execute_input":"2024-03-08T16:21:01.092644Z","iopub.status.idle":"2024-03-08T16:21:01.111461Z","shell.execute_reply.started":"2024-03-08T16:21:01.092598Z","shell.execute_reply":"2024-03-08T16:21:01.110600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Because the face_detection/xception/all_c23.p model seemed to be returning the best restuls in our testing, we will use it for running","metadata":{}},{"cell_type":"code","source":"torch.nn.Module.dump_patches = True\nmodel_path_23 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_c23.p'\nmodel_23 = torch.load(model_path_23, map_location=torch.device('cpu'))\nmodel_path_raw = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_raw.p'\nmodel_raw = torch.load(model_path_raw, map_location=torch.device('cpu'))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-08T15:20:17.189270Z","iopub.execute_input":"2024-03-08T15:20:17.189878Z","iopub.status.idle":"2024-03-08T15:20:17.399077Z","shell.execute_reply.started":"2024-03-08T15:20:17.189619Z","shell.execute_reply":"2024-03-08T15:20:17.398092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read metadata\nmetadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\n\n# Predict Fake\nfor video_fn in tqdm(metadata.query('label == \"FAKE\"').sample(77).index):\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_23, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_c23'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_c23'] = min_pred\n    metadata.loc[video_fn, 'max_pred_c23'] = max_pred\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_raw, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_raw'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_raw'] = min_pred\n    metadata.loc[video_fn, 'max_pred_raw'] = max_pred\n    \n# Predict Real\nfor video_fn in tqdm(metadata.query('label == \"REAL\"').sample(77).index):\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_23, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_c23'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_c23'] = min_pred\n    metadata.loc[video_fn, 'max_pred_c23'] = max_pred\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_raw, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_raw'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_raw'] = min_pred\n    metadata.loc[video_fn, 'max_pred_raw'] = max_pred","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:20:29.504294Z","iopub.execute_input":"2024-03-08T15:20:29.504605Z","iopub.status.idle":"2024-03-08T15:49:08.736390Z","shell.execute_reply.started":"2024-03-08T15:20:29.504553Z","shell.execute_reply":"2024-03-08T15:49:08.735453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute estimated score for 154 training samples","metadata":{}},{"cell_type":"code","source":"preds_df = metadata.dropna(subset=['avg_pred_raw']).copy()\npreds_df['label_binary'] = 0\npreds_df.loc[preds_df['label'] == \"FAKE\", 'label_binary'] = 1\npreds_df[['min_pred_c23','max_pred_c23',\n          'min_pred_raw','max_pred_raw']] = preds_df[['min_pred_c23','max_pred_c23',\n                                                      'min_pred_raw','max_pred_raw']].fillna(0.5)\npreds_df['naive_pred'] = 0.5\nscore_avg23 = log_loss(preds_df['label_binary'], preds_df['avg_pred_c23'])\nscore_min23 = log_loss(preds_df['label_binary'], preds_df['min_pred_c23'])\nscore_max23 = log_loss(preds_df['label_binary'], preds_df['max_pred_c23'])\nscore_avgraw = log_loss(preds_df['label_binary'], preds_df['avg_pred_raw'])\nscore_minraw = log_loss(preds_df['label_binary'], preds_df['min_pred_raw'])\nscore_maxraw = log_loss(preds_df['label_binary'], preds_df['max_pred_raw'])\nscore_naive = log_loss(preds_df['label_binary'], preds_df['naive_pred'])\npreds_df['max_pred_clipped'] = preds_df['max_pred_c23'].clip(0.4, 1)\nscore_max_clipped = log_loss(preds_df['label_binary'], preds_df['max_pred_clipped'])\npreds_df['max_pred_clipped_raw'] = preds_df['max_pred_raw'].clip(0.4, 1)\nscore_max_clipped_raw = log_loss(preds_df['label_binary'], preds_df['max_pred_clipped_raw'])\nprint('Score using average prediction of all frames all_c23.p: {:0.4f}'.format(score_avg23))\nprint('Score using minimum prediction of all frames all_c23.p: {:0.4f}'.format(score_min23))\nprint('Score using maximum prediction of all frames all_c23.p: {:0.4f}'.format(score_max23))\nprint('Score using 0.5 prediction of all frames: {:0.4f}'.format(score_naive))\nprint('Score using maximum clipped prediction of all frames: {:0.4f}'.format(score_max_clipped))\nprint('Score using average prediction of all frames all_raw.p: {:0.4f}'.format(score_avgraw))\nprint('Score using minimum prediction of all frames all_raw.p: {:0.4f}'.format(score_minraw))\nprint('Score using maximum prediction of all frames all_raw.p: {:0.4f}'.format(score_maxraw))\nprint('Score using maximum clipped prediction of all frames all_raw.p: {:0.4f}'.format(score_max_clipped_raw))","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:13.947881Z","iopub.execute_input":"2024-03-08T15:49:13.948249Z","iopub.status.idle":"2024-03-08T15:49:13.998450Z","shell.execute_reply.started":"2024-03-08T15:49:13.948195Z","shell.execute_reply":"2024-03-08T15:49:13.997191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot the Average vs Max Prediction Probability - Fake vs Real\nThe model appears to preform fairly poorly but shows that it is picking up on some signal.","metadata":{}},{"cell_type":"markdown","source":"## c23 model results","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(10, 10))\nsns.scatterplot(x='avg_pred_c23', y='max_pred_c23', data=metadata.dropna(subset=['avg_pred_c23']), hue='label')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:17.445469Z","iopub.execute_input":"2024-03-08T15:49:17.445775Z","iopub.status.idle":"2024-03-08T15:49:17.956284Z","shell.execute_reply.started":"2024-03-08T15:49:17.445729Z","shell.execute_reply":"2024-03-08T15:49:17.955184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## raw model results","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(10, 10))\nsns.scatterplot(x='avg_pred_raw', y='max_pred_raw', data=metadata.dropna(subset=['avg_pred_raw']), hue='label')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:21.675694Z","iopub.execute_input":"2024-03-08T15:49:21.675991Z","iopub.status.idle":"2024-03-08T15:49:22.059401Z","shell.execute_reply.started":"2024-03-08T15:49:21.675949Z","shell.execute_reply":"2024-03-08T15:49:22.058185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, d in metadata.groupby('label'):\n    d['avg_pred_c23'].plot(kind='hist', figsize=(15, 5), bins=20, alpha=0.8, title='Average Prediction distribution c23')\n    plt.legend(['FAKE','REAL'])\nplt.show()\nfor i, d in metadata.groupby('label'):\n    d['max_pred_c23'].plot(kind='hist', figsize=(15, 5), bins=20, title='Max Prediction distribution c23', alpha=0.8)\n    plt.legend(['FAKE','REAL'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:24.626414Z","iopub.execute_input":"2024-03-08T15:49:24.626759Z","iopub.status.idle":"2024-03-08T15:49:25.267075Z","shell.execute_reply.started":"2024-03-08T15:49:24.626697Z","shell.execute_reply":"2024-03-08T15:49:25.266330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, d in metadata.groupby('label'):\n    d['avg_pred_raw'].plot(kind='hist',\n                           figsize=(15, 5),\n                           bins=20,\n                           alpha=0.8,\n                           title='Average Prediction distribution raw')\n    plt.legend(['FAKE','REAL'])\nplt.show()\nfor i, d in metadata.groupby('label'):\n    d['max_pred_raw'].plot(kind='hist',\n                           figsize=(15, 5),\n                           bins=20,\n                           title='Max Prediction distribution raw',\n                           alpha=0.8)\n    plt.legend(['FAKE','REAL'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:28.186707Z","iopub.execute_input":"2024-03-08T15:49:28.187077Z","iopub.status.idle":"2024-03-08T15:49:29.070076Z","shell.execute_reply.started":"2024-03-08T15:49:28.186992Z","shell.execute_reply":"2024-03-08T15:49:29.068793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata['max_pred_c23'] = metadata['max_pred_c23'].round(6)\nmetadata.dropna(subset=['max_pred_c23']).sort_values('label')","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:32.601608Z","iopub.execute_input":"2024-03-08T15:49:32.601914Z","iopub.status.idle":"2024-03-08T15:49:32.631892Z","shell.execute_reply.started":"2024-03-08T15:49:32.601867Z","shell.execute_reply":"2024-03-08T15:49:32.631098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata['label_binary'] = 0\nmetadata.loc[metadata['label'] == \"FAKE\", 'label_binary'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:35.468771Z","iopub.execute_input":"2024-03-08T15:49:35.469134Z","iopub.status.idle":"2024-03-08T15:49:35.477503Z","shell.execute_reply.started":"2024-03-08T15:49:35.469066Z","shell.execute_reply":"2024-03-08T15:49:35.476582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test set.\n- Predict for a subset of frames per video to cut down on run time\n- Save the min/max and average prediction of all frames\n- Save results to sample submission CSV.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nss = pd.read_csv('../input/deepfake-detection-challenge/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-08T15:49:40.133407Z","iopub.execute_input":"2024-03-08T15:49:40.133717Z","iopub.status.idle":"2024-03-08T15:49:40.146564Z","shell.execute_reply.started":"2024-03-08T15:49:40.133659Z","shell.execute_reply":"2024-03-08T15:49:40.145859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_fn in tqdm(ss['filename'].unique()):\n    video_path = f'../input/deepfake-detection-challenge/test_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model, n_frames=4, cuda=False)\n    ss.loc[ss['filename'] == video_fn, 'avg_pred'] = mean_pred\n    ss.loc[ss['filename'] == video_fn, 'min_pred'] = min_pred\n    ss.loc[ss['filename'] == video_fn, 'max_pred'] = max_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use the Maximum frame predicted as \"Fake\" to be the final prediction\nss['label'] = ss['max_pred'].fillna(0.5).clip(0.4, 0.8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss['label'].plot(kind='hist', figsize=(15, 5), bins=50)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['filename','label']].to_csv('submission.csv', index=False)\nss.to_csv('submission_min_max.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}