{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n%%capture\n# Install facenet-pytorch (with internet use \"pip install facenet-pytorch\")\n#!pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-2.2.7-py3-none-any.whl\n#!cp /kaggle/input/decord/install.sh . && chmod  +x install.sh && ./install.sh \n#!pip install /kaggle/input/dfdcpackages/dlib-19.19.0-cp36-cp36m-linux_x86_64.whl\n#!pip install /kaggle/input/imutils/imutils-0.5.3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys, time\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport random\nfrom PIL import ImageFilter, Image\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n#from facenet_pytorch import MTCNN\nimport torchvision\n\nsys.path.append('../input/efficientnet')\nsys.path.append('../input/imutils/imutils-0.5.3')\nsys.path.append('../input/dsfdinference/')\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\nsys.path.insert(0, \"/kaggle/input/helpers\")\nsys.path.insert(0, \"/kaggle/input/timmmodels\")\nsys.path.insert(0,'/kaggle/working/reader/python')\n\nimport timm\n\n#from decord import VideoReader as decord_VideoReader\n#from decord import cpu, gpu\n#from decord.bridge import set_bridge\nfrom imutils.video import FileVideoStream \nfrom efficientnet import EfficientNet\n#from dsfd.detect import DSFDDetector, get_face_detections\n\nrandom.seed(0)\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\ntorch.backends.cudnn.deterministic = True\n\n%matplotlib inline\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the test videos"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"PyTorch version:\", torch.__version__)\nprint(\"CUDA version:\", torch.version.cuda)\nprint(\"cuDNN version:\", torch.backends.cudnn.version())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n#set_bridge('torch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from blazeface import BlazeFace\nfacedet = BlazeFace().to(device)\nfacedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\nfacedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n_ = facedet.train(False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_size = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import Normalize\n\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nnormalize_transform = Normalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def disable_grad(model):\n    for parameter in model.parameters():\n        parameter.requires_grad = False\n        \n    return model\n\n\ndef normalize(img):\n    y, x, _ = img.shape\n    \n    if y > x and x < 256:\n        ratio_x = x / y\n        ratio_y = y / x\n\n        return cv2.resize(img, (256, int(ratio_y * 256)))\n    elif y < x and y < 256:\n        ratio_x = x / y\n        ratio_y = y / x\n\n        return cv2.resize(img, (int(ratio_x * 256), 256))\n    else:\n        return cv2.resize(img, (256, 256))\n        \n\ndef weight_preds(preds, weights):\n    final_preds = []\n    for i in range(len(preds)):\n        for j in range(len(preds[i])):\n            if len(final_preds) != len(preds[i]):\n                final_preds.append(preds[i][j] * weights[i])\n            else:\n                final_preds[j] += preds[i][j] * weights[i]\n                \n    return torch.FloatTensor(final_preds)\n\n\ndef predict_faces(models, x, weigths, n):\n    x = torch.tensor(x, device=device).float()\n\n    # Preprocess the images.\n    x = x.permute((0, 3, 1, 2))\n\n    for i in range(len(x)):\n        x[i] = normalize_transform(x[i] / 255.)\n\n    # Make a prediction, then take the average.\n    with torch.no_grad():\n        y_pred = 0\n        preds = []\n        for i in range(len(models)):\n            preds.append(models[i](x).squeeze()[:n])\n        \n        del x\n        \n        y_pred = torch.sigmoid(weight_preds(preds, weigths)).mean().item()\n\n        return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from helpers.read_video_1 import VideoReader\nfrom helpers.face_extract_1 import FaceExtractor\n\nframes_per_video = 32\n\nvideo_reader = VideoReader()\nvideo_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video) #get_frames(x, batch_size=frames_per_video)\nface_extractor = FaceExtractor(video_read_fn, facedet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''import tensorflow as tf\ndetection_graph = tf.Graph()\nwith detection_graph.as_default():\n    od_graph_def = tf.compat.v1.GraphDef()\n    with tf.io.gfile.GFile('../input/mobilenet-face/frozen_inference_graph_face.pb', 'rb') as fid:\n        serialized_graph = fid.read()\n        od_graph_def.ParseFromString(serialized_graph)\n        tf.import_graph_def(od_graph_def, name='')\n        config = tf.compat.v1.ConfigProto()\n    config.gpu_options.allow_growth = True\n    sess=tf.compat.v1.Session(graph=detection_graph, config=config)\n    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\n    boxes_tensor = detection_graph.get_tensor_by_name('detection_boxes:0')    \n    scores_tensor = detection_graph.get_tensor_by_name('detection_scores:0')\n    num_detections = detection_graph.get_tensor_by_name('num_detections:0')\n    \n    \ndef get_mobilenet_face(image):\n    global boxes,scores,num_detections\n    (im_height,im_width)=image.shape[:-1]\n    imgs=np.array([image])\n    (boxes, scores) = sess.run(\n        [boxes_tensor, scores_tensor],\n        feed_dict={image_tensor: imgs})\n    max_=np.where(scores==scores.max())[0][0]\n    box=boxes[0][max_]\n    ymin, xmin, ymax, xmax = box\n    (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n                                ymin * im_height, ymax * im_height)\n    left, right, top, bottom = int(left), int(right), int(top), int(bottom)\n    return (left, right, top, bottom)\n\ndef crop_image(frame,bbox):\n    left, right, top, bottom=bbox\n    return frame[top:bottom,left:right]'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MetaModel(nn.Module):\n    def __init__(self, models=None, device='cuda:0', extended=False):\n        super(MetaModel, self).__init__()\n        \n        self.extended = extended\n        self.device = device\n        self.models = models\n        self.len = len(models)\n        \n        if self.extended:\n            self.bn = nn.BatchNorm1d(self.len)\n            self.relu = nn.ReLU()\n            self.dropout = nn.Dropout(0.2)\n\n        self.fc = nn.Linear(self.len, 1)\n        \n    def forward(self, x):\n        x = torch.cat(tuple(x), dim=1)\n        \n        if self.extended:\n            x = self.bn(x)\n            x = self.relu(x)\n            #x = self.dropout(x)\n            \n        x = self.fc(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ensemble configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_PATH = \"/kaggle/input/deepfake-detection-model-20k/\"\nWEIGTHS_EXT = '.pth'\n\nmodels = []\nweigths = []\n    \nraw_data_stack = \\\n[\n    ['0.8548137313946486 0.3376769562025044', 'efficientnet-b2'],\n    ['EfficientNetb3 0.8573518024606384 0.34558522378585194', 'efficientnet-b3'],\n    ['EfficientNetb4 0.8579110384582294 0.3383911053075265', 'efficientnet-b4'],\n    ['EfficientNet6 0.8602770369095758 0.33193617861157143', 'efficientnet-b6'],\n    ['EfficientNetb0 t2 0.8616966359803837 0.3698434531609828', 'efficientnet-b0'],\n    ['EfficientNetb1 t2 0.8410909403768391 0.36058002083572327', 'efficientnet-b1'],\n    ['EfficientNetb2 t2 0.8659554331928073 0.35598630783834084', 'efficientnet-b2'],\n    ['EfficientNetb3 t2 0.8486191172674868 0.3611779548592305', 'efficientnet-b3'],\n    ['EfficientNetb3 0.8635894347414609 0.328333642473084', 'efficientnet-b3'],\n    ['EfficientNetb6 0.8593736556826981 0.32286693639934694', 'efficientnet-b6'],\n    ['tf_efficientnet_b1_ns 0.8571367116923342 0.3341234226295108', 'tf_efficientnet_b1_ns'],\n    ['tf_efficientnet_b3_ns 0.8712466660930913 0.3277394129117183', 'tf_efficientnet_b3_ns'],\n    ['tf_efficientnet_b4_ns 0.8708595027101437 0.3152573955405342', 'tf_efficientnet_b4_ns'],\n    ['tf_efficientnet_b6_ns 0.8733115374688118 0.3156576980666498', 'tf_efficientnet_b6_ns'],\n]\n\nstack_models = []\nfor raw_model in raw_data_stack:\n    checkpoint = torch.load( MODELS_PATH + raw_model[0] + WEIGTHS_EXT, map_location=device)\n    \n    if '-' in raw_model[1]:\n        model = EfficientNet.from_name(raw_model[1])\n        model._fc = nn.Linear(model._fc.in_features, 1)\n    else:\n        model = timm.create_model(raw_model[1], pretrained=False)\n        model.classifier = nn.Linear(model.classifier.in_features, 1)\n    \n    model.load_state_dict(checkpoint)\n    _ = model.eval()\n    _ = disable_grad(model)\n    model = model.to(device)\n    stack_models.append(model)\n\n    del checkpoint, model\n    \n\nmeta_models = \\\n[\n    ['MetaModel 0.30638167556896007', slice(4, 8), False, 0.37780],\n    ['MetaModel 0.2919331893755284', slice(0, 4), False, 0.33357],\n    ['MetaModel 0.30281482560578044', slice(0, 8, None), True, 0.34077],\n    ['MetaModel 0.26302117601197256', slice(0, 10, None), False, 0.35134],\n    ['MetaModel 0.256337642808031', slice(10, 14, None), False, 0.32698],\n    ['MetaModel 0.264787397152165', slice(0, 14, None), False, 0.34974]\n]\n\nfor meta_raw in meta_models:\n\n    checkpoint = torch.load(MODELS_PATH + meta_raw[0] + WEIGTHS_EXT, map_location=device)\n    \n    model = MetaModel(models=raw_data_stack[meta_raw[1]], extended=meta_raw[2]).to(device)\n    #model = MetaModel(models=stack_models[meta_raw[1]], extended=meta_raw[2]).to(device)\n    \n    model.load_state_dict(checkpoint)\n    _ = model.eval()\n    _ = disable_grad(model)\n    model.to(device)\n    models.append(model)\n    weigths.append(meta_raw[3])\n\n    del model, checkpoint\n    \ntotal = sum([1-score for score in weigths])\nweigths = [(1-score) / total for score in weigths]\n\n'''checkpoint = torch.load(MODELS_PATH + 'MetaModel 0.256337642808031.pth', map_location=device)\nmeta = MetaModel(stack_models).to(device)\nmeta.load_state_dict(checkpoint)\n_ = meta.eval()\n_ = disable_grad(meta)\n\ndel checkpoint'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"from random import randint\ndef predict_on_video(video_path, batch_size):\n    try:\n        # Find the faces for N frames in the video.\n        faces = face_extractor.process_video(video_path)\n\n        # Only look at one face per frame.\n        face_extractor.keep_only_best_face(faces)\n\n        if len(faces) > 0:\n            # NOTE: When running on the CPU, the batch size must be fixed\n            # or else memory usage will blow up. (Bug in PyTorch?)\n            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n\n            # If we found any faces, prepare them for the model.\n            n = 0\n            for frame_data in faces:\n                for face in frame_data[\"faces\"]:\n                    # Resize to the model's required input size.\n                    # We keep the aspect ratio intact and add zero\n                    # padding if necessary.\n\n                    #resized_face = isotropically_resize_image(face, input_size)\n                    #resized_face = make_square_image(resized_face)\n                    \n                    resized_face = normalize(face)\n                    resized_face = torchvision.transforms.CenterCrop((input_size, input_size))(Image.fromarray(resized_face))\n                    #resized_face = cv2.resize(face, (input_size, input_size))\n                    \n                    if n < batch_size:\n                        x[n] = resized_face\n                        n += 1\n                    else:\n                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n\n                    # Test time augmentation: horizontal flips.\n                    # TODO: not sure yet if this helps or not\n                    #x[n] = cv2.flip(resized_face, 1)\n                    #n += 1\n\n            del faces\n\n            if n > 0:\n                x = torch.tensor(x, device=device).float()\n\n                # Preprocess the images.\n                x = x.permute((0, 3, 1, 2))\n\n                for i in range(len(x)):\n                    x[i] = normalize_transform(x[i] / 255.)\n\n                # Make a prediction, then take the average.\n                with torch.no_grad():\n                    y_pred = 0\n                    stacked_preds = []\n                    preds = []\n                    \n                    for i in range(len(stack_models)):\n                        stacked_preds.append(stack_models[i](x).squeeze()[:n].unsqueeze(dim=1))\n                    \n                    for i in range(len(models)):\n                        preds.append(models[i](stacked_preds[meta_models[i][1]]))\n                \n                    del x, stacked_preds\n                    \n                    y_pred = torch.sigmoid(weight_preds(preds, weigths)).mean().item() #torch.sigmoid(metav4(preds)).mean().item()\n                    \n                    del preds\n                    \n                    return y_pred\n\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n    \n    \n    return 0.5#predict_mobilenet(video_path, batch_size=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_on_video_single(video_path, batch_size):\n    try:\n        # Find the faces for N frames in the video.\n        faces = face_extractor.process_video(video_path)\n\n        # Only look at one face per frame.\n        face_extractor.keep_only_best_face(faces)\n\n        if len(faces) > 0:\n            # NOTE: When running on the CPU, the batch size must be fixed\n            # or else memory usage will blow up. (Bug in PyTorch?)\n            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n\n            # If we found any faces, prepare them for the model.\n            n = 0\n            for frame_data in faces:\n                for face in frame_data[\"faces\"]:\n                    # Resize to the model's required input size.\n                    # We keep the aspect ratio intact and add zero\n                    # padding if necessary.\n\n                    #resized_face = isotropically_resize_image(face, input_size)\n                    #resized_face = make_square_image(resized_face)\n\n                    #resized_face = torchvision.transforms.Resize((input_size, input_size))(Image.fromarray(face))\n                    resized_face = cv2.resize(face, (input_size, input_size))\n                    if n < batch_size:\n                        x[n] = resized_face\n                        n += 1\n                    else:\n                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n\n                    # Test time augmentation: horizontal flips.\n                    # TODO: not sure yet if this helps or not\n                    #x[n] = cv2.flip(resized_face, 1)\n                    #n += 1\n\n            del faces\n\n            if n > 0:\n                x = torch.tensor(x, device=device).float()\n\n                # Preprocess the images.\n                x = x.permute((0, 3, 1, 2))\n\n                for i in range(len(x)):\n                    x[i] = normalize_transform(x[i] / 255.)\n\n                # Make a prediction, then take the average.\n                with torch.no_grad():\n                    stacked_preds = []\n                    preds = []\n                    \n                    for i in range(len(stack_models)):\n                        stacked_preds.append(stack_models[i](x).squeeze()[:n].unsqueeze(dim=1))\n                    \n                    del x\n                    \n                    y_pred = torch.sigmoid(models[-1](stacked_preds)).mean().item()\n\n                    return y_pred\n\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n    \n    \n    return 0.5#predict_mobilenet(video_path, batch_size=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\nimport gc\n\ndef predict_on_video_set(videos, num_workers):\n    def process_file(i):\n        filename = videos[i]\n        y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)\n        \n        '''if y_pred > 0.95:\n            y_pred = 0.95\n        elif y_pred < 0.05:\n            y_pred = 0.05'''\n        \n        return y_pred\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        predictions = ex.map(process_file, range(len(videos)))\n        \n    return list(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Speed test\n\nThe leaderboard submission must finish within 9 hours. With 4000 test videos, that is `9*60*60/4000 = 8.1` seconds per video. So if the average time per video is greater than ~8 seconds, the kernel will be too slow!"},{"metadata":{"trusted":true},"cell_type":"code","source":"speed_test = False  # you have to enable this manually","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Elapsed 6.873434 min. Average per video: 8.248120 sec.\nif speed_test:\n    start_time = time.time()\n    speedtest_videos = test_videos[:5]\n    predictions = predict_on_video_set(speedtest_videos, num_workers=4)\n    elapsed = time.time() - start_time\n    print(\"Elapsed %f min. Average per video: %f sec.\" % (elapsed / 60, elapsed / len(speedtest_videos)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make the submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = predict_on_video_set(test_videos, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r reader && rm install.sh","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}