{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys, time\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport random\nfrom random import randint\nfrom PIL import ImageFilter, Image\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nsys.path.append('../input/efficientnet')\nsys.path.append('../input/imutils/imutils-0.5.3')\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\nsys.path.insert(0, \"/kaggle/input/helpers\")\nsys.path.insert(0, \"/kaggle/input/timmmodels\") \n\nimport timm\nfrom imutils.video import FileVideoStream \nfrom efficientnet import EfficientNet\n\nrandom.seed(0)\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\ntorch.backends.cudnn.deterministic = True\n\n%matplotlib inline\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the test videos"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"PyTorch version:\", torch.__version__)\nprint(\"CUDA version:\", torch.version.cuda)\nprint(\"cuDNN version:\", torch.backends.cudnn.version())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from blazeface import BlazeFace\nfacedet = BlazeFace().to(device)\nfacedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\nfacedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n_ = facedet.train(False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_size = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import Normalize\n\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nnormalize_transform = Normalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def disable_grad(model):\n    for parameter in model.parameters():\n        parameter.requires_grad = False\n        \n    return model\n        \n\ndef weight_preds(preds, weights):\n    final_preds = []\n    for i in range(len(preds)):\n        for j in range(len(preds[i])):\n            if len(final_preds) != len(preds[i]):\n                final_preds.append(preds[i][j] * weights[i])\n            else:\n                final_preds[j] += preds[i][j] * weights[i]\n                \n    return torch.FloatTensor(final_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from helpers.read_video_1 import VideoReader\nfrom helpers.face_extract_1 import FaceExtractor\n\nframes_per_video = 120\n\nvideo_reader = VideoReader()\nvideo_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video) #get_frames(x, batch_size=frames_per_video)\nface_extractor = FaceExtractor(video_read_fn, facedet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MetaModel(nn.Module):\n    def __init__(self, models=None, device='cuda:0', extended=False):\n        super(MetaModel, self).__init__()\n        \n        self.extended = extended\n        self.device = device\n        self.models = models\n        self.len = len(models)\n        \n        if self.extended:\n            self.bn = nn.BatchNorm1d(self.len)\n            self.relu = nn.ReLU()\n            self.dropout = nn.Dropout(0.2)\n\n        self.fc = nn.Linear(self.len, 1)\n        \n    def forward(self, x):\n        x = torch.cat(tuple(x), dim=1)\n        \n        if self.extended:\n            x = self.bn(x)\n            x = self.relu(x)\n            #x = self.dropout(x)\n            \n        x = self.fc(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ensemble configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_PATH = \"/kaggle/input/deepfake-detection-model-20k/\"\nWEIGTHS_EXT = '.pth'\n\nmodels = []\nweigths = []\n    \nraw_data_stack = \\\n[\n    ['0.8548137313946486 0.3376769562025044', 'efficientnet-b2'],\n    ['EfficientNetb3 0.8573518024606384 0.34558522378585194', 'efficientnet-b3'],\n    ['EfficientNetb4 0.8579110384582294 0.3383911053075265', 'efficientnet-b4'],\n    ['EfficientNet6 0.8602770369095758 0.33193617861157143', 'efficientnet-b6'],\n    ['EfficientNetb0 t2 0.8616966359803837 0.3698434531609828', 'efficientnet-b0'],\n    ['EfficientNetb1 t2 0.8410909403768391 0.36058002083572327', 'efficientnet-b1'],\n    ['EfficientNetb2 t2 0.8659554331928073 0.35598630783834084', 'efficientnet-b2'],\n    ['EfficientNetb3 t2 0.8486191172674868 0.3611779548592305', 'efficientnet-b3'],\n    ['EfficientNetb3 0.8635894347414609 0.328333642473084', 'efficientnet-b3'],\n    ['EfficientNetb6 0.8593736556826981 0.32286693639934694', 'efficientnet-b6'],\n    \n    ['tf_efficientnet_b1_ns 0.8571367116923342 0.3341234226295108', 'tf_efficientnet_b1_ns'],\n    ['tf_efficientnet_b3_ns 0.8712466660930913 0.3277394129117183', 'tf_efficientnet_b3_ns'],\n    ['tf_efficientnet_b4_ns 0.8708595027101437 0.3152573955405342', 'tf_efficientnet_b4_ns'],\n    ['tf_efficientnet_b6_ns 0.8733115374688118 0.3156576980666498', 'tf_efficientnet_b6_ns'],\n]\n\nstack_models = []\n\nfor raw_model in raw_data_stack:\n    checkpoint = torch.load( MODELS_PATH + raw_model[0] + WEIGTHS_EXT, map_location=device)\n    \n    if '-' in raw_model[1]:\n        model = EfficientNet.from_name(raw_model[1])\n        model._fc = nn.Linear(model._fc.in_features, 1)\n    else:\n        model = timm.create_model(raw_model[1], pretrained=False)\n        model.classifier = nn.Linear(model.classifier.in_features, 1)\n    \n    model.load_state_dict(checkpoint)\n    _ = model.eval()\n    _ = disable_grad(model)\n    model = model.to(device)\n    stack_models.append(model)\n\n    del checkpoint, model\n    \n\nmeta_models = \\\n[\n    ['MetaModel 0.30638167556896007', slice(4, 8), False, 0.37780],\n    ['MetaModel 0.2919331893755284', slice(0, 4), False, 0.33357],\n    ['MetaModel 0.30281482560578044', slice(0, 8, None), True, 0.34077],\n    ['MetaModel 0.26302117601197256', slice(0, 10, None), False, 0.35134],\n    ['MetaModel 0.256337642808031', slice(10, 14, None), False, 0.32698],\n]\n\nfor meta_raw in meta_models:\n\n    checkpoint = torch.load(MODELS_PATH + meta_raw[0] + WEIGTHS_EXT, map_location=device)\n    \n    model = MetaModel(models=raw_data_stack[meta_raw[1]], extended=meta_raw[2]).to(device)\n    \n    model.load_state_dict(checkpoint)\n    _ = model.eval()\n    _ = disable_grad(model)\n    model.to(device)\n    models.append(model)\n    weigths.append(meta_raw[3])\n\n    del model, checkpoint\n    \ntotal = sum([1-score for score in weigths])\nweigths = [(1-score) / total for score in weigths]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_on_video(video_path, batch_size):\n    try:\n        # Find the faces for N frames in the video.\n        faces = face_extractor.process_video(video_path)\n\n        # Only look at one face per frame.\n        face_extractor.keep_only_best_face(faces)\n\n        if len(faces) > 0:\n            # NOTE: When running on the CPU, the batch size must be fixed\n            # or else memory usage will blow up. (Bug in PyTorch?)\n            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n\n            # If we found any faces, prepare them for the model.\n            n = 0\n            for frame_data in faces:\n                for face in frame_data[\"faces\"]:\n                    # Resize to the model's required input size.\n                    resized_face = cv2.resize(face, (input_size, input_size))\n                    \n                    if n < batch_size:\n                        x[n] = resized_face\n                        n += 1\n                    else:\n                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n\n                    # Test time augmentation: horizontal flips.\n                    # TODO: not sure yet if this helps or not\n                    #x[n] = cv2.flip(resized_face, 1)\n                    #n += 1\n\n            del faces\n\n            if n > 0:\n                x = torch.tensor(x, device=device).float()\n\n                # Preprocess the images.\n                x = x.permute((0, 3, 1, 2))\n\n                for i in range(len(x)):\n                    x[i] = normalize_transform(x[i] / 255.)\n\n                # Make a prediction\n                with torch.no_grad():\n                    y_pred = 0\n                    stacked_preds = []\n                    preds = []\n                    \n                    for i in range(len(stack_models)):\n                        stacked_preds.append(stack_models[i](x).squeeze()[:n].unsqueeze(dim=1))\n                    \n                    for i in range(len(models)):\n                        preds.append(models[i](stacked_preds[meta_models[i][1]]))\n                \n                    del x, stacked_preds\n                    \n                    y_pred = torch.sigmoid(weight_preds(preds, weigths)).mean().item()\n                    \n                    del preds\n                    \n                    return y_pred\n\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n    \n    \n    return 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\nimport gc\n\ndef predict_on_video_set(videos, num_workers):\n    def process_file(i):\n        filename = videos[i]\n        y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)\n        \n        return y_pred\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        predictions = ex.map(process_file, range(len(videos)))\n        \n    return list(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make the submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = predict_on_video_set(test_videos, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}