{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport math\n\nfrom pytorchcv.model_provider import get_model\n\nimport torch\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\nclass MishFunction(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.tanh(F.softplus(x))   # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_variables[0]\n        sigmoid = torch.sigmoid(x)\n        tanh_sp = torch.tanh(F.softplus(x)) \n        return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp))\n\nclass Mish(nn.Module):\n    def forward(self, x):\n        return MishFunction.apply(x)\n\ndef to_Mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, Mish())\n        else:\n            to_Mish(child)\n\nclass Head(torch.nn.Module):\n  def __init__(self, in_f, out_f, hidden):\n    super(Head, self).__init__()\n    \n    self.f = nn.Flatten()\n    self.l = nn.Linear(in_f, hidden)\n    self.m = Mish()\n    self.d = nn.Dropout(0.75)\n    self.o = nn.Linear(hidden, out_f)\n    # self.o = nn.Linear(in_f, out_f)\n    self.b1 = nn.BatchNorm1d(in_f)\n    self.b2 = nn.BatchNorm1d(hidden)\n    self.r = nn.ReLU()\n\n  def forward(self, x):\n    x = self.f(x)\n    x = self.b1(x)\n    x = self.d(x)\n\n    x = self.l(x)\n    x = self.r(x)\n    x = self.b2(x)\n    x = self.d(x)\n\n    out = self.o(x)\n    return out\n\nclass FCN(torch.nn.Module):\n  def __init__(self, base, in_f, hidden):\n    super(FCN, self).__init__()\n    self.base = base\n    self.h1 = Head(in_f, 1, hidden)\n  \n  def forward(self, x):\n    x = self.base(x)\n    return self.h1(x)\n\nnet = []\n\nmodel = get_model(\"xception\", pretrained=False)\nmodel = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\nmodel[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\nmodel = FCN(model, 2048, 512)\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/deepfake-models/model2.pth')) # .402\nnet.append(model)\n\n# model = get_model(\"efficientnet_b1\", pretrained=False)\n# model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n# model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n# model = FCN(model, 1280, 512)\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/deepfake-models/model (.251).pth')) # .41879\n# net.append(model)\n\n# model = get_model(\"xception\", pretrained=False)\n# model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n# model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n# model = FCN(model, 2048, 512)\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/deepfake-models/model (.32555).pth')) # .439\n# net.append(model)\n\nmodel = get_model(\"efficientnet_b1\", pretrained=False)\nmodel = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\nmodel[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\nmodel = FCN(model, 1280, 512)\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/deepfake-models/model (.259).pth')) # .392\nnet.append(model)\n\n# model = get_model(\"efficientnet_b1\", pretrained=False)\n# model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n# model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n# model = FCN(model, 1280, 256)\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/deepfake-models/model_17 (.2486).pth')) # .4099\n# net.append(model)\n\nmodel = get_model(\"efficientnet_b1\", pretrained=False)\nmodel = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\nmodel[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\nmodel = FCN(model, 1280, 512)\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/deepfake-models/model_16 (.2755).pth')) # .40\nnet.append(model)\n\n# model = get_model(\"efficientnet_b1\", pretrained=False)\n# model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n# model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n# model = FCN(model, 1280, 512)\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/deepfake-models/model_20 (.243).pth')) # .410\n# net.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from pytorchcv.model_provider import get_model\n# # model = get_model(\"xception\", pretrained=True)\n# model = get_model(\"efficientnet_b1\", pretrained=False)\n# model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n# model[0].final_pool = nn.AdaptiveAvgPool2d(1)\n# # model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n\n# class LRCN(nn.Module):\n#     def __init__(self, base, in_f, out_f):\n#         super(LRCN, self).__init__()\n#         self.cnn = base\n        \n#         self.LSTM = nn.LSTM(\n#             input_size=in_f,\n#             hidden_size=256,\n#             num_layers=1,\n#             batch_first=True\n#         )\n\n#         self.f1 = nn.Linear(256, 128)\n#         self.f2 = nn.Linear(128, out_f)\n#         self.r = nn.ReLU()\n#         self.d = nn.Dropout(0.5)\n        \n#     def forward(self, x):\n#         batch_size, timesteps, C, H, W = x.size()\n#         x = x.view(batch_size * timesteps, C, H, W)\n#         x = self.cnn(x)\n#         x = x.view(batch_size, timesteps, -1)\n#         self.LSTM.flatten_parameters()\n#         x, (hn,hc) = self.LSTM(x)\n#         x = self.d(self.r(self.f1(x[:,-1,:])))\n#         x = self.f2(x)\n#         return x\n\n# model = LRCN(model, 1280, 1)\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/deepfake-models/lrcn (.337).pth'))\n# net.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import Normalize\n\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nnormalize_transform = Normalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport matplotlib.pyplot as plt\nimport cv2\nimport pandas as pd\nimport time\nimport tensorflow as tf\nimport numpy as np\nimport glob\nfrom tqdm import tqdm\n\nimport tensorflow as tf\ndetection_graph = tf.Graph()\nwith detection_graph.as_default():\n    od_graph_def = tf.compat.v1.GraphDef()\n    with tf.io.gfile.GFile('../input/mobilenet-face/frozen_inference_graph_face.pb', 'rb') as fid:\n        serialized_graph = fid.read()\n        od_graph_def.ParseFromString(serialized_graph)\n        tf.import_graph_def(od_graph_def, name='')\n        config = tf.compat.v1.ConfigProto()\n    config.gpu_options.allow_growth = True\n    sess=tf.compat.v1.Session(graph=detection_graph, config=config)\n    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\n    boxes_tensor = detection_graph.get_tensor_by_name('detection_boxes:0')    \n    scores_tensor = detection_graph.get_tensor_by_name('detection_scores:0')\n    num_detections = detection_graph.get_tensor_by_name('num_detections:0')\n\ndef get_mobilenet_face(image):\n    global boxes,scores,num_detections\n    (im_height,im_width)=image.shape[:-1]\n    imgs=np.array([image])\n    (boxes, scores) = sess.run(\n        [boxes_tensor, scores_tensor],\n        feed_dict={image_tensor: imgs})\n    max_=np.where(scores==scores.max())[0][0]\n    box=boxes[0][max_]\n    ymin, xmin, ymax, xmax = box\n    (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n                                ymin * im_height, ymax * im_height)\n    left, right, top, bottom = int(left), int(right), int(top), int(bottom)\n    return (left, right, top, bottom)\ndef crop_image(frame,bbox):\n    left, right, top, bottom=bbox\n    return frame[top:bottom,left:right]\ndef get_img(frame):\n    return cv2.resize(crop_image(frame,get_mobilenet_face(frame)),(160,160))\n\ndef detect_video(video):\n    capture = cv2.VideoCapture(video)\n    v_len = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_idxs = np.linspace(0,v_len,frame_count, endpoint=False, dtype=np.int)\n    imgs=[]\n    i=0\n    for frame_idx in range(int(v_len)):\n        ret = capture.grab()\n        if not ret: \n            pass\n        if frame_idx >= frame_idxs[i]:\n            ret, frame = capture.retrieve()\n            if not ret or frame is None:\n                pass\n            else:\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                try:\n                    face=get_img(frame)\n                except Exception as err:\n                    print(err)\n                    continue\n                face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)\n                imgs.append(face)\n            i += 1\n            if i >= len(frame_idxs):\n                break\n    if len(imgs)<frame_count:\n        return None\n    return imgs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Process all videos"},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = glob.glob('/kaggle/input/deepfake-detection-challenge/test_videos/*.mp4')\nfilenames.sort()\n\nframe_count=30\n\nnum_faces = 0\nprobs = []\nindexes = []\npbar = tqdm(filenames)\nfor filename in pbar:\n    try:\n        faces_ = detect_video(filename)\n        if faces_ is None:\n            probs.append(0.5)\n#             break\n            continue\n            \n        faces = []\n        for face in faces_:\n            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)\n            img = np.rollaxis(cv2.resize(face, (150,150)), -1, 0)\n            faces.append(img)\n        \n        faces2 = []\n        for face in faces_:\n            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)\n            img = np.rollaxis(cv2.resize(face, (224,224)), -1, 0)\n            faces2.append(img)\n        \n        faces = torch.from_numpy(np.array(faces)).float().cuda()\n        faces2 = torch.from_numpy(np.array(faces2)).float().cuda()\n        \n        inputs = []\n#         inputs.append(faces2)\n#         normalize = [False]\n        \n#         inputs.append(faces)\n#         inputs.append(faces)\n#         inputs.append(faces)\n#         inputs.append(faces2)\n#         inputs.append(faces2)\n#         inputs.append(faces2)\n#         inputs.append(faces2)\n#         normalize = [True,True,True,False,False,False,False]\n        \n        inputs.append(faces)\n        inputs.append(faces2)\n        inputs.append(faces2)\n        normalize = [True,False,False]\n        \n        probs_2 = []\n        \n        # CNN\n        with torch.no_grad():\n            for j,model in enumerate(net):\n                model.eval()\n                probs_ = []\n                for i,face in enumerate(inputs[j]):\n                    if normalize[j]:\n                        face = normalize_transform(face / 255.)\n                    out = model(face[None])\n                    out = torch.sigmoid(out.squeeze())\n                    probs_.append(out.item())\n                probs_2.append(np.mean(np.array(probs_)))\n        \n        # LRCN\n#         faces_ = []\n#         for face in faces:\n#             face = normalize_transform(face / 255.)\n#             faces_.append(face)\n#         faces_ = torch.stack(faces_, dim=0).cuda().float()\n        \n#         with torch.no_grad():\n#             for model in net:\n#                 model.eval()\n#                 out = model(faces_[None])\n#                 out = torch.sigmoid(out.squeeze())\n#                 probs_2.append(out.item())\n\n        mult = 1\n        for prob in probs_2:\n            mult *= prob\n        probs.append(mult**(1/float(len(net))))\n        \n        num_faces += len(faces)\n        pbar.set_description(f'Faces found: {num_faces}')\n#         break\n    except:\n        probs.append(0.5)\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probs = np.asarray(probs)\nprobs[probs!=probs] = 0.5 # Remove NaNs\n# probs = np.clip(probs, 0.01, 0.99)\n\nplt.hist(probs, 40)\n\nfilenames = [os.path.basename(f) for f in filenames]\n\nsubmission = pd.DataFrame({'filename': filenames, 'label': probs})\nsubmission.to_csv('submission.csv', index=False)\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}