{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport os\nfrom keras.layers import *\nfrom keras import Model\nimport matplotlib.pyplot as plt\nimport time\nfrom keras.applications.xception import Xception\nimport efficientnet.keras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"PyTorch version:\", torch.__version__)\nprint(\"CUDA version:\", torch.version.cuda)\nprint(\"cuDNN version:\", torch.backends.cudnn.version())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ngpu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\nsys.path.insert(0, \"/kaggle/input/deepfakes-inference-demo\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from blazeface import BlazeFace\nfacedet = BlazeFace().to(gpu)\nfacedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\nfacedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n_ = facedet.train(False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_size = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import Normalize\n\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nnormalize_transform = Normalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from helpers.read_video_1 import VideoReader\nfrom helpers.face_extract_1 import FaceExtractor\n\nframes_per_video = 10\n\nvideo_reader = VideoReader()\nvideo_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)\nface_extractor = FaceExtractor(video_read_fn, facedet)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def isotropically_resize_image(img, size, resample=cv2.INTER_AREA):\n    h, w = img.shape[:2]\n    if w > h:\n        h = h * size // w\n        w = size\n    else:\n        w = w * size // h\n        h = size\n\n    resized = cv2.resize(img, (w, h), interpolation=resample)\n    return resized\n\n\ndef make_square_image(img):\n    h, w = img.shape[:2]\n    size = max(h, w)\n    t = 0\n    b = size - h\n    l = 0\n    r = size - w\n    return cv2.copyMakeBorder(img, t, b, l, r, cv2.BORDER_CONSTANT, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torchvision.models as models\n\nclass HisResNeXt(models.resnet.ResNet):\n    def __init__(self, training=True):\n        super(HisResNeXt, self).__init__(block=models.resnet.Bottleneck,\n                                        layers=[3, 4, 6, 3], \n                                        groups=32, \n                                        width_per_group=4)\n        self.fc = nn.Linear(2048, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_graph = tf.Graph()\nwith detection_graph.as_default():\n    od_graph_def = tf.compat.v1.GraphDef()\n    with tf.io.gfile.GFile('../input/mobilenet-face/frozen_inference_graph_face.pb', 'rb') as fid:\n        serialized_graph = fid.read()\n        od_graph_def.ParseFromString(serialized_graph)\n        tf.import_graph_def(od_graph_def, name='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncheckpoint = torch.load(\"/kaggle/input/deepfakes-inference-demo/resnext.pth\", map_location=gpu)\n\nmodel = HisResNeXt().to(gpu)\nmodel.load_state_dict(checkpoint)\n_ = model.eval()\n\ndel checkpoint\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_on_video(video_path, batch_size):\n    try:\n        # Find the faces for N frames in the video.\n        faces = face_extractor.process_video(video_path)\n\n        # Only look at one face per frame.\n        face_extractor.keep_only_best_face(faces)\n        \n        if len(faces) > 0:\n            # NOTE: When running on the CPU, the batch size must be fixed\n            # or else memory usage will blow up. (Bug in PyTorch?)\n            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n\n            # If we found any faces, prepare them for the model.\n            n = 0\n            for frame_data in faces:\n                for face in frame_data[\"faces\"]:\n                    # Resize to the model's required input size.\n                    # We keep the aspect ratio intact and add zero\n                    # padding if necessary.                    \n                    resized_face = isotropically_resize_image(face, input_size)\n                    resized_face = make_square_image(resized_face)\n\n                    if n < batch_size:\n                        x[n] = resized_face\n                        n += 1\n                    else:\n                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n                    \n                    # Test time augmentation: horizontal flips.\n                    # TODO: not sure yet if this helps or not\n                    #x[n] = cv2.flip(resized_face, 1)\n                    #n += 1\n\n            if n > 0:\n                x = torch.tensor(x, device=gpu).float()\n\n                # Preprocess the images.\n                x = x.permute((0, 3, 1, 2))\n\n                for i in range(len(x)):\n                    x[i] = normalize_transform(x[i] / 255.)\n\n                # Make a prediction, then take the average.\n                with torch.no_grad():\n                    y_pred = model(x)\n                    y_pred = torch.sigmoid(y_pred.squeeze())\n                    return y_pred[:n].mean().item()\n\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n\n    return 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = detection_graph.as_default()\ncm.__enter__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsess=tf.compat.v1.Session(graph=detection_graph, config=config)\nimage_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\nboxes_tensor = detection_graph.get_tensor_by_name('detection_boxes:0')\nscores_tensor = detection_graph.get_tensor_by_name('detection_scores:0')\nnum_detections = detection_graph.get_tensor_by_name('num_detections:0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(images):\n    global boxes,scores,num_detections\n    im_heights,im_widths=[],[]\n    imgs=[]\n    for image in images:\n        (im_height,im_width)=image.shape[:-1]\n        imgs.append(image)\n        im_heights.append(im_height)\n        im_widths.append(im_widths)\n    imgs=np.array(imgs)\n    (boxes, scores_) = sess.run(\n        [boxes_tensor, scores_tensor],\n        feed_dict={image_tensor: imgs})\n    finals=[]\n    for x in range(boxes.shape[0]):\n        scores=scores_[x]\n        max_=np.where(scores==scores.max())[0][0]\n        box=boxes[x][max_]\n        ymin, xmin, ymax, xmax = box\n        (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n                                      ymin * im_height, ymax * im_height)\n        left, right, top, bottom = int(left), int(right), int(top), int(bottom)\n        image=imgs[x]\n        finals.append(cv2.cvtColor(cv2.resize(image[max([0,top-40]):bottom+80,max([0,left-40]):right+80],(240,240)),cv2.COLOR_BGR2RGB))\n    return finals\ndef detect_video(video):\n    frame_count=10\n    capture = cv2.VideoCapture(video)\n    v_len = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_idxs = np.linspace(0,v_len,frame_count, endpoint=False, dtype=np.int)\n    imgs=[]\n    i=0\n    for frame_idx in range(int(v_len)):\n        ret = capture.grab()\n        if not ret: \n            print(\"Error grabbing frame %d from movie %s\" % (frame_idx, video))\n        if frame_idx >= frame_idxs[i]:\n            if frame_idx-frame_idxs[i]>20:\n                return None\n            ret, frame = capture.retrieve()\n            if not ret or frame is None:\n                print(\"Error retrieving frame %d from movie %s\" % (frame_idx, video))\n            else:\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                imgs.append(frame)\n            i += 1\n            if i >= len(frame_idxs):\n                break\n    imgs=get_img(imgs)\n    if len(imgs)<10:\n        return None\n    return np.hstack(imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.isdir(f'./videos/'):\n    os.makedirs(f'./videos/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_predictions =[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x in tqdm(glob.glob('../input/deepfake-detection-challenge/test_videos/*.mp4')):\n    try:\n        filename=x.replace('../input/deepfake-detection-challenge/test_videos/','').replace('.mp4','.jpg')\n        a=detect_video(x)\n        \n        y_pred = predict_on_video(x, batch_size=frames_per_video)\n        res_predictions.append(y_pred)\n        if a is None:\n            continue\n        cv2.imwrite('./videos/'+filename,a)\n    except Exception as err:\n        print(err)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm.__exit__(None,Exception,'exit')\nsess.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bottleneck_EfficientNetB1 = efn.EfficientNetB1(weights=None,include_top=False,pooling='avg')\ninp=Input((10,240,240,3))\nx=TimeDistributed(bottleneck_EfficientNetB1)(inp)\nx = LSTM(128)(x)\nx = Dense(64, activation='elu')(x)\nx = Dense(1,activation='sigmoid')(x)\nmodel_EfficientNetB1=Model(inp,x)\n\nbottleneck_Xception = Xception(weights=None,\n                 include_top=False,pooling='avg')\ny=TimeDistributed(bottleneck_Xception)(inp)\ny = LSTM(128)(y)\ny = Dense(64, activation='elu')(y)\ny = Dense(1,activation='sigmoid')(y)\nmodel_Xception=Model(inp,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_EfficientNetB1.load_weights('../input/efficientnetb1dfdc/EfficientNetB1-e_2_b_4_f_30-10.h5')\nmodel_Xception.load_weights('../input/xceptiondfdc/Xception-e_2_b_4_f_30-10.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_birghtness(img):\n    return img/img.max()\n# %% [code]\ndef process_img(img,flip=False):\n    imgs=[]\n    for x in range(10):\n        if flip:\n            imgs.append(get_birghtness(cv2.flip(img[:,x*240:(x+1)*240,:],1)))\n        else:\n            imgs.append(get_birghtness(img[:,x*240:(x+1)*240,:]))\n    return np.array(imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/deepfake-detection-challenge/sample_submission.csv\")\ntest_files=glob.glob('./videos/*.jpg')\nsubmission=pd.DataFrame()\nsubmission['filename']=os.listdir(('../input/deepfake-detection-challenge/test_videos/'))\nsubmission['label']=0.5\nfilenames=[]\nbatch=[]\nbatch1=[]\npreds=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x in test_files:\n    #print(x)\n    img=process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB))\n    #print(img)\n    if img is None:\n        continue\n    batch.append(img)\n    batch1.append(process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB),True))\n    filenames.append(x.replace('./videos/','').replace('.jpg','.mp4'))\n    if len(batch)==16:\n        preds+=(((0.2*model_EfficientNetB1.predict(np.array(batch))))+((0.2*model_EfficientNetB1.predict(np.array(batch1))))\n               +((0.2*model_Xception.predict(np.array(batch))))+((0.2*model_Xception.predict(np.array(batch1))))).tolist()\n\n        #preds+=(((0.5*model_EfficientNetB1.predict(np.array(batch))))+((0.5*model_Xception.predict(np.array(batch))))).tolist()\n\n        batch=[]\n        batch1=[]\n\nif len(batch)!=0:\n    #preds+=(((0.5*model_EfficientNetB1.predict(np.array(batch))))+((0.5*model_Xception.predict(np.array(batch))))).tolist()\n    preds+=(((0.2*model_EfficientNetB1.predict(np.array(batch))))+((0.2*model_EfficientNetB1.predict(np.array(batch1))))\n       +((0.2*model_Xception.predict(np.array(batch))))+((0.2*model_Xception.predict(np.array(batch1))))).tolist()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(res_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_preds=[]\nfor x,y in zip(preds,res_predictions):\n    new_preds.append(x[0]+(0.2*y))\nprint(sum(new_preds)/len(new_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x,y in zip(new_preds,filenames):\n    #submission.loc[submission['filename']==y,'label']=min([max([0.1,x]),0.9])\n    submission.loc[submission['filename']==y,'label']=x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(submission['label']).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\n!rm -r videos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}