{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install Packages","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# import file utilities\nimport os\nimport glob\n\n# import charting\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib.animation import FuncAnimation, ArtistAnimation\n%matplotlib inline\n\nfrom IPython.display import HTML\n\n# import computer vision\nimport cv2\nfrom skimage.measure import compare_ssim\n\n# import model package\n!pip install torch-summary\nfrom torchsummary import summary\nfrom sklearn.metrics import log_loss\nXCEPTION_MODEL = '../input/deepfakemodelspackages/xception-b5690688.pth'","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:26:04.654936Z","iopub.execute_input":"2023-11-08T14:26:04.656078Z","iopub.status.idle":"2023-11-08T14:26:12.909204Z","shell.execute_reply.started":"2023-11-08T14:26:04.655539Z","shell.execute_reply":"2023-11-08T14:26:12.907906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Install packages\n!pip install ../input/deepfakemodelspackages/Pillow-6.2.1-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/munch-2.5.0-py2.py3-none-any.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/numpy-1.17.4-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/six-1.13.0-py2.py3-none-any.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/torchvision-0.4.2-cp36-cp36m-manylinux1_x86_64.whl -f ./ --no-index\n!pip install ../input/deepfakemodelspackages/tqdm-4.40.2-py2.py3-none-any.whl -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:26:12.912869Z","iopub.execute_input":"2023-11-08T14:26:12.913397Z","iopub.status.idle":"2023-11-08T14:26:54.035288Z","shell.execute_reply.started":"2023-11-08T14:26:12.913301Z","shell.execute_reply":"2023-11-08T14:26:54.033674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!pip install ../input/deepfakemodelspackages/dlib-19.19.0/dlib-19.19.0/ -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:26:54.037422Z","iopub.execute_input":"2023-11-08T14:26:54.037775Z","iopub.status.idle":"2023-11-08T14:33:32.661257Z","shell.execute_reply.started":"2023-11-08T14:26:54.037722Z","shell.execute_reply":"2023-11-08T14:33:32.659951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Perform EDA","metadata":{}},{"cell_type":"code","source":"# Load Data\nTEST_PATH = '../input/deepfake-detection-challenge/test_videos/'\nTRAIN_PATH = '../input/deepfake-detection-challenge/train_sample_videos/'\n\nmetadata = '../input/deepfake-detection-challenge/train_sample_videos/metadata.json'","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:32.663282Z","iopub.execute_input":"2023-11-08T14:33:32.663649Z","iopub.status.idle":"2023-11-08T14:33:32.669908Z","shell.execute_reply.started":"2023-11-08T14:33:32.663598Z","shell.execute_reply":"2023-11-08T14:33:32.668737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the filenames for train videos\ntrain_fns = sorted(glob.glob(TRAIN_PATH + '*.mp4'))\n\n# load the filenames for test videos\ntest_fns = sorted(glob.glob(TEST_PATH + '*.mp4'))\n\nprint('There are {} samples in the train set.'.format(len(train_fns)))\nprint('There are {} samples in the test set.'.format(len(test_fns)))","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:32.674373Z","iopub.execute_input":"2023-11-08T14:33:32.674869Z","iopub.status.idle":"2023-11-08T14:33:32.693854Z","shell.execute_reply.started":"2023-11-08T14:33:32.674791Z","shell.execute_reply":"2023-11-08T14:33:32.692527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta = pd.read_json(metadata).transpose()\nmeta.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:32.698025Z","iopub.execute_input":"2023-11-08T14:33:32.698465Z","iopub.status.idle":"2023-11-08T14:33:32.874859Z","shell.execute_reply.started":"2023-11-08T14:33:32.698399Z","shell.execute_reply":"2023-11-08T14:33:32.873797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = 'FAKE', 'REAL'\nsizes = [meta[meta.label == 'FAKE'].label.count(), meta[meta.label == 'REAL'].label.count()]\n\nfig1, ax1 = plt.subplots(figsize=(10,7))\nax1.pie(sizes, labels=labels, autopct='%1.1f%%',\n        shadow=True, startangle=90, colors=['#f4d53f', '#02a1d8'])\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\nplt.title('Labels', fontsize=16)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:32.876399Z","iopub.execute_input":"2023-11-08T14:33:32.876692Z","iopub.status.idle":"2023-11-08T14:33:33.071972Z","shell.execute_reply.started":"2023-11-08T14:33:32.876646Z","shell.execute_reply":"2023-11-08T14:33:33.070748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_frame(filename):\n    '''\n    Helper function to return the 1st frame of the video by filename\n    INPUT: \n        filename - the filename of the video\n    OUTPUT:\n        image - 1st frame of the video (RGB)\n    '''\n    # Playing video from file\n    cap = cv2.VideoCapture(filename)\n    ret, frame = cap.read()\n\n    # Our operations on the frame come here\n    image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    \n    # When everything done, release the capture\n    cap.release()\n    cv2.destroyAllWindows()\n    \n    return image\n\ndef get_label(filename, meta):\n    '''\n    Helper function to get a label from the filepath.\n    INPUT:\n        filename - filename of the video\n        meta - dataframe containing metadata.json\n    OUTPUT:\n        label - label of the video 'FAKE' or 'REAL'\n    '''\n    video_id = filename.split('/')[-1]\n    return meta.loc[video_id].label\n\ndef get_original_filename(filename, meta):\n    '''\n    Helper function to get the filename of the original image\n    INPUT:\n        filename - filename of the video\n        meta - dataframe containing metadata.json\n    OUTPUT:\n        original_filename - name of the original video\n    '''\n    video_id = filename.split('/')[-1]\n    original_id = meta.loc[video_id].original\n    \n    return original_id\n\ndef visualize_frame(filename, meta, train = True):\n    '''\n    Helper function to visualize the 1st frame of the video by filename and metadata\n    INPUT:\n        filename - video filename\n        meta - dataframe containing metadata.json\n        train - indicates that the video is among train samples and the label can be retrived from metadata\n    '''\n    # get the 1st frame of the video\n    image = get_frame(filename)\n\n    # Display the 1st frame of the video\n    fig, axs = plt.subplots(1,3, figsize=(20,7))\n    axs[0].imshow(image) \n    axs[0].axis('off')\n    axs[0].set_title('Original frame')\n    \n    # Extract the face with haar cascades\n    face_cascade = cv2.CascadeClassifier('../input/haarcascades/haarcascade_frontalface_default.xml')\n\n    # run the detector\n    # the output here is an array of detections; the corners of each detection box\n    # if necessary, modify these parameters until you successfully identify every face in a given image\n    faces = face_cascade.detectMultiScale(image, 1.2, 3)\n\n    # make a copy of the original image to plot detections on\n    image_with_detections = image.copy()\n\n    # loop over the detected faces, mark the image where each face is found\n    for (x,y,w,h) in faces:\n        # draw a rectangle around each detected face\n        # you may also need to change the width of the rectangle drawn depending on image resolution\n        cv2.rectangle(image_with_detections,(x,y),(x+w,y+h),(255,0,0),3)\n\n    axs[1].imshow(image_with_detections)\n    axs[1].axis('off')\n    axs[1].set_title('Highlight faces')\n    \n    # crop out the 1st face\n    crop_img = image.copy()\n    for (x,y,w,h) in faces:\n        crop_img = image[y:y+h, x:x+w]\n        break;\n        \n    # plot the 1st face\n    axs[2].imshow(crop_img)\n    axs[2].axis('off')\n    axs[2].set_title('Zoom-in face')\n    \n    if train:\n        plt.suptitle('Image {image} label: {label}'.format(image = filename.split('/')[-1], label=get_label(filename, meta)))\n    else:\n        plt.suptitle('Image {image}'.format(image = filename.split('/')[-1]))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:33.074275Z","iopub.execute_input":"2023-11-08T14:33:33.075083Z","iopub.status.idle":"2023-11-08T14:33:33.107498Z","shell.execute_reply.started":"2023-11-08T14:33:33.075001Z","shell.execute_reply":"2023-11-08T14:33:33.105623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_frame(train_fns[0], meta)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:33.110155Z","iopub.execute_input":"2023-11-08T14:33:33.111594Z","iopub.status.idle":"2023-11-08T14:33:34.480057Z","shell.execute_reply.started":"2023-11-08T14:33:33.111506Z","shell.execute_reply":"2023-11-08T14:33:34.478538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_frame(train_fns[4], meta)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:34.482172Z","iopub.execute_input":"2023-11-08T14:33:34.482621Z","iopub.status.idle":"2023-11-08T14:33:35.427323Z","shell.execute_reply.started":"2023-11-08T14:33:34.482552Z","shell.execute_reply":"2023-11-08T14:33:35.425955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_frame(train_fns[8], meta)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:35.429401Z","iopub.execute_input":"2023-11-08T14:33:35.429848Z","iopub.status.idle":"2023-11-08T14:33:36.442048Z","shell.execute_reply.started":"2023-11-08T14:33:35.429784Z","shell.execute_reply":"2023-11-08T14:33:36.440627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_frame('../input/deepfake-detection-challenge/train_sample_videos/afoovlsmtx.mp4', meta)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:36.443803Z","iopub.execute_input":"2023-11-08T14:33:36.444337Z","iopub.status.idle":"2023-11-08T14:33:37.387878Z","shell.execute_reply.started":"2023-11-08T14:33:36.444248Z","shell.execute_reply":"2023-11-08T14:33:37.386525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_frame('../input/deepfake-detection-challenge/train_sample_videos/agrmhtjdlk.mp4', meta)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:37.390078Z","iopub.execute_input":"2023-11-08T14:33:37.390905Z","iopub.status.idle":"2023-11-08T14:33:38.286448Z","shell.execute_reply.started":"2023-11-08T14:33:37.390609Z","shell.execute_reply":"2023-11-08T14:33:38.285494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_frames(filename):\n    '''\n    Get all frames from the video\n    INPUT:\n        filename - video filename\n    OUTPUT:\n        frames - the array of video frames\n    '''\n    frames = []\n    cap = cv2.VideoCapture(filename)\n\n    while(cap.isOpened()):\n        ret, frame = cap.read()\n                \n        if not ret:\n            break;\n            \n        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        frames.append(image)\n\n    cap.release()\n    cv2.destroyAllWindows()\n    return frames\n\ndef create_animation(filename):\n    '''\n    Function to plot the animation with matplotlib\n    INPUT:\n        filename - filename of the video\n    '''\n    fig = plt.figure(figsize=(10,7))\n    frames = get_frames(filename)\n\n    ims = []\n    for frame in frames:\n        im = plt.imshow(frame, animated=True)\n        ims.append([im])\n\n    animation = ArtistAnimation(fig, ims, interval=30, repeat_delay=1000)\n    plt.show()\n    return animation\n\ndef visualize_several_frames(frames, step=100, cols = 3, title=''):\n    '''\n    Function to visualize the frames from the video\n    INPUT:\n        filename - filename of the video\n        step - the step between the video frames to visualize\n        cols - number of columns of frame grid\n    '''\n    n_frames = len(range(0, len(frames), step))\n    rows = n_frames // cols\n    if n_frames % cols > 0:\n        rows = rows + 1\n    \n    fig, axs = plt.subplots(rows, cols, figsize=(20,20))\n    for i in range(0, n_frames):\n        frame = frames[i]\n        \n        r = i // cols\n        c = i % cols\n        \n        axs[r,c].imshow(frame)\n        axs[r,c].axis('off')\n        axs[r,c].set_title(str(i))\n        \n    plt.suptitle(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:38.288052Z","iopub.execute_input":"2023-11-08T14:33:38.288609Z","iopub.status.idle":"2023-11-08T14:33:38.302846Z","shell.execute_reply.started":"2023-11-08T14:33:38.288548Z","shell.execute_reply":"2023-11-08T14:33:38.301907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = get_frames(train_fns[0])\nvisualize_several_frames(frames, step=50, cols = 2, title=train_fns[0].split('/')[-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:38.304179Z","iopub.execute_input":"2023-11-08T14:33:38.304679Z","iopub.status.idle":"2023-11-08T14:33:43.608152Z","shell.execute_reply.started":"2023-11-08T14:33:38.30462Z","shell.execute_reply":"2023-11-08T14:33:43.607108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_frames_zoomed(filename):\n    '''\n    Get all frames from the video zoomed into the face\n    INPUT:\n        filename - video filename\n    OUTPUT:\n        frames - the array of video frames\n    '''\n    frames = []\n    cap = cv2.VideoCapture(filename)\n    \n    face_cascade = cv2.CascadeClassifier('../input/haarcascades/haarcascade_frontalface_default.xml')\n\n    while(cap.isOpened()):\n        ret, frame = cap.read()\n                \n        if not ret:\n            break;\n            \n        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        \n        faces = face_cascade.detectMultiScale(image, 1.2, 3)\n        image_with_detections = image.copy()\n\n        crop_img = image.copy()\n        for (x,y,w,h) in faces:\n            crop_img = image[y:y+h, x:x+w]\n            break;\n        \n        frames.append(crop_img)\n\n    cap.release()\n    cv2.destroyAllWindows()\n    return frames\n\ndef create_animation_zoomed(filename):\n    '''\n    Function to create the animated cropped faces out of the video\n    INPUT:\n        filename - filename of the video\n    '''\n    fig, ax = plt.subplots(1,1, figsize=(10,7))\n    frames = get_frames_zoomed(filename)\n\n    def update(frame_number):\n        plt.axis('off')\n        plt.imshow(frames[frame_number])\n\n    animation = FuncAnimation(fig, update, interval=30, repeat=True)\n    return animation","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:43.609767Z","iopub.execute_input":"2023-11-08T14:33:43.610287Z","iopub.status.idle":"2023-11-08T14:33:43.621961Z","shell.execute_reply.started":"2023-11-08T14:33:43.61021Z","shell.execute_reply":"2023-11-08T14:33:43.620875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animation = create_animation_zoomed(train_fns[0])\nHTML(animation.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:33:43.62362Z","iopub.execute_input":"2023-11-08T14:33:43.62401Z","iopub.status.idle":"2023-11-08T14:36:57.336803Z","shell.execute_reply.started":"2023-11-08T14:33:43.623948Z","shell.execute_reply":"2023-11-08T14:36:57.335826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize the zoomed in frames\nframes_face = get_frames_zoomed(train_fns[0])\nvisualize_several_frames(frames_face, step=55, cols = 2, title=train_fns[0].split('/')[-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-08T14:36:57.338226Z","iopub.execute_input":"2023-11-08T14:36:57.338688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore the Similarity between Frames","metadata":{}},{"cell_type":"code","source":"def get_similarity_scores(frames):\n    '''\n    Get the list of similarity scores between the frames.\n    '''\n    scores = []\n    for i in range(1, len(frames)):\n        frame = frames[i]\n        prev_frame = frames[i-1]\n        \n        if frame.shape[0] != prev_frame.shape[0]:\n            if  frame.shape[0] > prev_frame.shape[0]:\n                frame = frame[:prev_frame.shape[0], :prev_frame.shape[0], :]\n            else:\n                prev_frame = prev_frame[:frame.shape[0], :frame.shape[0], :]\n        \n        (score, diff) = compare_ssim(frame, prev_frame, full=True, multichannel=True)\n        scores.append(score)\n    return scores\n\ndef plot_scores(scores):\n    '''\n    Plot the similarity scores\n    '''\n    plt.figure(figsize=(12,7))\n    plt.plot(scores)\n    plt.title('Similarity Scores')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = get_similarity_scores(frames)\nplot_scores(scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_dist = np.argmax(scores[1:50])\nmax_dist\nplt.imshow(frames_face[max_dist])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(frames_face[max_dist+5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copy in FaceForensics Code and Modify to run in kernel","metadata":{}},{"cell_type":"code","source":"## xception.py\n\"\"\"\nPorted to pytorch thanks to [tstandley](https://github.com/tstandley/Xception-PyTorch)\n@author: tstandley\nAdapted by cadene\nCreates an Xception Model as defined in:\nFrancois Chollet\nXception: Deep Learning with Depthwise Separable Convolutions\nhttps://arxiv.org/pdf/1610.02357.pdf\nThis weights ported from the Keras implementation. Achieves the following performance on the validation set:\nLoss:0.9173 Prec@1:78.892 Prec@5:94.292\nREMEMBER to set your image size to 3x299x299 for both test and validation\nnormalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],\n                                  std=[0.5, 0.5, 0.5])\nThe resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n\"\"\"\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom torch.nn import init\n\npretrained_settings = {\n    'xception': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1000,\n            'scale': 0.8975 # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n        }\n    }\n}\n\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):\n        super(SeparableConv2d,self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)\n        self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):\n        super(Block, self).__init__()\n\n        if out_filters != in_filters or strides!=1:\n            self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)\n            self.skipbn = nn.BatchNorm2d(out_filters)\n        else:\n            self.skip=None\n\n        self.relu = nn.ReLU(inplace=True)\n        rep=[]\n\n        filters=in_filters\n        if grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n            filters = out_filters\n\n        for i in range(reps-1):\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n\n        if not start_with_relu:\n            rep = rep[1:]\n        else:\n            rep[0] = nn.ReLU(inplace=False)\n\n        if strides != 1:\n            rep.append(nn.MaxPool2d(3,strides,1))\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self,inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x+=skip\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Xception optimized for the ImageNet dataset, as specified in\n    https://arxiv.org/pdf/1610.02357.pdf\n    \"\"\"\n    def __init__(self, num_classes=1000):\n        \"\"\" Constructor\n        Args:\n            num_classes: number of classes\n        \"\"\"\n        super(Xception, self).__init__()\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.relu = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32,64,3,bias=False)\n        self.bn2 = nn.BatchNorm2d(64)\n        #do relu here\n\n        self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)\n        self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)\n        self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)\n\n        self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)\n\n        self.conv3 = SeparableConv2d(1024,1536,3,1,1)\n        self.bn3 = nn.BatchNorm2d(1536)\n\n        #do relu here\n        self.conv4 = SeparableConv2d(1536,2048,3,1,1)\n        self.bn4 = nn.BatchNorm2d(2048)\n\n        self.fc = nn.Linear(2048, num_classes)\n\n        # #------- init weights --------\n        # for m in self.modules():\n        #     if isinstance(m, nn.Conv2d):\n        #         n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n        #         m.weight.data.normal_(0, math.sqrt(2. / n))\n        #     elif isinstance(m, nn.BatchNorm2d):\n        #         m.weight.data.fill_(1)\n        #         m.bias.data.zero_()\n        # #-----------------------------\n\n    def features(self, input):\n        x = self.conv1(input)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        return x\n\n    def logits(self, features):\n        x = self.relu(features)\n\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef xception(num_classes=1000, pretrained='imagenet'):\n    model = Xception(num_classes=num_classes)\n    if pretrained:\n        settings = pretrained_settings['xception'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        model = Xception(num_classes=num_classes)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n\n    # TODO: ugly\n    model.last_linear = model.fc\n    del model.fc\n    return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## models.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\n\n\nimport torch\n# import pretrainedmodels\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from network.xception import xception\nimport math\nimport torchvision\n\n\ndef return_pytorch04_xception(pretrained=True):\n    # Raises warning \"src not broadcastable to dst\" but thats fine\n    model = xception(pretrained=False)\n    if pretrained:\n        # Load model in torch 0.4+\n        model.fc = model.last_linear\n        del model.last_linear\n        state_dict = torch.load(\n            #'/home/ondyari/.torch/models/xception-b5690688.pth')\n            XCEPTION_MODEL)\n        for name, weights in state_dict.items():\n            if 'pointwise' in name:\n                state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)\n        model.load_state_dict(state_dict)\n        model.last_linear = model.fc\n        del model.fc\n    return model\n\n\nclass TransferModel(nn.Module):\n    \"\"\"\n    Simple transfer learning model that takes an imagenet pretrained model with\n    a fc layer as base model and retrains a new fc layer for num_out_classes\n    \"\"\"\n    def __init__(self, modelchoice, num_out_classes=2, dropout=0.0):\n        super(TransferModel, self).__init__()\n        self.modelchoice = modelchoice\n        if modelchoice == 'xception':\n            self.model = return_pytorch04_xception()\n            # Replace fc\n            num_ftrs = self.model.last_linear.in_features\n            if not dropout:\n                self.model.last_linear = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                print('Using dropout', dropout)\n                self.model.last_linear = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        elif modelchoice == 'resnet50' or modelchoice == 'resnet18':\n            if modelchoice == 'resnet50':\n                self.model = torchvision.models.resnet50(pretrained=True)\n            if modelchoice == 'resnet18':\n                self.model = torchvision.models.resnet18(pretrained=True)\n            # Replace fc\n            num_ftrs = self.model.fc.in_features\n            if not dropout:\n                self.model.fc = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                self.model.fc = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        else:\n            raise Exception('Choose valid model, e.g. resnet50')\n\n    def set_trainable_up_to(self, boolean, layername=\"Conv2d_4a_3x3\"):\n        \"\"\"\n        Freezes all layers below a specific layer and sets the following layers\n        to true if boolean else only the fully connected final layer\n        :param boolean:\n        :param layername: depends on network, for inception e.g. Conv2d_4a_3x3\n        :return:\n        \"\"\"\n        # Stage-1: freeze all the layers\n        if layername is None:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = True\n                return\n        else:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = False\n        if boolean:\n            # Make all layers following the layername layer trainable\n            ct = []\n            found = False\n            for name, child in self.model.named_children():\n                if layername in ct:\n                    found = True\n                    for params in child.parameters():\n                        params.requires_grad = True\n                ct.append(name)\n            if not found:\n                raise Exception('Layer not found, cant finetune!'.format(\n                    layername))\n        else:\n            if self.modelchoice == 'xception':\n                # Make fc trainable\n                for param in self.model.last_linear.parameters():\n                    param.requires_grad = True\n\n            else:\n                # Make fc trainable\n                for param in self.model.fc.parameters():\n                    param.requires_grad = True\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n\ndef model_selection(modelname, num_out_classes,\n                    dropout=None):\n    \"\"\"\n    :param modelname:\n    :return: model, image size, pretraining<yes/no>, input_list\n    \"\"\"\n    if modelname == 'xception':\n        return TransferModel(modelchoice='xception',\n                             num_out_classes=num_out_classes), 299, \\\n               True, ['image'], None\n    elif modelname == 'resnet18':\n        return TransferModel(modelchoice='resnet18', dropout=dropout,\n                             num_out_classes=num_out_classes), \\\n               224, True, ['image'], None\n    else:\n        raise NotImplementedError(modelname)\n\n# if __name__ == '__main__':\n#     model, image_size, *_ = model_selection('resnet18', num_out_classes=2)\n#     print(model)\n#     model = model.cuda()\n#     from torchsummary import summary\n#     input_s = (3, image_size, image_size)\n#     print(summary(model, input_s))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## transform.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nfrom torchvision import transforms\n\nxception_default_data_transforms = {\n    'train': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ]),\n    'val': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize((299, 299)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## detect_from_video.py\n\"\"\"\nEvaluates a folder of video files or a single file with a xception binary\nclassification network.\nUsage:\npython detect_from_video.py\n    -i <folder with video files or path to video file>\n    -m <path to model file>\n    -o <path to output folder, will write one or multiple output videos there>\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\nfrom os.path import join\nimport cv2\nimport dlib\nimport torch\nimport torch.nn as nn\nfrom PIL import Image as pil_image\nfrom tqdm.notebook import tqdm\n\n# from network.models import model_selection\n# from dataset.transform import xception_default_data_transforms\n\ndef get_boundingbox(face, width, height, scale=1.3, minsize=None):\n    \"\"\"\n    Expects a dlib face to generate a quadratic bounding box.\n    :param face: dlib face class\n    :param width: frame width\n    :param height: frame height\n    :param scale: bounding box size multiplier to get a bigger face region\n    :param minsize: set minimum bounding box size\n    :return: x, y, bounding_box_size in opencv form\n    \"\"\"\n    x1 = face.left()\n    y1 = face.top()\n    x2 = face.right()\n    y2 = face.bottom()\n    size_bb = int(max(x2 - x1, y2 - y1) * scale)\n    if minsize:\n        if size_bb < minsize:\n            size_bb = minsize\n    center_x, center_y = (x1 + x2) // 2, (y1 + y2) // 2\n\n    # Check for out of bounds, x-y top left corner\n    x1 = max(int(center_x - size_bb // 2), 0)\n    y1 = max(int(center_y - size_bb // 2), 0)\n    # Check for too big bb size for given x, y\n    size_bb = min(width - x1, size_bb)\n    size_bb = min(height - y1, size_bb)\n\n    return x1, y1, size_bb\n\n\ndef preprocess_image(image, cuda=True):\n    \"\"\"\n    Preprocesses the image such that it can be fed into our network.\n    During this process we envoke PIL to cast it into a PIL image.\n    :param image: numpy image in opencv form (i.e., BGR and of shape\n    :return: pytorch tensor of shape [1, 3, image_size, image_size], not\n    necessarily casted to cuda\n    \"\"\"\n    # Revert from BGR\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    # Preprocess using the preprocessing function used during training and\n    # casting it to PIL image\n    preprocess = xception_default_data_transforms['test']\n    preprocessed_image = preprocess(pil_image.fromarray(image))\n    # Add first dimension as the network expects a batch\n    preprocessed_image = preprocessed_image.unsqueeze(0)\n    if cuda:\n        preprocessed_image = preprocessed_image.cuda()\n    return preprocessed_image\n\n\ndef predict_with_model(image, model, post_function=nn.Softmax(dim=1),\n                       cuda=True):\n    \"\"\"\n    Predicts the label of an input image. Preprocesses the input image and\n    casts it to cuda if required\n    :param image: numpy image\n    :param model: torch model with linear layer at the end\n    :param post_function: e.g., softmax\n    :param cuda: enables cuda, must be the same parameter as the model\n    :return: prediction (1 = fake, 0 = real)\n    \"\"\"\n    # Preprocess\n    preprocessed_image = preprocess_image(image, cuda)\n\n    # Model prediction\n    output = model(preprocessed_image)\n    output = post_function(output)\n\n    # Cast to desired\n    _, prediction = torch.max(output, 1)    # argmax\n    prediction = float(prediction.cpu().numpy())\n\n    return int(prediction), output\n\ndef test_full_image_network(video_path, model, output_path,\n                            start_frame=0, end_frame=None, cuda=True):\n    \"\"\"\n    Reads a video and evaluates a subset of frames with the a detection network\n    that takes in a full frame. Outputs are only given if a face is present\n    and the face is highlighted using dlib.\n    :param video_path: path to video file\n    :param model_path: path to model file (should expect the full sized image)\n    :param output_path: path where the output video is stored\n    :param start_frame: first frame to evaluate\n    :param end_frame: last frame to evaluate\n    :param cuda: enable cuda\n    :return:\n    \n    # Modified to take in the model file instead of model\n    \"\"\"\n    #print('Starting: {}'.format(video_path))\n\n    # Read and write\n    reader = cv2.VideoCapture(video_path)\n\n    video_fn = video_path.split('/')[-1].split('.')[0]+'.avi'\n    os.makedirs(output_path, exist_ok=True)\n    fourcc = cv2.VideoWriter_fourcc(*'MJPG')\n    fps = reader.get(cv2.CAP_PROP_FPS)\n    num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT))\n    writer = None\n\n    # Face detector\n    face_detector = dlib.get_frontal_face_detector()\n\n    # Load model\n#     model, *_ = model_selection(modelname='xception', num_out_classes=2)\n#     if model_path is not None:\n#         model = torch.load(model_path)\n#         print('Model found in {}'.format(model_path))\n#     else:\n#         print('No model found, initializing random model.')\n#     if cuda:\n#         model = model.cuda()\n\n    # Text variables\n    font_face = cv2.FONT_HERSHEY_SIMPLEX\n    thickness = 2\n    font_scale = 1\n\n    # Frame numbers and length of output video\n    frame_num = 0\n    assert start_frame < num_frames - 1\n    end_frame = end_frame if end_frame else num_frames\n    pbar = tqdm(total=end_frame-start_frame)\n\n    while reader.isOpened():\n        _, image = reader.read()\n        if image is None:\n            break\n        frame_num += 1\n\n        if frame_num < start_frame:\n            continue\n        pbar.update(1)\n\n        # Image size\n#         print('getting image size')\n        height, width = image.shape[:2]\n\n        # Init output writer\n#         print('init output writer')\n        if writer is None:\n            writer = cv2.VideoWriter(join(output_path, video_fn), fourcc, fps,\n                                     (height, width)[::-1])\n\n        # 2. Detect with dlib\n#         print('detect with dlib')\n        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        faces = face_detector(gray, 1)\n        if len(faces):\n            # For now only take biggest face\n            face = faces[0]\n\n            # --- Prediction ---------------------------------------------------\n            # Face crop with dlib and bounding box scale enlargement\n            x, y, size = get_boundingbox(face, width, height)\n            cropped_face = image[y:y+size, x:x+size]\n\n            # Actual prediction using our model\n            prediction, output = predict_with_model(cropped_face, model,\n                                                    cuda=cuda)\n            # ------------------------------------------------------------------\n\n            # Text and bb\n            x = face.left()\n            y = face.top()\n            w = face.right() - x\n            h = face.bottom() - y\n            label = 'fake' if prediction == 1 else 'real'\n            color = (0, 255, 0) if prediction == 0 else (0, 0, 255)\n            output_list = ['{0:.2f}'.format(float(x)) for x in\n                           output.detach().cpu().numpy()[0]]\n            cv2.putText(image, str(output_list)+'=>'+label, (x, y+h+30),\n                        font_face, font_scale,\n                        color, thickness, 2)\n            # draw box over face\n            cv2.rectangle(image, (x, y), (x + w, y + h), color, 2)\n\n        if frame_num >= end_frame:\n            break\n\n        # Show\n#         print('show result')\n        # cv2.imshow('test', image)\n#         cv2.waitKey(33)     # About 30 fps\n        writer.write(image)\n    pbar.close()\n    if writer is not None:\n        writer.release()\n        #print('Finished! Output saved under {}'.format(output_path))\n    else:\n        pass\n        #print('Input video file was empty')\n    return","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Write Files to disk for importing model","metadata":{}},{"cell_type":"code","source":"!mkdir network","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/__init__.py","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/xception.py\n\"\"\"\nPorted to pytorch thanks to [tstandley](https://github.com/tstandley/Xception-PyTorch)\n@author: tstandley\nAdapted by cadene\nCreates an Xception Model as defined in:\nFrancois Chollet\nXception: Deep Learning with Depthwise Separable Convolutions\nhttps://arxiv.org/pdf/1610.02357.pdf\nThis weights ported from the Keras implementation. Achieves the following performance on the validation set:\nLoss:0.9173 Prec@1:78.892 Prec@5:94.292\nREMEMBER to set your image size to 3x299x299 for both test and validation\nnormalize = transforms.Normalize(mean=[0.5, 0.5, 0.5],\n                                  std=[0.5, 0.5, 0.5])\nThe resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n\"\"\"\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom torch.nn import init\n\npretrained_settings = {\n    'xception': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/xception-b5690688.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1000,\n            'scale': 0.8975 # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n        }\n    }\n}\n\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):\n        super(SeparableConv2d,self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)\n        self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):\n        super(Block, self).__init__()\n\n        if out_filters != in_filters or strides!=1:\n            self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)\n            self.skipbn = nn.BatchNorm2d(out_filters)\n        else:\n            self.skip=None\n\n        self.relu = nn.ReLU(inplace=True)\n        rep=[]\n\n        filters=in_filters\n        if grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n            filters = out_filters\n\n        for i in range(reps-1):\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n\n        if not start_with_relu:\n            rep = rep[1:]\n        else:\n            rep[0] = nn.ReLU(inplace=False)\n\n        if strides != 1:\n            rep.append(nn.MaxPool2d(3,strides,1))\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self,inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x+=skip\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Xception optimized for the ImageNet dataset, as specified in\n    https://arxiv.org/pdf/1610.02357.pdf\n    \"\"\"\n    def __init__(self, num_classes=1000):\n        \"\"\" Constructor\n        Args:\n            num_classes: number of classes\n        \"\"\"\n        super(Xception, self).__init__()\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.relu = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32,64,3,bias=False)\n        self.bn2 = nn.BatchNorm2d(64)\n        #do relu here\n\n        self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)\n        self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)\n        self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)\n\n        self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)\n\n        self.conv3 = SeparableConv2d(1024,1536,3,1,1)\n        self.bn3 = nn.BatchNorm2d(1536)\n\n        #do relu here\n        self.conv4 = SeparableConv2d(1536,2048,3,1,1)\n        self.bn4 = nn.BatchNorm2d(2048)\n\n        self.fc = nn.Linear(2048, num_classes)\n\n        # #------- init weights --------\n        # for m in self.modules():\n        #     if isinstance(m, nn.Conv2d):\n        #         n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n        #         m.weight.data.normal_(0, math.sqrt(2. / n))\n        #     elif isinstance(m, nn.BatchNorm2d):\n        #         m.weight.data.fill_(1)\n        #         m.bias.data.zero_()\n        # #-----------------------------\n\n    def features(self, input):\n        x = self.conv1(input)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        return x\n\n    def logits(self, features):\n        x = self.relu(features)\n\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef xception(num_classes=1000, pretrained='imagenet'):\n    model = Xception(num_classes=num_classes)\n    if pretrained:\n        settings = pretrained_settings['xception'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        model = Xception(num_classes=num_classes)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n\n    # TODO: ugly\n    model.last_linear = model.fc\n    del model.fc\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile network/models.py\n\"\"\"\nAuthor: Andreas Rössler\n\"\"\"\nimport os\nimport argparse\n\n\nimport torch\n#import pretrainedmodels\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom network.xception import xception\nimport math\nimport torchvision\n\n\ndef return_pytorch04_xception(pretrained=True):\n    # Raises warning \"src not broadcastable to dst\" but thats fine\n    model = xception(pretrained=False)\n    if pretrained:\n        # Load model in torch 0.4+\n        model.fc = model.last_linear\n        del model.last_linear\n        state_dict = torch.load(\n            '/home/ondyari/.torch/models/xception-b5690688.pth')\n        for name, weights in state_dict.items():\n            if 'pointwise' in name:\n                state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)\n        model.load_state_dict(state_dict)\n        model.last_linear = model.fc\n        del model.fc\n    return model\n\n\nclass TransferModel(nn.Module):\n    \"\"\"\n    Simple transfer learning model that takes an imagenet pretrained model with\n    a fc layer as base model and retrains a new fc layer for num_out_classes\n    \"\"\"\n    def __init__(self, modelchoice, num_out_classes=2, dropout=0.0):\n        super(TransferModel, self).__init__()\n        self.modelchoice = modelchoice\n        if modelchoice == 'xception':\n            self.model = return_pytorch04_xception()\n            # Replace fc\n            num_ftrs = self.model.last_linear.in_features\n            if not dropout:\n                self.model.last_linear = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                print('Using dropout', dropout)\n                self.model.last_linear = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        elif modelchoice == 'resnet50' or modelchoice == 'resnet18':\n            if modelchoice == 'resnet50':\n                self.model = torchvision.models.resnet50(pretrained=True)\n            if modelchoice == 'resnet18':\n                self.model = torchvision.models.resnet18(pretrained=True)\n            # Replace fc\n            num_ftrs = self.model.fc.in_features\n            if not dropout:\n                self.model.fc = nn.Linear(num_ftrs, num_out_classes)\n            else:\n                self.model.fc = nn.Sequential(\n                    nn.Dropout(p=dropout),\n                    nn.Linear(num_ftrs, num_out_classes)\n                )\n        else:\n            raise Exception('Choose valid model, e.g. resnet50')\n\n    def set_trainable_up_to(self, boolean, layername=\"Conv2d_4a_3x3\"):\n        \"\"\"\n        Freezes all layers below a specific layer and sets the following layers\n        to true if boolean else only the fully connected final layer\n        :param boolean:\n        :param layername: depends on network, for inception e.g. Conv2d_4a_3x3\n        :return:\n        \"\"\"\n        # Stage-1: freeze all the layers\n        if layername is None:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = True\n                return\n        else:\n            for i, param in self.model.named_parameters():\n                param.requires_grad = False\n        if boolean:\n            # Make all layers following the layername layer trainable\n            ct = []\n            found = False\n            for name, child in self.model.named_children():\n                if layername in ct:\n                    found = True\n                    for params in child.parameters():\n                        params.requires_grad = True\n                ct.append(name)\n            if not found:\n                raise Exception('Layer not found, cant finetune!'.format(\n                    layername))\n        else:\n            if self.modelchoice == 'xception':\n                # Make fc trainable\n                for param in self.model.last_linear.parameters():\n                    param.requires_grad = True\n\n            else:\n                # Make fc trainable\n                for param in self.model.fc.parameters():\n                    param.requires_grad = True\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n\ndef model_selection(modelname, num_out_classes,\n                    dropout=None):\n    \"\"\"\n    :param modelname:\n    :return: model, image size, pretraining<yes/no>, input_list\n    \"\"\"\n    if modelname == 'xception':\n        return TransferModel(modelchoice='xception',\n                             num_out_classes=num_out_classes), 299, \\\n               True, ['image'], None\n    elif modelname == 'resnet18':\n        return TransferModel(modelchoice='resnet18', dropout=dropout,\n                             num_out_classes=num_out_classes), \\\n               224, True, ['image'], None\n    else:\n        raise NotImplementedError(modelname)\n\n\nif __name__ == '__main__':\n    model, image_size, *_ = model_selection('resnet18', num_out_classes=2)\n    print(model)\n    model = model.cuda()\n    from torchsummary import summary\n    input_s = (3, image_size, image_size)\n    print(summary(model, input_s))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained Models from FaceForensics++","metadata":{}},{"cell_type":"code","source":"metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\n\ndef predict_model(video_fn, model,\n                  start_frame=0, end_frame=30,\n                  plot_every_x_frames = 5):\n    \"\"\"\n    Given a video and model, starting frame and end frame.\n    Predict on all frames.\n    \n    \"\"\"\n    fn = video_fn.split('.')[0]\n    label = metadata.loc[video_fn]['label']\n    original = metadata.loc[video_fn]['original']\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    output_path = './'\n    test_full_image_network(video_path, model, output_path, start_frame=0, end_frame=30, cuda=False)\n    # Read output\n    vidcap = cv2.VideoCapture(f'{fn}.avi')\n    success,image = vidcap.read()\n    count = 0\n    fig, axes = plt.subplots(3, 2, figsize=(20, 15))\n    axes = axes = axes.flatten()\n    i = 0\n    while success:\n        # Show every xth frame\n        if count % plot_every_x_frames == 0:\n\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            axes[i].imshow(image)\n            axes[i].set_title(f'{fn} - frame {count} - true label: {label}')\n            axes[i].xaxis.set_visible(False)\n            axes[i].yaxis.set_visible(False)\n            i += 1\n        success,image = vidcap.read()\n        count += 1\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing Full Image Models","metadata":{}},{"cell_type":"markdown","source":"## full raw","metadata":{}},{"cell_type":"code","source":"model_path = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_raw.p'\nmodel = torch.load(model_path, map_location=torch.device('cpu'))\nsummary(model, input_size=(3, 299, 299))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model)\npredict_model('crezycjqyk.mp4', model)\npredict_model('ebchwmwayp.mp4', model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## C40","metadata":{}},{"cell_type":"code","source":"model_path_full_c40 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_c40.p'\nmodel_full_c40 = torch.load(model_path_full_c40, map_location=torch.device('cpu'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_full_c40)\npredict_model('crezycjqyk.mp4', model_full_c40)\npredict_model('ebchwmwayp.mp4', model_full_c40)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## C23","metadata":{}},{"cell_type":"code","source":"model_path_full23 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/full/xception/full_c23.p'\nmodel_full_c23 = torch.load(model_path_full23, map_location=torch.device('cpu'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_model('bbhtdfuqxq.mp4', model_full_c23)\npredict_model('crezycjqyk.mp4', model_full_c23)\npredict_model('ebchwmwayp.mp4', model_full_c23)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validate Predictions on train set","metadata":{}},{"cell_type":"code","source":"def video_file_frame_pred(video_path, model,\n                          start_frame=0, end_frame=300,\n                          cuda=True, n_frames=5):\n    \"\"\"\n    Predict and give result as numpy array\n    \"\"\"\n    pred_frames = [int(round(x)) for x in np.linspace(start_frame, end_frame, n_frames)]\n    predictions = []\n    outputs = []\n    # print('Starting: {}'.format(video_path))\n\n    # Read and write\n    reader = cv2.VideoCapture(video_path)\n\n    video_fn = video_path.split('/')[-1].split('.')[0]+'.avi'\n    fourcc = cv2.VideoWriter_fourcc(*'MJPG')\n    fps = reader.get(cv2.CAP_PROP_FPS)\n    num_frames = int(reader.get(cv2.CAP_PROP_FRAME_COUNT))\n    writer = None\n\n    # Face detector\n    face_detector = dlib.get_frontal_face_detector()\n\n    # Text variables\n    font_face = cv2.FONT_HERSHEY_SIMPLEX\n    thickness = 2\n    font_scale = 1\n\n    # Frame numbers and length of output video\n    frame_num = 0\n    assert start_frame < num_frames - 1\n    end_frame = end_frame if end_frame else num_frames\n    while reader.isOpened():\n        _, image = reader.read()\n        if image is None:\n            break\n        frame_num += 1\n        if frame_num in pred_frames:\n            height, width = image.shape[:2]\n            # 2. Detect with dlib\n            gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n            faces = face_detector(gray, 1)\n            if len(faces):\n                # For now only take biggest face\n                face = faces[0]\n                # --- Prediction ---------------------------------------------------\n                # Face crop with dlib and bounding box scale enlargement\n                x, y, size = get_boundingbox(face, width, height)\n                cropped_face = image[y:y+size, x:x+size]\n\n                # Actual prediction using our model\n                prediction, output = predict_with_model(cropped_face, model,\n                                                        cuda=cuda)\n                predictions.append(prediction)\n                outputs.append(output)\n                # ------------------------------------------------------------------\n        if frame_num >= end_frame:\n            break\n    # Figure out how to do this with torch\n    preds_np = [x.detach().cpu().numpy()[0][1] for x in outputs]\n    if len(preds_np) == 0:\n        return predictions, outputs, 0.5, 0.5, 0.5\n    try:\n        mean_pred = np.mean(preds_np)\n    except:\n        # couldnt find faces\n        mean_pred = 0.5\n    min_pred = np.min(preds_np)\n    max_pred = np.max(preds_np)\n    return predictions, outputs, mean_pred, min_pred, max_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.nn.Module.dump_patches = True\nmodel_path_23 = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_c23.p'\nmodel_23 = torch.load(model_path_23, map_location=torch.device('cpu'))\nmodel_path_raw = '../input/deepfakemodelspackages/faceforensics_models/faceforensics++_models_subset/face_detection/xception/all_raw.p'\nmodel_raw = torch.load(model_path_raw, map_location=torch.device('cpu'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read metadata\nmetadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\n\n# Predict Fake\nfor video_fn in tqdm(metadata.query('label == \"FAKE\"').sample(77).index):\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_23, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_c23'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_c23'] = min_pred\n    metadata.loc[video_fn, 'max_pred_c23'] = max_pred\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_raw, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_raw'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_raw'] = min_pred\n    metadata.loc[video_fn, 'max_pred_raw'] = max_pred\n    \n# Predict Real\nfor video_fn in tqdm(metadata.query('label == \"REAL\"').sample(77).index):\n    video_path = f'../input/deepfake-detection-challenge/train_sample_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_23, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_c23'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_c23'] = min_pred\n    metadata.loc[video_fn, 'max_pred_c23'] = max_pred\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model_raw, n_frames=4, cuda=False)\n    metadata.loc[video_fn, 'avg_pred_raw'] = mean_pred\n    metadata.loc[video_fn, 'min_pred_raw'] = min_pred\n    metadata.loc[video_fn, 'max_pred_raw'] = max_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df = metadata.dropna(subset=['avg_pred_raw']).copy()\npreds_df['label_binary'] = 0\npreds_df.loc[preds_df['label'] == \"FAKE\", 'label_binary'] = 1\npreds_df[['min_pred_c23','max_pred_c23',\n          'min_pred_raw','max_pred_raw']] = preds_df[['min_pred_c23','max_pred_c23',\n                                                      'min_pred_raw','max_pred_raw']].fillna(0.5)\npreds_df['naive_pred'] = 0.5\nscore_avg23 = log_loss(preds_df['label_binary'], preds_df['avg_pred_c23'])\nscore_min23 = log_loss(preds_df['label_binary'], preds_df['min_pred_c23'])\nscore_max23 = log_loss(preds_df['label_binary'], preds_df['max_pred_c23'])\nscore_avgraw = log_loss(preds_df['label_binary'], preds_df['avg_pred_raw'])\nscore_minraw = log_loss(preds_df['label_binary'], preds_df['min_pred_raw'])\nscore_maxraw = log_loss(preds_df['label_binary'], preds_df['max_pred_raw'])\nscore_naive = log_loss(preds_df['label_binary'], preds_df['naive_pred'])\npreds_df['max_pred_clipped'] = preds_df['max_pred_c23'].clip(0.4, 1)\nscore_max_clipped = log_loss(preds_df['label_binary'], preds_df['max_pred_clipped'])\npreds_df['max_pred_clipped_raw'] = preds_df['max_pred_raw'].clip(0.4, 1)\nscore_max_clipped_raw = log_loss(preds_df['label_binary'], preds_df['max_pred_clipped_raw'])\nprint('Score using average prediction of all frames all_c23.p: {:0.4f}'.format(score_avg23))\nprint('Score using minimum prediction of all frames all_c23.p: {:0.4f}'.format(score_min23))\nprint('Score using maximum prediction of all frames all_c23.p: {:0.4f}'.format(score_max23))\nprint('Score using 0.5 prediction of all frames: {:0.4f}'.format(score_naive))\nprint('Score using maximum clipped prediction of all frames: {:0.4f}'.format(score_max_clipped))\nprint('Score using average prediction of all frames all_raw.p: {:0.4f}'.format(score_avgraw))\nprint('Score using minimum prediction of all frames all_raw.p: {:0.4f}'.format(score_minraw))\nprint('Score using maximum prediction of all frames all_raw.p: {:0.4f}'.format(score_maxraw))\nprint('Score using maximum clipped prediction of all frames all_raw.p: {:0.4f}'.format(score_max_clipped_raw))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot the Average vs Max Prediction Probability - Fake vs Real","metadata":{}},{"cell_type":"markdown","source":"## C23 Result","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(10, 10))\nsns.scatterplot(x='avg_pred_c23', y='max_pred_c23', data=metadata.dropna(subset=['avg_pred_c23']), hue='label')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Raw result","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(10, 10))\nsns.scatterplot(x='avg_pred_raw', y='max_pred_raw', data=metadata.dropna(subset=['avg_pred_raw']), hue='label')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, d in metadata.groupby('label'):\n    d['avg_pred_c23'].plot(kind='hist', figsize=(15, 5), bins=20, alpha=0.8, title='Average Prediction distribution c23')\n    plt.legend(['FAKE','REAL'])\nplt.show()\nfor i, d in metadata.groupby('label'):\n    d['max_pred_c23'].plot(kind='hist', figsize=(15, 5), bins=20, title='Max Prediction distribution c23', alpha=0.8)\n    plt.legend(['FAKE','REAL'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, d in metadata.groupby('label'):\n    d['avg_pred_raw'].plot(kind='hist',\n                           figsize=(15, 5),\n                           bins=20,\n                           alpha=0.8,\n                           title='Average Prediction distribution raw')\n    plt.legend(['FAKE','REAL'])\nplt.show()\nfor i, d in metadata.groupby('label'):\n    d['max_pred_raw'].plot(kind='hist',\n                           figsize=(15, 5),\n                           bins=20,\n                           title='Max Prediction distribution raw',\n                           alpha=0.8)\n    plt.legend(['FAKE','REAL'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata['max_pred_c23'] = metadata['max_pred_c23'].round(6)\nmetadata.dropna(subset=['max_pred_c23']).sort_values('label')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata['label_binary'] = 0\nmetadata.loc[metadata['label'] == \"FAKE\", 'label_binary'] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test set.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nss = pd.read_csv('../input/deepfake-detection-challenge/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_fn in tqdm(ss['filename'].unique()):\n    video_path = f'../input/deepfake-detection-challenge/test_videos/{video_fn}'\n    predictions, outputs, mean_pred, min_pred, max_pred = video_file_frame_pred(video_path, model, n_frames=4, cuda=False)\n    ss.loc[ss['filename'] == video_fn, 'avg_pred'] = mean_pred\n    ss.loc[ss['filename'] == video_fn, 'min_pred'] = min_pred\n    ss.loc[ss['filename'] == video_fn, 'max_pred'] = max_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use the Maximum frame predicted as \"Fake\" to be the final prediction\nss['label'] = ss['max_pred'].fillna(0.5).clip(0.4, 0.8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss['label'].plot(kind='hist', figsize=(15, 5), bins=50)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['filename','label']].to_csv('submission.csv', index=False)\nss.to_csv('submission_min_max.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}