{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Forked from https://www.kaggle.com/gpreda/deepfake-starter-kit to get sample images\n\nSome time ago I wrote YOLOv2 like algorithm trained on WIDER Face for my university project. I think that it may be useful for this competition.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n%matplotlib inline \nimport cv2 as cv\nfrom matplotlib.patches import Rectangle","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-29T14:36:12.359353Z","iopub.execute_input":"2022-03-29T14:36:12.359683Z","iopub.status.idle":"2022-03-29T14:36:12.369049Z","shell.execute_reply.started":"2022-03-29T14:36:12.359633Z","shell.execute_reply":"2022-03-29T14:36:12.368035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Everything needed for model inference code starts here","metadata":{}},{"cell_type":"code","source":"from tensorflow.python.keras.layers import Conv2D, Input, ZeroPadding2D, Dense, Lambda\nfrom tensorflow.python.keras.models import Model\nfrom tensorflow.python.keras.applications.mobilenet_v2 import MobileNetV2\nimport tensorflow as tf\ntf.compat.v1.disable_eager_execution()\nimport math\nimport numpy as np\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:12.373434Z","iopub.execute_input":"2022-03-29T14:36:12.373747Z","iopub.status.idle":"2022-03-29T14:36:12.380424Z","shell.execute_reply.started":"2022-03-29T14:36:12.373701Z","shell.execute_reply":"2022-03-29T14:36:12.379659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load most optimal model I've trained","metadata":{}},{"cell_type":"code","source":"def load_mobilenetv2_224_075_detector(path):\n    input_tensor = Input(shape=(224, 224, 3))\n    output_tensor = MobileNetV2(weights=None, include_top=False, input_tensor=input_tensor, alpha=0.75).output\n    output_tensor = ZeroPadding2D()(output_tensor)\n    output_tensor = Conv2D(kernel_size=(3, 3), filters=5)(output_tensor)\n\n    model = Model(inputs=input_tensor, outputs=output_tensor)\n    model.load_weights(path)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:12.381808Z","iopub.execute_input":"2022-03-29T14:36:12.382185Z","iopub.status.idle":"2022-03-29T14:36:12.39398Z","shell.execute_reply.started":"2022-03-29T14:36:12.382125Z","shell.execute_reply":"2022-03-29T14:36:12.39306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mobilenetv2 = load_mobilenetv2_224_075_detector(\"../input/facedetection-mobilenetv2/facedetection-mobilenetv2-size224-alpha0.75.h5\")\nmobilenetv2.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:12.395949Z","iopub.execute_input":"2022-03-29T14:36:12.39636Z","iopub.status.idle":"2022-03-29T14:36:19.751426Z","shell.execute_reply.started":"2022-03-29T14:36:12.396303Z","shell.execute_reply":"2022-03-29T14:36:19.750503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There'are 1920x1080 (16:9) and 1080x1920 (9:16) images in this competition as I can see from the samples (if you will find other, you can easily add them to SHOTS and SHOTS_T constants respectively)\n\nModel was trained on 1:1 aspect ratio images, so if we wanna use 16:9 and 9:16 images, we need to split them into 2 pieces, also we can split them to smaller (ex 10) intersecting pieces to get more accurate predictions for smaller faces.","metadata":{}},{"cell_type":"code","source":"# Converts A:B aspect rate to B:A\ndef transpose_shots(shots):\n    return [(shot[1], shot[0], shot[3], shot[2], shot[4]) for shot in shots]\n\n#That constant describe pieces for 16:9 images\nSHOTS = {\n    # fast less accurate\n    '2-16/9' : {\n        'aspect_ratio' : 16/9,\n        'shots' : [\n             (0, 0, 9/16, 1, 1),\n             (7/16, 0, 9/16, 1, 1)\n        ]\n    },\n    # slower more accurate\n    '10-16/9' : {\n        'aspect_ratio' : 16/9,\n        'shots' : [\n             (0, 0, 9/16, 1, 1),\n             (7/16, 0, 9/16, 1, 1),\n             (0, 0, 5/16, 5/9, 0.5),\n             (0, 4/9, 5/16, 5/9, 0.5),\n             (11/48, 0, 5/16, 5/9, 0.5),\n             (11/48, 4/9, 5/16, 5/9, 0.5),\n             (22/48, 0, 5/16, 5/9, 0.5),\n             (22/48, 4/9, 5/16, 5/9, 0.5),\n             (11/16, 0, 5/16, 5/9, 0.5),\n             (11/16, 4/9, 5/16, 5/9, 0.5),\n        ]\n    }\n}\n\n# 9:16 respectively\nSHOTS_T = {\n    '2-9/16' : {\n        'aspect_ratio' : 9/16,\n        'shots' : transpose_shots(SHOTS['2-16/9']['shots'])\n    },\n    '10-9/16' : {\n        'aspect_ratio' : 9/16,\n        'shots' : transpose_shots(SHOTS['10-16/9']['shots'])\n    }\n}\n\ndef r(x):\n    return int(round(x))\n\ndef sigmoid(x):\n    return 1 / (np.exp(-x) + 1)\n\ndef non_max_suppression(boxes, p, iou_threshold):\n\n    if len(boxes) == 0:\n        return np.array([])\n\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    indexes = np.argsort(p)\n    true_boxes_indexes = []\n\n    while len(indexes) > 0:\n        true_boxes_indexes.append(indexes[-1])\n\n        intersection = np.maximum(np.minimum(x2[indexes[:-1]], x2[indexes[-1]]) - np.maximum(x1[indexes[:-1]], x1[indexes[-1]]), 0) * np.maximum(np.minimum(y2[indexes[:-1]], y2[indexes[-1]]) - np.maximum(y1[indexes[:-1]], y1[indexes[-1]]), 0)\n        iou = intersection / ((x2[indexes[:-1]] - x1[indexes[:-1]]) * (y2[indexes[:-1]] - y1[indexes[:-1]]) + (x2[indexes[-1]] - x1[indexes[-1]]) * (y2[indexes[-1]] - y1[indexes[-1]]) - intersection)\n\n        indexes = np.delete(indexes, -1)\n        indexes = np.delete(indexes, np.where(iou >= iou_threshold)[0])\n\n    return boxes[true_boxes_indexes]\n\ndef union_suppression(boxes, threshold):\n    if len(boxes) == 0:\n        return np.array([])\n\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    indexes = np.argsort((x2 - x1) * (y2 - y1))\n    result_boxes = []\n\n    while len(indexes) > 0:\n        intersection = np.maximum(np.minimum(x2[indexes[:-1]], x2[indexes[-1]]) - np.maximum(x1[indexes[:-1]], x1[indexes[-1]]), 0) * np.maximum(np.minimum(y2[indexes[:-1]], y2[indexes[-1]]) - np.maximum(y1[indexes[:-1]], y1[indexes[-1]]), 0)\n        min_s = np.minimum((x2[indexes[:-1]] - x1[indexes[:-1]]) * (y2[indexes[:-1]] - y1[indexes[:-1]]), (x2[indexes[-1]] - x1[indexes[-1]]) * (y2[indexes[-1]] - y1[indexes[-1]]))\n        ioms = intersection / (min_s + 1e-9)\n        neighbours = np.where(ioms >= threshold)[0]\n        if len(neighbours) > 0:\n            result_boxes.append([min(np.min(x1[indexes[neighbours]]), x1[indexes[-1]]), min(np.min(y1[indexes[neighbours]]), y1[indexes[-1]]), max(np.max(x2[indexes[neighbours]]), x2[indexes[-1]]), max(np.max(y2[indexes[neighbours]]), y2[indexes[-1]])])\n        else:\n            result_boxes.append([x1[indexes[-1]], y1[indexes[-1]], x2[indexes[-1]], y2[indexes[-1]]])\n\n        indexes = np.delete(indexes, -1)\n        indexes = np.delete(indexes, neighbours)\n\n    return result_boxes\n\nclass FaceDetector():\n    \"\"\"\n    That's API you can easily use to detect faces\n    \n    __init__ parameters:\n    -------------------------------\n    model - model to infer\n    shots - list of aspect ratios that images could be (described earlier)\n    image_size - model's input size (hardcoded for mobilenetv2)\n    grids - model's output size (hardcoded for mobilenetv2)\n    union_threshold - threshold for union of predicted boxes within multiple shots\n    iou_threshold - IOU threshold for non maximum suppression used to merge YOLO detected boxes for one shot,\n                    you do need to change this because there are one face per image as I can see from the samples\n    prob_threshold - probability threshold for YOLO algorithm, you can balance beetween precision and recall using this threshold\n    \n    detect parameters:\n    -------------------------------\n    frame - (1920, 1080, 3) or (1080, 1920, 3) RGB Image\n    returns: list of 4 element tuples (left corner x, left corner y, right corner x, right corner y) of detected boxes within [0, 1] range (see box draw code below)\n    \"\"\"\n    def __init__(self, model=mobilenetv2, shots=[SHOTS['10-16/9'], SHOTS_T['10-9/16']], image_size=224, grids=7, iou_threshold=0.1, union_threshold=0.1, prob_threshold=0.65):\n        self.model = model\n        self.shots = shots\n        self.image_size = image_size\n        self.grids = grids\n        self.iou_threshold = iou_threshold\n        self.union_threshold = union_threshold\n        self.prob_threshold = prob_threshold\n        \n    \n    def detect(self, frame):\n        original_frame_shape = frame.shape\n\n        aspect_ratio = None\n        for shot in self.shots:\n            if abs(frame.shape[1] / frame.shape[0] - shot[\"aspect_ratio\"]) < 1e-9:\n                aspect_ratio = shot[\"aspect_ratio\"]\n                shots = shot\n        \n        assert aspect_ratio is not None\n        \n        c = min(frame.shape[0], frame.shape[1] / aspect_ratio)\n        slice_h_shift = r((frame.shape[0] - c) / 2)\n        slice_w_shift = r((frame.shape[1] - c * aspect_ratio) / 2)\n        if slice_w_shift != 0 and slice_h_shift == 0:\n            frame = frame[:, slice_w_shift:-slice_w_shift]\n        elif slice_w_shift == 0 and slice_h_shift != 0:\n            frame = frame[slice_h_shift:-slice_h_shift, :]\n\n        frames = []\n        for s in shots[\"shots\"]:\n            frames.append(cv2.resize(frame[r(s[1] * frame.shape[0]):r((s[1] + s[3]) * frame.shape[0]), r(s[0] * frame.shape[1]):r((s[0] + s[2]) * frame.shape[1])], (self.image_size, self.image_size), interpolation=cv2.INTER_NEAREST))\n        frames = np.array(frames)\n\n        predictions = self.model.predict(frames, batch_size=len(frames), verbose=0)\n\n        boxes = []\n        prob = []\n        shots = shots['shots']\n        for i in range(len(shots)):\n            slice_boxes = []\n            slice_prob = []\n            for j in range(predictions.shape[1]):\n                for k in range(predictions.shape[2]):\n                    p = sigmoid(predictions[i][j][k][4])\n                    if not(p is None) and p > self.prob_threshold:\n                        px = sigmoid(predictions[i][j][k][0])\n                        py = sigmoid(predictions[i][j][k][1])\n                        pw = min(math.exp(predictions[i][j][k][2] / self.grids), self.grids)\n                        ph = min(math.exp(predictions[i][j][k][3] / self.grids), self.grids)\n                        if not(px is None) and not(py is None) and not(pw is None) and not(ph is None) and pw > 1e-9 and ph > 1e-9:\n                            cx = (px + j) / self.grids\n                            cy = (py + k) / self.grids\n                            wx = pw / self.grids\n                            wy = ph / self.grids\n                            if wx <= shots[i][4] and wy <= shots[i][4]:\n                                lx = min(max(cx - wx / 2, 0), 1)\n                                ly = min(max(cy - wy / 2, 0), 1)\n                                rx = min(max(cx + wx / 2, 0), 1)\n                                ry = min(max(cy + wy / 2, 0), 1)\n\n                                lx *= shots[i][2]\n                                ly *= shots[i][3]\n                                rx *= shots[i][2]\n                                ry *= shots[i][3]\n\n                                lx += shots[i][0]\n                                ly += shots[i][1]\n                                rx += shots[i][0]\n                                ry += shots[i][1]\n\n                                slice_boxes.append([lx, ly, rx, ry])\n                                slice_prob.append(p)\n\n            slice_boxes = np.array(slice_boxes)\n            slice_prob = np.array(slice_prob)\n\n            slice_boxes = non_max_suppression(slice_boxes, slice_prob, self.iou_threshold)\n\n            for sb in slice_boxes:\n                boxes.append(sb)\n\n\n        boxes = np.array(boxes)\n        boxes = union_suppression(boxes, self.union_threshold)\n\n        for i in range(len(boxes)):\n            boxes[i][0] /= original_frame_shape[1] / frame.shape[1]\n            boxes[i][1] /= original_frame_shape[0] / frame.shape[0]\n            boxes[i][2] /= original_frame_shape[1] / frame.shape[1]\n            boxes[i][3] /= original_frame_shape[0] / frame.shape[0]\n\n            boxes[i][0] += slice_w_shift / original_frame_shape[1]\n            boxes[i][1] += slice_h_shift / original_frame_shape[0]\n            boxes[i][2] += slice_w_shift / original_frame_shape[1]\n            boxes[i][3] += slice_h_shift / original_frame_shape[0]\n\n        return list(boxes)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:19.754791Z","iopub.execute_input":"2022-03-29T14:36:19.755084Z","iopub.status.idle":"2022-03-29T14:36:19.839056Z","shell.execute_reply.started":"2022-03-29T14:36:19.755037Z","shell.execute_reply":"2022-03-29T14:36:19.837757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That's the end of detector code","metadata":{}},{"cell_type":"code","source":"detector = FaceDetector()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:19.840831Z","iopub.execute_input":"2022-03-29T14:36:19.841125Z","iopub.status.idle":"2022-03-29T14:36:19.854603Z","shell.execute_reply.started":"2022-03-29T14:36:19.841074Z","shell.execute_reply":"2022-03-29T14:36:19.853466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\n\ntrain_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\njson_file = [file for file in train_list if  file.endswith('json')][0]\n\ndef get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:19.855899Z","iopub.execute_input":"2022-03-29T14:36:19.856163Z","iopub.status.idle":"2022-03-29T14:36:20.043752Z","shell.execute_reply.started":"2022-03-29T14:36:19.856119Z","shell.execute_reply":"2022-03-29T14:36:20.042745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_image_from_video(video_path):\n    '''\n    input: video_path - path for video\n    process:\n    1. perform a video capture from the video\n    2. read the image\n    3. display the image\n    '''\n    capture_image = cv.VideoCapture(video_path) \n    ret, frame = capture_image.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n    ax.imshow(frame)\n    boxes = detector.detect(frame)\n    # lets's draw boxes, just multiply each predicted [0, 1] relative coordinate to image side in pixels respectively\n    for box in boxes:\n        lx = int(round(box[0] * frame.shape[1]))\n        ly = int(round(box[1] * frame.shape[0]))\n        rx = int(round(box[2] * frame.shape[1]))\n        ry = int(round(box[3] * frame.shape[0]))\n        # x, y, w, h here\n        ax.add_patch(Rectangle((lx,ly),rx - lx,ry - ly,linewidth=2,edgecolor='r',facecolor='none'))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-29T14:36:20.045881Z","iopub.execute_input":"2022-03-29T14:36:20.046156Z","iopub.status.idle":"2022-03-29T14:36:20.057433Z","shell.execute_reply.started":"2022-03-29T14:36:20.046112Z","shell.execute_reply":"2022-03-29T14:36:20.056336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fake_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='FAKE'].sample(10).index)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-29T14:36:20.059382Z","iopub.execute_input":"2022-03-29T14:36:20.059756Z","iopub.status.idle":"2022-03-29T14:36:20.078674Z","shell.execute_reply.started":"2022-03-29T14:36:20.059678Z","shell.execute_reply":"2022-03-29T14:36:20.077775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_file in fake_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-29T14:36:20.081136Z","iopub.execute_input":"2022-03-29T14:36:20.081585Z","iopub.status.idle":"2022-03-29T14:36:31.023571Z","shell.execute_reply.started":"2022-03-29T14:36:20.081489Z","shell.execute_reply":"2022-03-29T14:36:31.022782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='REAL'].sample(10).index)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-29T14:36:31.024614Z","iopub.execute_input":"2022-03-29T14:36:31.024976Z","iopub.status.idle":"2022-03-29T14:36:31.031684Z","shell.execute_reply.started":"2022-03-29T14:36:31.024938Z","shell.execute_reply":"2022-03-29T14:36:31.030694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:36:31.032985Z","iopub.execute_input":"2022-03-29T14:36:31.033257Z","iopub.status.idle":"2022-03-29T14:36:40.055837Z","shell.execute_reply.started":"2022-03-29T14:36:31.033214Z","shell.execute_reply":"2022-03-29T14:36:40.05486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please upvote, if you find this kernel useful, I'll publish train kernel so you can train another CNN lately if you would like that kernel.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}