{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Face detection model comparision\nThis compares multiple face detection models, Thanks for people who shared their models, I here share a library I found too, faced, which seems pretty fast but not always accurate.\nall models suffer from imperfections as you will see below.\n\n## summary\n- yolo v2 in my opinion strikes a good balance between speed vs accuarcy\n"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n%matplotlib inline\n# %matplotlib notebook\nimport cv2 as cv\nfrom matplotlib.patches import Rectangle\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Yolo v2 Mobile Net\nsource : https://www.kaggle.com/drjerk/detect-faces-using-yolo"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Conv2D, Input, ZeroPadding2D, Dense, Lambda\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications.mobilenet_v2 import MobileNetV2\nimport tensorflow as tf\ntf.compat.v1.disable_eager_execution()\nimport math\nimport numpy as np\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_mobilenetv2_224_075_detector(path):\n    input_tensor = Input(shape=(224, 224, 3))\n    output_tensor = MobileNetV2(weights=None, include_top=False, input_tensor=input_tensor, alpha=0.75).output\n    output_tensor = ZeroPadding2D()(output_tensor)\n    output_tensor = Conv2D(kernel_size=(3, 3), filters=5)(output_tensor)\n\n    model = Model(inputs=input_tensor, outputs=output_tensor)\n    model.load_weights(path)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mobilenetv2 = load_mobilenetv2_224_075_detector(\"../input/facedetection-mobilenetv2/facedetection-mobilenetv2-size224-alpha0.75.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There'are 1920x1080 (16:9) and 1080x1920 (9:16) images in this competition as I can see from the samples (if you will find other, you can easily add them to SHOTS and SHOTS_T constants respectively)\n\nModel was trained on 1:1 aspect ratio images, so if we wanna use 16:9 and 9:16 images, we need to split them into 2 pieces, also we can split them to smaller (ex 10) intersecting pieces to get more accurate predictions for smaller faces."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converts A:B aspect rate to B:A\ndef transpose_shots(shots):\n    return [(shot[1], shot[0], shot[3], shot[2], shot[4]) for shot in shots]\n\n#That constant describe pieces for 16:9 images\nSHOTS = {\n    # fast less accurate\n    '2-16/9' : {\n        'aspect_ratio' : 16/9,\n        'shots' : [\n             (0, 0, 9/16, 1, 1),\n             (7/16, 0, 9/16, 1, 1)\n        ]\n    },\n    # slower more accurate\n    '10-16/9' : {\n        'aspect_ratio' : 16/9,\n        'shots' : [\n             (0, 0, 9/16, 1, 1),\n             (7/16, 0, 9/16, 1, 1),\n             (0, 0, 5/16, 5/9, 0.5),\n             (0, 4/9, 5/16, 5/9, 0.5),\n             (11/48, 0, 5/16, 5/9, 0.5),\n             (11/48, 4/9, 5/16, 5/9, 0.5),\n             (22/48, 0, 5/16, 5/9, 0.5),\n             (22/48, 4/9, 5/16, 5/9, 0.5),\n             (11/16, 0, 5/16, 5/9, 0.5),\n             (11/16, 4/9, 5/16, 5/9, 0.5),\n        ]\n    }\n}\n\n# 9:16 respectively\nSHOTS_T = {\n    '2-9/16' : {\n        'aspect_ratio' : 9/16,\n        'shots' : transpose_shots(SHOTS['2-16/9']['shots'])\n    },\n    '10-9/16' : {\n        'aspect_ratio' : 9/16,\n        'shots' : transpose_shots(SHOTS['10-16/9']['shots'])\n    }\n}\n\ndef r(x):\n    return int(round(x))\n\ndef sigmoid(x):\n    return 1 / (np.exp(-x) + 1)\n\ndef non_max_suppression(boxes, p, iou_threshold):\n    if len(boxes) == 0:\n        return np.array([])\n\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    indexes = np.argsort(p)\n    true_boxes_indexes = []\n\n    while len(indexes) > 0:\n        true_boxes_indexes.append(indexes[-1])\n\n        intersection = np.maximum(np.minimum(x2[indexes[:-1]], x2[indexes[-1]]) - np.maximum(x1[indexes[:-1]], x1[indexes[-1]]), 0) * np.maximum(np.minimum(y2[indexes[:-1]], y2[indexes[-1]]) - np.maximum(y1[indexes[:-1]], y1[indexes[-1]]), 0)\n        iou = intersection / ((x2[indexes[:-1]] - x1[indexes[:-1]]) * (y2[indexes[:-1]] - y1[indexes[:-1]]) + (x2[indexes[-1]] - x1[indexes[-1]]) * (y2[indexes[-1]] - y1[indexes[-1]]) - intersection)\n\n        indexes = np.delete(indexes, -1)\n        indexes = np.delete(indexes, np.where(iou >= iou_threshold)[0])\n\n    return boxes[true_boxes_indexes]\n\ndef union_suppression(boxes, threshold):\n    if len(boxes) == 0:\n        return np.array([])\n\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    indexes = np.argsort((x2 - x1) * (y2 - y1))\n    result_boxes = []\n\n    while len(indexes) > 0:\n        intersection = np.maximum(np.minimum(x2[indexes[:-1]], x2[indexes[-1]]) - np.maximum(x1[indexes[:-1]], x1[indexes[-1]]), 0) * np.maximum(np.minimum(y2[indexes[:-1]], y2[indexes[-1]]) - np.maximum(y1[indexes[:-1]], y1[indexes[-1]]), 0)\n        min_s = np.minimum((x2[indexes[:-1]] - x1[indexes[:-1]]) * (y2[indexes[:-1]] - y1[indexes[:-1]]), (x2[indexes[-1]] - x1[indexes[-1]]) * (y2[indexes[-1]] - y1[indexes[-1]]))\n        ioms = intersection / (min_s + 1e-9)\n        neighbours = np.where(ioms >= threshold)[0]\n        if len(neighbours) > 0:\n            result_boxes.append([min(np.min(x1[indexes[neighbours]]), x1[indexes[-1]]), min(np.min(y1[indexes[neighbours]]), y1[indexes[-1]]), max(np.max(x2[indexes[neighbours]]), x2[indexes[-1]]), max(np.max(y2[indexes[neighbours]]), y2[indexes[-1]])])\n        else:\n            result_boxes.append([x1[indexes[-1]], y1[indexes[-1]], x2[indexes[-1]], y2[indexes[-1]]])\n\n        indexes = np.delete(indexes, -1)\n        indexes = np.delete(indexes, neighbours)\n\n    return result_boxes\n\nclass FaceDetector():\n    \"\"\"\n    That's API you can easily use to detect faces\n    \n    __init__ parameters:\n    -------------------------------\n    model - model to infer\n    shots - list of aspect ratios that images could be (described earlier)\n    image_size - model's input size (hardcoded for mobilenetv2)\n    grids - model's output size (hardcoded for mobilenetv2)\n    union_threshold - threshold for union of predicted boxes within multiple shots\n    iou_threshold - IOU threshold for non maximum suppression used to merge YOLO detected boxes for one shot,\n                    you do need to change this because there are one face per image as I can see from the samples\n    prob_threshold - probability threshold for YOLO algorithm, you can balance beetween precision and recall using this threshold\n    \n    detect parameters:\n    -------------------------------\n    frame - (1920, 1080, 3) or (1080, 1920, 3) RGB Image\n    returns: list of 4 element tuples (left corner x, left corner y, right corner x, right corner y) of detected boxes within [0, 1] range (see box draw code below)\n    \"\"\"\n    def __init__(self, model=mobilenetv2, shots=[SHOTS['10-16/9'], SHOTS_T['10-9/16']], image_size=224, grids=7, iou_threshold=0.1, union_threshold=0.1):\n        self.model = model\n        self.shots = shots\n        self.image_size = image_size\n        self.grids = grids\n        self.iou_threshold = iou_threshold\n        self.union_threshold = union_threshold\n        self.prob_threshold = 0.7\n        \n    \n    def detect(self, frame, threshold = 0.7):\n        original_frame_shape = frame.shape\n        self.prob_threshold = threshold\n        aspect_ratio = None\n        for shot in self.shots:\n            if abs(frame.shape[1] / frame.shape[0] - shot[\"aspect_ratio\"]) < 1e-9:\n                aspect_ratio = shot[\"aspect_ratio\"]\n                shots = shot\n        \n        assert aspect_ratio is not None\n        \n        c = min(frame.shape[0], frame.shape[1] / aspect_ratio)\n        slice_h_shift = r((frame.shape[0] - c) / 2)\n        slice_w_shift = r((frame.shape[1] - c * aspect_ratio) / 2)\n        if slice_w_shift != 0 and slice_h_shift == 0:\n            frame = frame[:, slice_w_shift:-slice_w_shift]\n        elif slice_w_shift == 0 and slice_h_shift != 0:\n            frame = frame[slice_h_shift:-slice_h_shift, :]\n\n        frames = []\n        for s in shots[\"shots\"]:\n            frames.append(cv2.resize(frame[r(s[1] * frame.shape[0]):r((s[1] + s[3]) * frame.shape[0]), r(s[0] * frame.shape[1]):r((s[0] + s[2]) * frame.shape[1])], (self.image_size, self.image_size), interpolation=cv2.INTER_NEAREST))\n        frames = np.array(frames)\n\n        predictions = self.model.predict(frames, batch_size=len(frames), verbose=0)\n\n        boxes = []\n        prob = []\n        shots = shots['shots']\n        for i in range(len(shots)):\n            slice_boxes = []\n            slice_prob = []\n            for j in range(predictions.shape[1]):\n                for k in range(predictions.shape[2]):\n                    p = sigmoid(predictions[i][j][k][4])\n                    if not(p is None) and p > self.prob_threshold:\n                        px = sigmoid(predictions[i][j][k][0])\n                        py = sigmoid(predictions[i][j][k][1])\n                        pw = min(math.exp(predictions[i][j][k][2] / self.grids), self.grids)\n                        ph = min(math.exp(predictions[i][j][k][3] / self.grids), self.grids)\n                        if not(px is None) and not(py is None) and not(pw is None) and not(ph is None) and pw > 1e-9 and ph > 1e-9:\n                            cx = (px + j) / self.grids\n                            cy = (py + k) / self.grids\n                            wx = pw / self.grids\n                            wy = ph / self.grids\n                            if wx <= shots[i][4] and wy <= shots[i][4]:\n                                lx = min(max(cx - wx / 2, 0), 1)\n                                ly = min(max(cy - wy / 2, 0), 1)\n                                rx = min(max(cx + wx / 2, 0), 1)\n                                ry = min(max(cy + wy / 2, 0), 1)\n\n                                lx *= shots[i][2]\n                                ly *= shots[i][3]\n                                rx *= shots[i][2]\n                                ry *= shots[i][3]\n\n                                lx += shots[i][0]\n                                ly += shots[i][1]\n                                rx += shots[i][0]\n                                ry += shots[i][1]\n\n                                slice_boxes.append([lx, ly, rx, ry])\n                                slice_prob.append(p)\n\n            slice_boxes = np.array(slice_boxes)\n            slice_prob = np.array(slice_prob)\n\n            slice_boxes = non_max_suppression(slice_boxes, slice_prob, self.iou_threshold)\n\n            for sb in slice_boxes:\n                boxes.append(sb)\n\n\n        boxes = np.array(boxes)\n        boxes = union_suppression(boxes, self.union_threshold)\n\n        for i in range(len(boxes)):\n            boxes[i][0] /= original_frame_shape[1] / frame.shape[1]\n            boxes[i][1] /= original_frame_shape[0] / frame.shape[0]\n            boxes[i][2] /= original_frame_shape[1] / frame.shape[1]\n            boxes[i][3] /= original_frame_shape[0] / frame.shape[0]\n\n            boxes[i][0] += slice_w_shift / original_frame_shape[1]\n            boxes[i][1] += slice_h_shift / original_frame_shape[0]\n            boxes[i][2] += slice_w_shift / original_frame_shape[1]\n            boxes[i][3] += slice_h_shift / original_frame_shape[0]\n\n        return list(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_boxes_points(boxes, frame_shape):\n    result = []\n    for box in boxes:\n        lx = int(round(box[0] * frame_shape[1]))\n        ly = int(round(box[1] * frame_shape[0]))\n        rx = int(round(box[2] * frame_shape[1]))\n        ry = int(round(box[3] * frame_shape[0]))\n        result.append((lx, ly, rx, ry))\n    return result ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That's the end of detector code"},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo_model = FaceDetector()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"   ## Faced\n   source : https://github.com/iitzco/faced"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/facedpy/faced-0.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import faced \nfrom faced.utils import annotate_image\n# Receives RGB numpy image (HxWxC) and\n# returns (x_center, y_center, width, height, prob) tuples. \nfaced_model = faced.FaceDetector()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blazeface\nsource : https://www.kaggle.com/humananalog/starter-blazeface-pytorch"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport torch\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\n\nfrom blazeface import BlazeFace\n\ngpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nblazeface = BlazeFace().to(gpu)\nblazeface.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\nblazeface.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n\n# Optionally change the thresholds:\nblazeface.min_score_thresh = 0.75\nblazeface.min_suppression_threshold = 0.3\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_blaze_boxes(detections, with_keypoints=False):\n    result = []\n    if isinstance(detections, torch.Tensor):\n        detections = detections.cpu().numpy()\n\n    if detections.ndim == 1:\n        detections = np.expand_dims(detections, axis=0)\n\n    img_shape = (128, 128)\n    for i in range(detections.shape[0]):\n        ymin = detections[i, 0] * img_shape[0]\n        xmin = detections[i, 1] * img_shape[1]\n        ymax = detections[i, 2] * img_shape[0]\n        xmax = detections[i, 3] * img_shape[1]\n        result.append((xmin, ymin, xmax, ymax))\n    return result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MTCNN\nwheel source:  https://www.kaggle.com/unkownhihi/mtcnn-package"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mtcnn import MTCNN\nmtcnn = MTCNN()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# some helper functions\ndef image_resize(image, width = None, height = None, inter = cv.INTER_AREA):\n    # initialize the dimensions of the image to be resized and\n    # grab the image size\n    dim = None\n    (h, w) = image.shape[:2]\n\n    # if both the width and height are None, then return the\n    # original image\n    if width is None and height is None:\n        return image\n\n    # check to see if the width is None\n    if width is None:\n        # calculate the ratio of the height and construct the\n        # dimensions\n        r = height / float(h)\n        dim = (int(w * r), height)\n\n    # otherwise, the height is None\n    else:\n        # calculate the ratio of the width and construct the\n        # dimensions\n        r = width / float(w)\n        dim = (width, int(h * r))\n\n    # resize the image\n    resized = cv.resize(image, dim, interpolation = inter)\n\n    # return the resized image\n    return resized\n\ndef scale_boxes(boxes, scale_w, scale_h):\n    sb = []\n    for b in boxes:\n        sb.append((b[0] * scale_w, b[1] * scale_h, b[2] * scale_w, b[3] * scale_h))\n    return sb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1 frame test"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\nvideos_dir = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/'\nsample_real = f'{videos_dir}/abarnvbtwb.mp4'\nsample_fake = f'{videos_dir}/eepezmygaq.mp4'\n\ntrain_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\njson_file = [file for file in train_list if  file.endswith('json')][0]\n\ndef get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def display_image_from_video(video_path, silent=False):\n    ts = time.time()\n    cap = cv.VideoCapture(video_path)\n    _, frame = cap.read()\n    assert _ == True\n    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n    cap.release()\n    frame_copy = np.array(frame)\n\n    td = time.time()\n    bboxes = faced_model.predict(frame, 0.8)\n    print(f'faced time: {time.time() - td}')\n\n    ty = time.time()\n    yolo_boxes = yolo_model.detect(frame, 0.7)\n    print(f'yolo time: {time.time() - ty}')\n    \n    # to scale boxes back since blaze only takes 128x128\n    scale_w = frame.shape[1] / 128.0 \n    scale_h = frame.shape[0] / 128.0\n    \n    tb = time.time()\n    blaze_output = blazeface.predict_on_image(cv.resize(frame, (128,128)))\n    print(f'blaze detection: {time.time() - tb}')\n    scaled_boxes = scale_boxes(get_blaze_boxes(blaze_output), scale_w, scale_h)\n\n    tm = time.time()\n    result = mtcnn.detect_faces(frame)\n    print(f'mtcnn detection: {time.time() - tm}')\n    \n    # Plotting boxes \n    # faced\n    ann_img = annotate_image(frame_copy, bboxes)\n    fig = plt.figure(figsize=(10,10))\n\n    ax = fig.add_subplot(111)\n    ax.imshow(frame_copy)\n    ax.set_title(video_path)\n    # yolo boxes\n    yb = get_boxes_points(yolo_boxes, frame.shape)\n    \n    # mtcnn\n    for r in range(len(result)):\n        x, y, w, h = result[r]['box']\n        ax.add_patch(Rectangle((x,y),w, h, linewidth=2,edgecolor='white',facecolor='none'))\n        \n    # yolo\n    for b in yb:\n        lx, ly, rx, ry = b\n        # x, y, w, h here\n        ax.add_patch(Rectangle((lx,ly),rx - lx,ry - ly,linewidth=2,edgecolor='red',facecolor='none'))\n    \n    # blazeface\n    for b in scaled_boxes:\n        lx, ly, rx, ry = b\n        rect = Rectangle((lx, ly), rx - lx, ry - ly, linewidth=2, edgecolor=\"yellow\", facecolor=\"none\")\n        ax.add_patch(rect)\n\n    print(f'time: {time.time() - ts}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture \n# first time run takes more time to warm up models\ndisplay_image_from_video(sample_real)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image_from_video(sample_real)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Detection accuracy on different samples"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fake_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='FAKE'].sample(50).index)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%time\nfor v in fake_train_sample_video:\n    print(f'processing video {v}')\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, v))\n    print('====================================================================')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"real_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='REAL'].sample(30).index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}