{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":880837,"sourceType":"datasetVersion","datasetId":469700},{"sourceId":896104,"sourceType":"datasetVersion","datasetId":479182},{"sourceId":1003630,"sourceType":"datasetVersion","datasetId":442595}],"dockerImageVersionId":29845,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"from facenet_pytorch import MTCNN\nfrom PIL import Image\nimport torch\nfrom imutils.video import FileVideoStream\nimport cv2\nimport time\nimport glob\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nfilenames = glob.glob('/kaggle/input/deepfake-detection-challenge/test_videos/*.mp4')[:30]","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-12-24T19:12:25.963787Z","iopub.execute_input":"2023-12-24T19:12:25.964252Z","iopub.status.idle":"2023-12-24T19:12:27.970672Z","shell.execute_reply.started":"2023-12-24T19:12:25.964188Z","shell.execute_reply":"2023-12-24T19:12:27.969734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:12:27.972329Z","iopub.execute_input":"2023-12-24T19:12:27.972592Z","iopub.status.idle":"2023-12-24T19:12:27.979168Z","shell.execute_reply.started":"2023-12-24T19:12:27.972541Z","shell.execute_reply":"2023-12-24T19:12:27.978271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-2.2.7-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:11:23.411549Z","iopub.execute_input":"2023-12-24T19:11:23.411858Z","iopub.status.idle":"2023-12-24T19:11:50.999863Z","shell.execute_reply.started":"2023-12-24T19:11:23.411812Z","shell.execute_reply":"2023-12-24T19:11:50.998889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/d/meckdahl/imutils/imutils-0.5.3","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:11:51.001786Z","iopub.execute_input":"2023-12-24T19:11:51.002098Z","iopub.status.idle":"2023-12-24T19:12:19.041148Z","shell.execute_reply.started":"2023-12-24T19:11:51.002005Z","shell.execute_reply":"2023-12-24T19:12:19.040122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Haar Cascade","metadata":{}},{"cell_type":"markdown","source":"**Алгоритм:** Haar Cascade довольно простой алгоритм, использующий предподсчитанные featuremaps для пикселей различных соотношений на картинке, которые используются для обучения классификатора,  реализующегося в виде градиентного бустинга решающих деревьев. Поскольку сам предподсчет происходит динамически, он не занимает много времени, а потому главное преимущество Haar Cascade - его быстродействие.","metadata":{}},{"cell_type":"code","source":"class HaarDetector():\n\n    def __init__(self,faceCascadePath):\n        self.faceCascade=cv2.CascadeClassifier(faceCascadePath)\n\n\n    def detect(self, image, scaleFactor=1.1,\n               minNeighbors=5,\n               minSize=(5,5)):\n        \n        rects=self.faceCascade.detectMultiScale(image,\n                                                scaleFactor=scaleFactor,\n                                                minNeighbors=minNeighbors,\n                                                minSize=minSize)\n        return rects\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:29:58.597221Z","iopub.execute_input":"2023-12-24T19:29:58.597592Z","iopub.status.idle":"2023-12-24T19:29:58.604453Z","shell.execute_reply.started":"2023-12-24T19:29:58.597544Z","shell.execute_reply":"2023-12-24T19:29:58.603494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frontal_cascade_path=cv2.data.haarcascades + \"haarcascade_frontalface_default.xml\"\n\nfd=HaarDetector(frontal_cascade_path)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:29:58.938835Z","iopub.execute_input":"2023-12-24T19:29:58.939192Z","iopub.status.idle":"2023-12-24T19:29:58.978878Z","shell.execute_reply.started":"2023-12-24T19:29:58.939124Z","shell.execute_reply":"2023-12-24T19:29:58.978154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_detection_haar(filenames, scaleFactor, minNeighbors, minSize):\n    frames = []\n    frames_processed = 0\n    faces_detected = 0\n    batch_size = 60\n    start = time.time()\n\n    for filename in tqdm(filenames):\n\n        v_cap = FileVideoStream(filename).start()\n        v_len = int(v_cap.stream.get(cv2.CAP_PROP_FRAME_COUNT))\n\n        for j in range(v_len):\n\n            frame = v_cap.read()\n            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n            frames.append(frame)\n\n            if len(frames) >= batch_size or j == v_len - 1:\n                faces = []\n                for fr in frames:\n                    faces.append(fd.detect(fr,\n                               scaleFactor=scaleFactor,\n                               minNeighbors=minNeighbors,\n                               minSize=minSize))\n\n                frames_processed += len(frames)\n                faces_detected += len(faces)\n                frames = []\n\n                print(\n                    f'Frames per second: {frames_processed / (time.time() - start):.3f},',\n                    f'faces detected: {faces_detected}\\r',\n                    end=''\n                )\n\n        v_cap.stop()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:29:59.308055Z","iopub.execute_input":"2023-12-24T19:29:59.308354Z","iopub.status.idle":"2023-12-24T19:29:59.318846Z","shell.execute_reply.started":"2023-12-24T19:29:59.308309Z","shell.execute_reply":"2023-12-24T19:29:59.317989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_detection_haar(filenames, \n            scaleFactor=1.9, \n            minNeighbors=3, \n            minSize=(40,40))","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:30:49.772843Z","iopub.execute_input":"2023-12-24T19:30:49.773249Z","iopub.status.idle":"2023-12-24T19:35:47.067079Z","shell.execute_reply.started":"2023-12-24T19:30:49.77318Z","shell.execute_reply":"2023-12-24T19:35:47.066453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Haar Cascade Detection - алгоритм, применяемый в случаях, когда возникает ограниченность вычислительных возможностей, но зачастую это дает просадку в качестве детекции, особенно на данных с низким разрешением. В нашем случае на full resolution видео модель обрабатывала в среднем ~35 кадров в секунду и отметила 8995 детектированных лиц.","metadata":{}},{"cell_type":"markdown","source":"# MTCNN","metadata":{}},{"cell_type":"markdown","source":"**Алгоритм:** модель использует три нейронных сети: P-Net, R-Net и O-Net, для качественного определения ограничительной рамки, а также положения лица в трехмерном пространстве за счет фичей положения глаз, носа и рта человека. На вход каждой из сетей подается Image Pyramid кандидатов ([см. статью](https://arxiv.org/ftp/arxiv/papers/1604/)), что позволяет очень точно определить как присутствие лица на изображении, так и координаты bounding box.","metadata":{}},{"cell_type":"code","source":"class MTCNN_Detector(object):\n    \n    def __init__(self, stride=1, resize=1, *args, **kwargs):\n        self.stride = stride\n        self.resize = resize\n        self.mtcnn = MTCNN(*args, **kwargs)\n        \n    def __call__(self, frames):\n        \n        if self.resize != 1:\n            frames = [\n                cv2.resize(f, (int(f.shape[1] * self.resize), int(f.shape[0] * self.resize)))\n                    for f in frames\n            ]\n                      \n        boxes, probs = self.mtcnn.detect(frames[::self.stride])\n\n        faces = []\n        for i, frame in enumerate(frames):\n            box_ind = int(i / self.stride)\n            if boxes[box_ind] is None:\n                continue\n            for box in boxes[box_ind]:\n                box = [int(b) for b in box]\n                faces.append(frame[box[1]:box[3], box[0]:box[2]])\n        \n        return faces","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:12:36.058858Z","iopub.execute_input":"2023-12-24T19:12:36.0592Z","iopub.status.idle":"2023-12-24T19:12:36.071873Z","shell.execute_reply.started":"2023-12-24T19:12:36.059149Z","shell.execute_reply":"2023-12-24T19:12:36.070888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mtcnn = MTCNN_Detector(\n    stride=1,\n    resize=1,\n    margin=14,\n    factor=0.6,\n    keep_all=True,\n    device=device\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:13:01.650957Z","iopub.execute_input":"2023-12-24T19:13:01.65132Z","iopub.status.idle":"2023-12-24T19:13:05.827138Z","shell.execute_reply.started":"2023-12-24T19:13:01.651266Z","shell.execute_reply":"2023-12-24T19:13:05.826362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_detection_mtcnn(mtcnn, filenames):\n    frames = []\n    frames_processed = 0\n    faces_detected = 0\n    batch_size = 60\n    start = time.time()\n\n    for filename in tqdm(filenames):\n\n        v_cap = FileVideoStream(filename).start()\n        v_len = int(v_cap.stream.get(cv2.CAP_PROP_FRAME_COUNT))\n\n        for j in range(v_len):\n\n            frame = v_cap.read()\n            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n            frames.append(frame)\n\n            if len(frames) >= batch_size or j == v_len - 1:\n\n                faces = mtcnn(frames)\n\n                frames_processed += len(frames)\n                faces_detected += len(faces)\n                frames = []\n\n                print(\n                    f'Frames per second: {frames_processed / (time.time() - start):.3f},',\n                    f'faces detected: {faces_detected}\\r',\n                    end=''\n                )\n\n        v_cap.stop()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:13:49.24868Z","iopub.execute_input":"2023-12-24T19:13:49.248984Z","iopub.status.idle":"2023-12-24T19:13:49.258418Z","shell.execute_reply.started":"2023-12-24T19:13:49.248937Z","shell.execute_reply":"2023-12-24T19:13:49.257658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_detection_mtcnn(mtcnn, filenames)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:13:49.941495Z","iopub.execute_input":"2023-12-24T19:13:49.941773Z","iopub.status.idle":"2023-12-24T19:19:07.366155Z","shell.execute_reply.started":"2023-12-24T19:13:49.941731Z","shell.execute_reply":"2023-12-24T19:19:07.365337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MTCNN очень точно детектирует лица на отдельных изображениях, но из за обработки всех кадров подряд, теряет в скорости детекции. На нашем конкретном датасете MTCNN обрабатывала в среднем ~25 кадров в секунду и обнаружила 10978 лиц на всех кадрах.","metadata":{}},{"cell_type":"markdown","source":"# Fast MTCNN detector\n\n**Алгоритм:** используемый алгоритм представляет собой модификацию MTCNN, в которой распознавание лиц выполняется только для каждых _N_ кадров и применяется ко всем кадрам. Например, с батчем из 9 кадров мы передаем кадры 0, 3 и 6 в MTCNN. Затем bounding boxes, возвращенные для кадра 0, будут просто применяться к кадрам 1 и 2. Аналогичным образом, обнаруженные лица для кадра 3 применяются к кадрам 4 и 5, а лица для кадров 6 применяются к кадрам 7. и 8.\n\nХотя алгоритм требует, чтобы лица не перемещались между кадрами существенно, это, как правило, хорошее приближение для небольшого количества страйдов. Если шаг равен 3, мы полагаем, что лицо существенно не меняет положение в течение дополнительных 2 кадров, или ~0,07 секунды. Если лица движутся быстрее, они, скорее всего, в любом случае будут очень размытыми. Более того, обеспечение обрезки граней с небольшими отступами снижает влияние смещения граней.","metadata":{}},{"cell_type":"markdown","source":"Класс ниже представляет собой оболочку оригинальной MTCNN из `facenet-pytorch` для реализации описанного выше алгоритма.","metadata":{}},{"cell_type":"code","source":"class FastMTCNN(object):\n    \n    def __init__(self, stride, resize=1, *args, **kwargs):\n\n        self.stride = stride\n        self.resize = resize\n        self.mtcnn = MTCNN(*args, **kwargs)\n        \n    def __call__(self, frames):\n\n        if self.resize != 1:\n            frames = [\n                cv2.resize(f, (int(f.shape[1] * self.resize), int(f.shape[0] * self.resize)))\n                    for f in frames\n            ]\n                      \n        boxes, probs = self.mtcnn.detect(frames[::self.stride])\n\n        faces = []\n        for i, frame in enumerate(frames):\n            box_ind = int(i / self.stride)\n            if boxes[box_ind] is None:\n                continue\n            for box in boxes[box_ind]:\n                box = [int(b) for b in box]\n                faces.append(frame[box[1]:box[3], box[0]:box[2]])\n        \n        return faces","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:20:54.121483Z","iopub.execute_input":"2023-12-24T19:20:54.121843Z","iopub.status.idle":"2023-12-24T19:20:54.134193Z","shell.execute_reply.started":"2023-12-24T19:20:54.121796Z","shell.execute_reply":"2023-12-24T19:20:54.133398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fast_mtcnn = FastMTCNN(\n    stride=4,\n    resize=1,\n    margin=14,\n    factor=0.6,\n    keep_all=True,\n    device=device\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:20:55.461643Z","iopub.execute_input":"2023-12-24T19:20:55.461971Z","iopub.status.idle":"2023-12-24T19:20:55.483092Z","shell.execute_reply.started":"2023-12-24T19:20:55.46191Z","shell.execute_reply":"2023-12-24T19:20:55.48232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_detection_fmtcnn(fast_mtcnn, filenames):\n    frames = []\n    frames_processed = 0\n    faces_detected = 0\n    batch_size = 60\n    start = time.time()\n\n    for filename in tqdm(filenames):\n\n        v_cap = FileVideoStream(filename).start()\n        v_len = int(v_cap.stream.get(cv2.CAP_PROP_FRAME_COUNT))\n\n        for j in range(v_len):\n\n            frame = v_cap.read()\n            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n            frames.append(frame)\n\n            if len(frames) >= batch_size or j == v_len - 1:\n\n                faces = fast_mtcnn(frames)\n\n                frames_processed += len(frames)\n                faces_detected += len(faces)\n                frames = []\n\n                print(\n                    f'Frames per second: {frames_processed / (time.time() - start):.3f},',\n                    f'faces detected: {faces_detected}\\r',\n                    end=''\n                )\n\n        v_cap.stop()\n\nrun_detection_fmtcnn(fast_mtcnn, filenames)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T19:20:57.375693Z","iopub.execute_input":"2023-12-24T19:20:57.37651Z","iopub.status.idle":"2023-12-24T19:22:39.830512Z","shell.execute_reply.started":"2023-12-24T19:20:57.376081Z","shell.execute_reply":"2023-12-24T19:22:39.829324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"FastMTCNN значительно улучшила вычислительное время относительно MTCNN и качество детекции относительно HaarCascade со средней скоростью обработки видео ~90 кадров в секунду и 11037 детектированными лицами.","metadata":{}},{"cell_type":"markdown","source":"# Выводы","metadata":{}},{"cell_type":"markdown","source":"FastMTCNN показала наилучшие результаты на датасете с большим количеством людей в кадре по скорости обработки и точности детекции. Относительно исследованных моделей, данная является в разы превосходящей остальные.","metadata":{}}]}