{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/timesler/comparison-of-face-detection-packages　を参考にした","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%capture\n!pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-2.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\nimport torch\nfrom tqdm.notebook import tqdm\nimport time\n\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/aagfhgtpmv.mp4'\n\nreader = cv2.VideoCapture(sample)\nimages_1080_1920 = []\nimages_720_1280 = []\nimages_540_960 = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(reader.get(cv2.CAP_PROP_FRAME_WIDTH))\nprint(reader.get(cv2.CAP_PROP_FRAME_HEIGHT))\nprint(reader.get(cv2.CAP_PROP_FRAME_COUNT))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in tqdm(range(int(reader.get(cv2.CAP_PROP_FRAME_COUNT)))):\n    _, image = reader.read()\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    images_1080_1920.append(image)\n    images_720_1280.append(cv2.resize(image, (1280, 720)))\n    images_540_960.append(cv2.resize(image, (960, 540)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* OpenCVは画像を読む時、BGRになるらしい。へーへーへー。"},{"metadata":{"trusted":true},"cell_type":"code","source":"reader.release()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_1080_1920 = np.stack(images_1080_1920)\nimages_720_1280 = np.stack(images_720_1280)\nimages_540_960 = np.stack(images_540_960)\n\nprint('Shapes:')\nprint(images_1080_1920.shape)\nprint(images_720_1280.shape)\nprint(images_540_960.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_faces(images, figsize=(10.8/2, 19.2/2), start_frame=0, end_frame=0):\n    shape = images[0].shape\n    if end_frame == 0:\n        end_frame = len(images) - 1\n    images = images[np.linspace(start_frame, end_frame, 16).astype(int)]\n    im_plot = []\n    for i in range(0, 16, 4):\n        im_plot.append(np.concatenate(images[i:i+4], axis=1))\n    im_plot = np.concatenate(im_plot, axis=0)\n    \n    fig, ax = plt.subplots(1, 1, figsize=figsize)\n    ax.imshow(im_plot)\n    ax.xaxis.set_visible(False)\n    ax.yaxis.set_visible(False)\n\n    ax.grid(False)\n    fig.tight_layout()\n\ndef timer(detector, detect_fn, images, *args):\n    start = time.time()\n    faces = detect_fn(detector, images, *args)\n    elapsed = time.time() - start\n    print(f', {elapsed:.3f} seconds')\n    return faces, elapsed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(images_1080_1920, figsize=(10.8, 19.2), start_frame=0, end_frame=15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 顔検出"},{"metadata":{},"cell_type":"markdown","source":"## The facenet-pytorch package"},{"metadata":{"trusted":true},"cell_type":"code","source":"from facenet_pytorch import MTCNN\ndetector = MTCNN(device=device, post_process=False)\n\ndef detect_facenet_pytorch(detector, images, batch_size):\n    faces = []\n    for lb in np.arange(0, len(images), batch_size):\n        imgs_pil = [Image.fromarray(image) for image in images[lb:lb+batch_size]]\n        faces.extend(detector(imgs_pil))\n    return faces\n\ntimes_facenet_pytorch = []    # batched","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Detecting faces in 1080x1920 frames', end='')\nfaces, elapsed = timer(detector, detect_facenet_pytorch, images_1080_1920, 20)\ntimes_facenet_pytorch.append(elapsed)\n\nplot_faces(torch.stack(faces).permute(0, 2, 3, 1).int().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.stack(faces).shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* tensorの軸の順番の並び替えが必要らしい。いちいちめんどくさいね"},{"metadata":{},"cell_type":"markdown","source":"* xとyを入れ替えてみる"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(torch.stack(faces).permute(0, 3, 2, 1).int().numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 思ったとおりできました"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(torch.stack(faces).permute(0, 2, 3, 1).int().numpy(), start_frame=0, end_frame=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\nDATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\n\ndef play_video(video_file, subset=TRAIN_SAMPLE_FOLDER):\n    '''\n    Display video\n    param: video_file - the name of the video file to display\n    param: subset - the folder where the video file is located (can be TRAIN_SAMPLE_FOLDER or TEST_Folder)\n    '''\n    video_url = open(os.path.join(DATA_FOLDER, subset,video_file),'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n    return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"play_video(\"aagfhgtpmv.mp4\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* これはfake。顔が途中でキモいことになっている"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(torch.stack(faces).permute(0, 2, 3, 1).int().numpy(), start_frame=16, end_frame=31)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(torch.stack(faces).permute(0, 2, 3, 1).int().numpy(), start_frame=32, end_frame=48)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 口裂け男"},{"metadata":{},"cell_type":"markdown","source":"# フレーム間の顔の類似度を出してみる"},{"metadata":{},"cell_type":"markdown","source":"## 前準備としてグレーアウトする"},{"metadata":{"trusted":true},"cell_type":"code","source":"faces2 = torch.stack(faces).permute(0, 2, 3, 1).int().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(faces2[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv2.cvtColor(faces2[0], cv2.COLOR_RGB2GRAY)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* うまくいかない"},{"metadata":{"trusted":true},"cell_type":"code","source":"faces2[0].dtype","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* int32だとだめらしい（マイナスが入っていなくても、入る可能性があるdtypeだからだめ）\n\n参考：　https://qiita.com/nonbiri15/items/e8996bd157a6155a0db1"},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_gray_face = cv2.cvtColor(faces2[0].astype(\"uint16\"), cv2.COLOR_RGB2GRAY)\nplt.imshow(tmp_gray_face)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* これは俺のイメージしているgrayではない。。。"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(tmp_gray_face, cmap = \"gray\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* grayになりました"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 特徴点抽出をしてその距離を出して、類似度とする"},{"metadata":{},"cell_type":"markdown","source":"参考にしたのは　https://qiita.com/best_not_best/items/c9497ffb5240622ede01"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://tetlab117.hatenablog.com/entry/2017/09/28/163638\nbf = cv2.BFMatcher(cv2.NORM_HAMMING)\n\n# ORBとAKAZEは特徴点や特徴量を抽出するアルゴリズム\n# コメントアウトを調節することによりどちらでも行える\n\n# detector = cv2.ORB_create()\ndetector = cv2.AKAZE_create()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# まずは0フレーム目と1フレーム目の類似度\ntarget_img = cv2.cvtColor(faces2[0].astype(\"uint16\"), cv2.COLOR_RGB2GRAY)\ncomparing_img = cv2.cvtColor(faces2[1].astype(\"uint16\"), cv2.COLOR_RGB2GRAY)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax1 = fig.add_subplot(1, 2, 1)\nax1.imshow(target_img, cmap = \"gray\")\nax2 = fig.add_subplot(1, 2, 2)\nax2.imshow(comparing_img, cmap = \"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detector = cv2.AKAZE_create()\n(target_kp, target_des) = detector.detectAndCompute(target_img, None)\ntarget_kp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* うまくいかない\n* 理由は画像が小さすぎるから？\n\n参考にしたのは　https://tetlab117.hatenablog.com/entry/2017/09/28/163638　だが、ここは大きい顔写真しかない"},{"metadata":{},"cell_type":"markdown","source":"# ヒストグラム比較をやってみる"},{"metadata":{},"cell_type":"markdown","source":"* 白黒じゃなくてカラー画像に戻してからやる"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_img = faces2[0].astype(\"uint16\")\ncomparing_img = faces2[1].astype(\"uint16\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_hist = cv2.calcHist([target_img], [0], None, [256], [0, 256])\ncomparing_hist = cv2.calcHist([comparing_img], [0], None, [256], [0, 256])\nret = cv2.compareHist(target_hist, comparing_hist, 0)\nprint(ret)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 何かしらは出そう"},{"metadata":{},"cell_type":"markdown","source":"* 1フレーム目から300フレーム目までやってみる"},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_similarity = []\nfor i in range(0,299,1):\n    target_hist = cv2.calcHist([faces2[i].astype(\"uint16\")], [0], None, [256], [0, 256])\n    comparing_hist = cv2.calcHist([faces2[i+1].astype(\"uint16\")], [0], None, [256], [0, 256])\n    hist_similarity.append(cv2.compareHist(target_hist, comparing_hist, 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20, 4))\nplt.plot(hist_similarity)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_faces(torch.stack(faces).permute(0, 2, 3, 1).int().numpy(), start_frame=125, end_frame=140)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* そりゃ顔じゃねーから類似度下がるよね"},{"metadata":{},"cell_type":"markdown","source":"* 45-48フレームあたりは口裂け感が強いんだけどぱっと見よくわからないよね"},{"metadata":{},"cell_type":"markdown","source":"## fakeから10個、not fakeから10個取り出して、グラフを書いてみる"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_histgram(file):\n    sample = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/' + file\n    images_1080_1920 = []\n\n    reader = cv2.VideoCapture(sample)\n\n    for i in tqdm(range(int(reader.get(cv2.CAP_PROP_FRAME_COUNT)))):\n        _, image = reader.read()\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        images_1080_1920.append(image)\n    reader.release()\n\n    images_1080_1920 = np.stack(images_1080_1920)\n\n    detector = MTCNN(device=device, post_process=False)\n    faces, elapsed = timer(detector, detect_facenet_pytorch, images_1080_1920, 20)\n    times_facenet_pytorch.append(elapsed)\n\n    faces2 = torch.stack(faces).permute(0, 2, 3, 1).int().numpy()\n\n\n    hist_similarity = []\n    for i in range(0,100,1):\n        target_hist = cv2.calcHist([faces2[i].astype(\"uint16\")], [0], None, [256], [0, 256])\n        comparing_hist = cv2.calcHist([faces2[i+1].astype(\"uint16\")], [0], None, [256], [0, 256])\n        hist_similarity.append(cv2.compareHist(target_hist, comparing_hist, 0))\n\n    plt.figure(figsize=(20, 4))\n    plt.plot(hist_similarity)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\n\ndef get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, \"metadata.json\"))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)\nmeta_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in meta_train_df.loc[meta_train_df.label == \"FAKE\"].head(10).index.to_list():\n    plot_histgram(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# エッジ検出してみる"},{"metadata":{"trusted":true},"cell_type":"code","source":"canny_img = cv2.Canny(cv2.cvtColor(faces2[0].astype(\"uint8\"), cv2.COLOR_RGB2GRAY), 50, 110)\n# Cannyはuint16はだめらしい","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(canny_img, cmap = \"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"canny_img = cv2.Canny(cv2.cvtColor(faces2[45].astype(\"uint8\"), cv2.COLOR_RGB2GRAY), 50, 110)\nplt.imshow(canny_img, cmap = \"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}