{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":true,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":1065734,"sourceType":"datasetVersion","datasetId":591223}],"dockerImageVersionId":29844,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Novice Approach to Deepfake Detection Challenge","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"%matplotlib inline\n%load_ext autoreload\n%autoreload 2","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"execution":{"iopub.status.busy":"2023-11-30T09:31:27.480615Z","iopub.execute_input":"2023-11-30T09:31:27.480937Z","iopub.status.idle":"2023-11-30T09:31:27.503111Z","shell.execute_reply.started":"2023-11-30T09:31:27.480893Z","shell.execute_reply":"2023-11-30T09:31:27.501921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport json\nimport matplotlib.gridspec as gridspec\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom pathlib import Path\nfrom skimage.measure import compare_ssim\nfrom sklearn.metrics import log_loss, classification_report\nimport subprocess\nimport sys\nimport tensorflow as tf\nimport time\nfrom tqdm.notebook import tqdm","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"execution":{"iopub.status.busy":"2023-11-30T09:31:34.109623Z","iopub.execute_input":"2023-11-30T09:31:34.109932Z","iopub.status.idle":"2023-11-30T09:31:46.526745Z","shell.execute_reply.started":"2023-11-30T09:31:34.109886Z","shell.execute_reply":"2023-11-30T09:31:46.525894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Variables","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"KAGGLE = os.getenv('KAGGLE_KERNEL_RUN_TYPE') != None\n\nPATH = [Path('/home/jupyter/.fastai/data/dfdc'),\n            Path('/kaggle/input/deepfake-detection-challenge')][KAGGLE]\n\nVIDEOS = [PATH/'test/videos', PATH/'test_videos'][KAGGLE]\n\nDATASET_DIR = [Path('./dfdc-na'),\n               Path('/kaggle/input/dfdc-na')][KAGGLE]\n\nsys.path.append(str(DATASET_DIR))\n\nTPU_NAME = 'dfdc-2'\n\n# Scale factor applied to video frames\nFRAME_SCALE = 1.0\n\n# Size of faces to be returned from videos\nIMAGE_SIZE = (260, 260)\n\nBUCKET = None\nif BUCKET is not None:\n    from google.cloud import storage\n    client = storage.Client()\n    bucket = client.get_bucket(BUCKET)\n\nTFRECORD_DIR = [f'gs://{BUCKET}', '.'][1]\nTFRECORD_PREFIX = 'na_videos_faces'\n\n#this is the meta data from the json files with a few extra columns appended\ndf_meta = pd.read_pickle(f'{DATASET_DIR}/df_meta.pkl')\ndf_meta.cluster = df_meta.cluster.astype(np.int64)\n\n#these are the bounding box predictions from the detected faces in the original videos\ndf_boxes = pd.read_pickle(f'{DATASET_DIR}/df_boxes.pkl')\ndf_boxes['zip_no'] = df_boxes.index.get_level_values(0).map(df_meta.zip_no)\ndf_boxes['cluster'] = df_boxes.index.get_level_values(0).map(df_meta.cluster)\n\n# this filters the boxes to only those of high probability for the purpose of creating\n# the tranining data\ndf_box_probs = df_boxes.reset_index().groupby(['filename', 'box_idx']).agg({'frame_idx': 'count', 'prob': 'mean'})\ndf_box_probs = df_box_probs[(df_box_probs.prob > 0.98) & (df_box_probs.frame_idx > 15)]\n\n# This filters  the test videos to fake videos and their corresponding original videos\n# in test set so that the book runs with access to only the test videos.\ntest_names = [t.name for t in VIDEOS.glob('*.mp4')]\n\ndf_test_meta = df_meta.loc[df_meta.index.isin(test_names)\n                           & df_meta.original.isin(test_names)\n                           & (df_meta.original.isin(df_box_probs.index.get_level_values(0)))].copy()\n\n# this is a simplistic split for illustrative purposes only\ndf_test_meta['test_split'] = np.random.random(len(df_test_meta)) > 0.8\ndf_test_meta.test_split = df_test_meta.test_split.map({True: 'valid', False: 'train'})","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"execution":{"iopub.status.busy":"2023-11-30T09:31:50.884399Z","iopub.execute_input":"2023-11-30T09:31:50.884714Z","iopub.status.idle":"2023-11-30T09:31:53.836328Z","shell.execute_reply.started":"2023-11-30T09:31:50.884666Z","shell.execute_reply":"2023-11-30T09:31:53.835683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions ","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"def crop(ndarray, box):\n    \"\"\"Crops an ndarray to a given bounding box.\"\"\"\n    new_nd_array = np.zeros(ndarray.shape)\n    left, upper, right, lower = tuple(map(lambda x: int(max(0, x)), box))\n    return ndarray[slice(upper, lower),slice(left, right),:]\n\ndef iou(pred_record):\n    m = tf.keras.metrics.MeanIoU(num_classes=2)\n    m.update_state(pred_record.mask_actual,\n                   pred_record.mask_pred)\n    return m.result().numpy()\n\ndef get_fake_faces(vid_name, idxs=None, n_frames=None, orig_vid=None, box_idx=0):\n    \"\"\"Returns records of faces from fake videos.\"\"\"\n    if orig_vid is None:\n        orig_name = df_meta.loc[vid_name].original\n        orig_frames, orig_idxs = get_frames(orig_name, n_frames=n_frames, idxs=idxs)\n    else:\n        orig_name, orig_frames, orig_idxs = orig_vid\n\n    vid_frames, idxs = get_frames(vid_name, idxs=orig_idxs)\n    orig_boxes = df_boxes.loc[orig_name]\n    orig_boxes = orig_boxes.loc[orig_boxes.index.get_level_values(1)==box_idx]\n\n    # makes it possible to get boxes from frames different than the\n    # original ones that are in the df_boxes dataframe\n    vid_boxes = pd.merge_asof(pd.Series(idxs, name='new_frame_idx'),\n                     orig_boxes, left_on='new_frame_idx',\n                     right_on='frame_idx', direction='nearest').set_index('new_frame_idx')\n\n    faces_list = []\n    for frame_idx, frame, orig_frame in zip(idxs, vid_frames, orig_frames):            \n        orig_box = scale_box(vid_boxes.loc[frame_idx].square_box)\n        box = scale_center_box(orig_box)\n\n        face = crop(frame, box)\n        orig_face = crop(orig_frame, box)\n\n        thresh = get_thresh(face, orig_face)\n\n        face_mask = resize_pad_square(thresh, IMAGE_SIZE[0])\n        face_mask = np.ndarray.astype(face_mask > 1, np.bool)\n        mask_rle = dense_to_brle(face_mask.flatten())\n        fake_face = resize_pad_square(face, IMAGE_SIZE[0])\n        real_face = resize_pad_square(orig_face, IMAGE_SIZE[0])\n\n        faces_list.append({'face': fake_face,\n                           'orig_face': real_face,\n                           'mask_rle': mask_rle,\n                           'mask': face_mask,\n                           'frame_idx': frame_idx,\n                           'scale': FRAME_SCALE,\n                           'name': vid_name,\n                           'orig_name': orig_name,\n                           'box': list(box),\n                           'box_idx': box_idx,\n                           'label': df_meta.loc[vid_name].label,\n                           'label_code': df_meta.loc[vid_name].label_code,\n                           'cluster': df_meta.loc[vid_name].cluster,\n                           'zip_no': df_meta.loc[vid_name].zip_no\n                          })\n\n    return faces_list\n\ndef get_frames(vid_name, n_frames=20, start_frame=0, end_frame=None, idxs=None):\n    \"\"\"Get frames from a given video name.\"\"\"\n    vid_path = str(VIDEOS/vid_name)\n\n    v_cap = FileVideoStream(vid_path, n_frames=n_frames,\n                            start_frame=start_frame, end_frame=end_frame,\n                            idxs=idxs,transform=transform).start()\n    frames = []\n    while v_cap.running():\n        frame, idx = v_cap.read()\n        frames.append(frame)\n    return frames, v_cap.idxs\n\ndef get_real_faces(vid_name, n_frames=None, idxs=None, box_idx=0):\n    \"\"\"Returns records of faces from real videos.\"\"\"\n    vid_frames, idxs = get_frames(vid_name, idxs=idxs, n_frames=n_frames)\n    orig_boxes = df_boxes.loc[vid_name]\n    orig_boxes = orig_boxes.loc[orig_boxes.index.get_level_values(1)==box_idx]\n\n    # makes it possible to get boxes from frames different than the\n    # original ones that are in the df_boxes dataframe\n    vid_boxes = pd.merge_asof(pd.Series(idxs, name='new_frame_idx'),\n                     orig_boxes, left_on='new_frame_idx',\n                     right_on='frame_idx', direction='nearest').set_index('new_frame_idx')\n\n    faces_list = []\n    for frame_idx, orig_frame in zip(idxs, vid_frames):\n        box = scale_box(vid_boxes.loc[frame_idx].square_box)\n        box = scale_center_box(box)\n        \n        face = crop(orig_frame, box)\n        face = resize_pad_square(face, IMAGE_SIZE[0])\n        mask_rle = np.array((0, IMAGE_SIZE[0] * IMAGE_SIZE[1]), np.int64)\n        \n        faces_list.append({'face': face,\n                           'mask_rle': mask_rle,\n                           'frame_idx': frame_idx,\n                           'scale': FRAME_SCALE,\n                           'name': vid_name,\n                           'orig_name': vid_name,\n                           'box': list(box),\n                           'box_idx': box_idx,\n                           'label': df_meta.loc[vid_name].label,\n                           'label_code': df_meta.loc[vid_name].label_code,\n                           'cluster': df_meta.loc[vid_name].cluster,\n                           'zip_no': df_meta.loc[vid_name].zip_no\n                          })\n\n    return faces_list\n\ndef get_thresh(frame, orig_frame, min_diff=210):\n    \"\"\"Returns a thresholded difference between two rgb images.\"\"\"\n    frame_gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n    orig_frame_gray = cv2.cvtColor(orig_frame, cv2.COLOR_BGR2GRAY)\n    (score, diff) = compare_ssim(frame_gray, orig_frame_gray, full=True)\n    diff = (diff * 255).astype(\"uint8\")\n    thresh = cv2.threshold(diff, min_diff, 255, cv2.THRESH_BINARY_INV)[1]\n    return thresh\n\ndef plot_learning_curves(history):\n    \"\"\"Plots losses and metrics from keras model.fit object.\"\"\"\n    pd.DataFrame(history.history).plot(figsize=(8, 5))\n    plt.grid(True)\n    plt.gca().set_ylim(0, 1)\n    plt.show()\n    \ndef print_output(output):\n    \"\"\"Prints output from string.\"\"\"\n    for l in output.split('\\n'):\n        print(l)\n        \ndef print_pred_metrics(label_actual, label_pred, pred_prob):\n    \"\"\"Prints prediction evaluation metrics and report.\"\"\"\n    print(f'log loss: {log_loss(label_actual, pred_prob.clip(0.001, 0.999))}')\n    print(classification_report(label_actual, label_pred))\n    print(pd.crosstab(label_actual, label_pred, margins=True))\n\ndef resize_pad_square(ndarray, size, box=None):\n    \"\"\"Crops ndarray to box if provided, pads square and then resizes to square size.\"\"\"\n    if box != None:\n        left, upper, right, lower = box\n        new_ndarray = ndarray.copy()[max(upper,0):lower, max(left,0):right]\n    else:\n        new_ndarray = ndarray.copy()    \n\n    # pad square\n    r, c = new_ndarray.shape[:2]\n    left_pad = top_pad = right_pad = bottom_pad = 0\n    \n    if r < c:\n        top_pad = int((c - r) // 2)\n        bottom_pad = int(c - r - top_pad)\n    if c < r:\n        left_pad = int((r - c) // 2)\n        right_pad = int(r - c - left_pad) \n    \n    if sum((left_pad, top_pad, right_pad, bottom_pad)) > 0:\n        pads = ((top_pad, bottom_pad), (left_pad, right_pad), (0,0))\n\n        if np.ndim(new_ndarray) == 2:\n            pads = pads[:2]            \n            \n        new_ndarray = np.pad(new_ndarray, pads, mode='reflect')\n    \n    # resize\n    shrink = min(new_ndarray.shape[:2]) < size\n    interpolation = cv2.INTER_AREA if shrink else cv2.INTER_LINEAR\n    \n    resized_array = cv2.resize(new_ndarray, (size,size), interpolation=interpolation)\n\n    return resized_array\n        \ndef run_command(command):\n    \"\"\"Runs command line command as a subprocess returning output as string.\"\"\"\n    STDOUT = subprocess.PIPE\n    process = subprocess.run(command, shell=True, check=False,\n                             stdout=STDOUT, stderr=STDOUT, universal_newlines=True)    \n    return process.stdout\n    \ndef scale_box(box):\n    \"\"\"Used to adjust scale of box to apply to frame scaled by specified factor.\"\"\"\n    return tuple(map(lambda x: int(x * FRAME_SCALE), box))\n    \ndef scale_center_box(box, scale=1.3):\n    \"\"\"Scales box by specified factor about its center.\"\"\"\n    left, upper, right, lower = box\n    row_delta = int((lower - upper) * (1 - scale) / 2)\n    col_delta = int((right - left) * (1 - scale) / 2)\n    left += col_delta\n    right -= col_delta\n    upper += row_delta\n    lower -= row_delta\n    return left, upper, right, lower\n        \ndef show_images(imgs, titles=None, hw=(4,6), rc=(3,3)):\n    \"\"\"Show list of images with optiona list of titles.\"\"\"\n    h, w = hw\n    r, c = rc\n    fig=plt.figure(figsize=(w*c, h*r))\n    gs1 = gridspec.GridSpec(r, c, fig, hspace=0.2, wspace=0.05)\n    for i in range(r*c):\n        img = imgs[i].squeeze()\n        ax = fig.add_subplot(gs1[i])\n        if titles != None:\n            ax.set_title(titles[i], {'fontsize': 10})\n        plt.imshow(img)\n        plt.axis('off')\n    plt.show()\n\ndef show_pred(r):\n    \"\"\"Show segmentation prediction.\"\"\"\n    print(r.name, f'label: {r.label}', f'prob: {r.prob:0.4f}', f'iou: {r.iou:0.4f}')\n    show_images([r.face, r.mask_actual, r.mask_pred],\n                ['face', 'actual mask', 'predicted_mask'], hw=(3,3), rc=(1,3))\n    \ndef transform(frame):\n    \"\"\"Transform applied to frames returned from FileVideoStream.\"\"\"\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    h = int(frame.shape[0] * FRAME_SCALE)\n    w = int(frame.shape[1] * FRAME_SCALE)    \n    frame = cv2.resize(frame, (w,h))\n    return frame","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"hide_input":false,"execution":{"iopub.status.busy":"2023-11-30T09:31:59.225829Z","iopub.execute_input":"2023-11-30T09:31:59.226192Z","iopub.status.idle":"2023-11-30T09:31:59.355869Z","shell.execute_reply.started":"2023-11-30T09:31:59.226147Z","shell.execute_reply":"2023-11-30T09:31:59.355206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## More Imports ","metadata":{"heading_collapsed":true}},{"cell_type":"code","source":"if KAGGLE:\n    for package in ['classification_models', 'segmentation_models',\n                    'efficientnet', 'run_length_encoding']:\n        print_output(run_command(f\"pip install {str(DATASET_DIR/package)}\"))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"execution":{"iopub.status.busy":"2023-11-30T09:32:10.764106Z","iopub.execute_input":"2023-11-30T09:32:10.764386Z","iopub.status.idle":"2023-11-30T09:36:05.425950Z","shell.execute_reply.started":"2023-11-30T09:32:10.764344Z","shell.execute_reply":"2023-11-30T09:36:05.424944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"sys.path.append(str(DATASET_DIR))\nsys.path.append(str(DATASET_DIR/'vggface2_Keras/src'))\n\nfrom efficientnet import tfkeras as efn\nfrom filevideostream_na import FileVideoStream\nfrom rle.tf_impl import brle_to_dense\nfrom rle.np_impl import dense_to_brle, rle_length\n\nimport segmentation_models as sm\nsm.set_framework('tf.keras')\n\nfrom vggface2_Keras.src.model import Vggface2_ResNet50","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"hidden":true,"execution":{"iopub.status.busy":"2023-11-30T09:36:05.430114Z","iopub.execute_input":"2023-11-30T09:36:05.430400Z","iopub.status.idle":"2023-11-30T09:36:05.993062Z","shell.execute_reply.started":"2023-11-30T09:36:05.430356Z","shell.execute_reply":"2023-11-30T09:36:05.992362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"markdown","source":"I spent more hours than I'd care to disclose competing in this challenge. I wasn't able to create a model on my own that produced a good leaderboard score. I did, however, learn a ton about creating data pipelines, Tensorflow and TPUs and trained a lot of models. In an effort to solidify my learnings and in hopes of passing along something useful from my trials and tribulations, this book provides a few details on my approach to key parts of the challenge.\n\nI thought using TPUs would potentially be a competitive advantage in this competition given the large size of the dataset. Thank you Google for the generous grant, which once getting past the learning curve of tfrecords, definitely led to faster training and idea-iteration.\n\n**training and validation splits**\n\nDeveloping good training and validation datasets were important elements of this challenge. I tried to develop a training set comprised of correctly labeled faces, meaning that if they were labeled as fake, they in fact had altereed pixels in them (I didn't do anything with the audio). For the training and validation splits, I also wanted to create sets that were independent with respect to actor. I used some simplistic approaches early on and then a clustering analysis provided in [this kernel](https://www.kaggle.com/hmendonca/proper-clustering-with-facenet-embeddings-eda).\n\n**tfrecords**\n\nThe first section below details my tfrecod set up, which ended up being really cool. [Tfrecords](https://www.tensorflow.org/tutorials/load_data/tfrecord) are basically serialized training examples that can be read back into [tf.data.Datasets](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) super efficiently. In order to use TPUs, you create tfrecords and store them on gcs so they can be fed to the TPUs. For my tfrecord set up, I created a single tfrecord for each face detected in each video, storing each face from each frame of the video as a sequence.\n\n**datasets**\n\nThe second section focuses on the datasets that get created from the tfrecords. I really enjoyed the [tf.data.Datasets](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) api. It made it really easy to do thinks like balancing the datasets between fake and real examples, perform augmentation and preprocessing. It also made it possible to use a single set of tfrecords to feed single face cnn, sequence and segmentation models.\n\n**models**\n\nThe third section provides basic versions of the three types of models I used. There is really nothing special in the first two, a single face cnn and an multi-face LSTM. I used the excellent Efficientnet implementatios in the [qubvel/efficientnet](https://github.com/qubvel/efficientnet) repo for the cnn. I also tried using a pretrained resnet50 model trained on the vggface2 dataset from [WeidiXie/\nKeras-VGGFace2-ResNet50](https://github.com/WeidiXie/Keras-VGGFace2-ResNet50), unfreezing the weights in the last convolution block, as illustrated in the LSTM example.\n\nThe third model I tried was the one I was most optimistic about, but in the end couldn't make it work. The idea was to create segmentation masks showing the altered pixels in the fake faces by thresholding the difference between the original and fake frames and then train a segmentation model to detect the altered pixels. My thinking was that this was valuable additional information the model could use to develop relationships beyond simple binary classification. I used the excelellent implementation of a Unet with Efficientnet encoder in the [qubvel/segmentation_models](https://github.com/qubvel/segmentation_models) repo. I added a few additional layers after the segmentation output to calculate the binary classification loss and then weighted the segmentation loss and the binary classification loss. I thought this might actually help get to a better binary classification model by focusing the model's attention on the altered pixels and that it would also be useful to get predictions of which pixels were fake in addition a prediction of binary classification.","metadata":{}},{"cell_type":"code","source":"fake_name = df_test_meta[df_test_meta.label == 'FAKE'].index[0]\n\nframe_idxs = [0, 30, 60, 90]\nfaces = get_fake_faces(fake_name, idxs=frame_idxs)\n\nface_list = []\ntitle_list = []\nfor n in range(len(frame_idxs)):\n    face = faces[n]\n    keys = ['orig_face', 'face', 'mask']\n    title_list.extend([f\"fram: {face['frame_idx']}, box: {face['box_idx']} - {key[:4]}\" for key in keys])\n    face_list.extend([faces[n][key] for key in keys])\n\nshow_images(face_list, title_list, hw=(3,3), rc=(2,6))","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:37:42.305073Z","iopub.execute_input":"2023-11-30T09:37:42.305400Z","iopub.status.idle":"2023-11-30T09:37:44.355877Z","shell.execute_reply.started":"2023-11-30T09:37:42.305356Z","shell.execute_reply":"2023-11-30T09:37:44.355038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**predictions**\n\nThe last section details my approach to predictions. A couple of things worth noting. First off, the determination of a video's classification from potentially multiple faces from multiple frames was not as straightforward as simply taking the average of all of the predictions for a given video. The predictions roll up from a face detected in a frame through multiple frames and ultimately lead to a prediction for a video. I ended up pulling faces from 16 frames per video and then deciding that a video would be predicted as fake if at least three faces were predicted as such with a probability of at least 80%. I then converted that back into a probability that a video was fake for purposes of the log loss evaluation by dividing the actual number of faces over 80% by 4.5, such that a video with three frames over the 80% threshold would be considered to be fake with 67% probability and a video with 1 face over 80% would be considered to have a 22% probability of being fake.\n\nI only provide predictions for the segmentation model. Again, I couldn't make the results work, but the predictions are really cool in a lot of cases, highlighting where the faces are fake in addition to predicting whether the video is fake.\n\n**results and key learnings**\n\nThe best leaderboard score I could produce was in excess of 0.5. I was really committed to not just copying kernels with better scores and running them, but really building my own pipelines and models. I would do this differently going forward.\n\nI tried a ton of different models and datasets. I tried tightly cropped faces versus loosely cropped and even whole frame models. I tried single frame models with only one fake video per original. I tried overweighting the real videos. I tried sequences of frames from short durations of the videos as well as spread throughout the videos. I tried different pretrained weights, including vggface, vggface2 and imagenet. I tried different cnn models, from Resnet50 to different Efficientnets. I tried training my own version of an Efficientnet model on the vggface2 dataset. I tried large batch sizes, small batch sizes, freezing weights and unfreezing weights. I could never seem to make anything produce a good validation result.\n\nOn the one hand, all of those iterations were incredibly useful in just learning how to go from raw data to training a model and making predictions. On the other hand, clearly other people were figuring out how to make models work. There were obviously things I was missing or doing incorrectly that were preventing me from achieving good results. With that in mind, here is what I would have done differently in hindsight.\n\n1. I would have taken a **hybrid approach to learning from other kernels**, taking down the ones with better scores, really dissecting them to understand how they were working, and then endeavoring to recreate my own versions, especially as a novice.\n2. I would have spent more time **making sure that I had a really good validation set**. I used a couple of simplistic splits early on and then someone else's cluster analysis to approximate actor mutual exclusivity, but I was never really confident that I had a great validation set. Consequently as I was training models and watching validation results closely, I always had some doubt about whether I could trust them. Given the number of decision to make, it is really important to be able to trust your validation results, especially given the limit of two leaderboard submissions per day.\n3. I would do **more testing of my data pipeline**. There was a lot going on between the mp4 video files and the batches of training examples being fed into the models. My initial operating procedure was to write the code for the dataset that I wanted to create, then print out a few batches and if they looked basically right, send them through the model. That obviously leaves a lot of uncertainty as to whether the data that you think is running through the model is actually running through the model. Later in my develop cycle, I started doing more quantitative testing. Real-fake, original video and actor distribution within the dataset all ended up being important considerations, so I would cycle through the dataset to check the actual distribution within batches and across the entire dataset. The pipeline itself also had a lot of moving parts, that frequently broke as I was adding new features and trying new ideas. Having something akin to unit tests would have been much more efficient in the end.\n4. I would **try joining a team**. Going it alone is obviously beneficial from the standpoint of learning by having to do everything, but as a complete novice, there is a ton to learn and being able to learn some of the basics from people who have done it successfully before is ultimately likely a much faster way to get up the learning curve.","metadata":{}},{"cell_type":"markdown","source":"# Training and validation splits ","metadata":{}},{"cell_type":"markdown","source":"Creating a good training dataset turned out to be a non-trivial exercise, even for models based on frames without detecting faces. The first challenge was that the meta data didn't identify the actor in the original video, so it wasn't possible to create training and validation splits that were mutually exclusive with respect to actors based on the unique original video name. I ended up using the clustering analysis done in [this kernel](https://www.kaggle.com/hmendonca/proper-clustering-with-facenet-embeddings-eda), which essentially used a face recognition model to get an embedding for each original video and then performed an analysis to assign each to one of 700 some odd clusters. The histogram below shows the number of clusters containing the number of videos specified on the x axis.","metadata":{}},{"cell_type":"code","source":"df_clusters = pd.read_feather(str(DATASET_DIR/'face_clusters.feather'))\ndf_clusters.cluster.value_counts().hist(bins=list(range(0,100)));","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:37:52.219519Z","iopub.execute_input":"2023-11-30T09:37:52.219873Z","iopub.status.idle":"2023-11-30T09:37:52.846136Z","shell.execute_reply.started":"2023-11-30T09:37:52.219811Z","shell.execute_reply":"2023-11-30T09:37:52.844930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The median number of videos per cluster was 10 and there was one cluster with 464 entries in it. I ended up selecting a validation set of approximately 1900 original videos by selecting all clusters with less than 15, but more than five videos in it. Each of the rows below is a separate cluster with each column in each row a face from a different video in the cluster. The clustering looked like it did a decent job of identifying the actor in each video.","metadata":{"hide_input":true}},{"cell_type":"code","source":"face_list = []\ntitle_list = []\nrows, cols = (6,8)\nfor c in range(rows):\n    cluster = (df_test_meta[df_test_meta.label == 'REAL']\n               .groupby(['cluster'])\n               .count()['label_code']\n               .sort_values(ascending=False)\n              ).index[c]\n\n    for v in range(cols):\n        vid_name = df_test_meta.loc[(df_test_meta.cluster == cluster) & (df_test_meta.label == 'REAL')].index[v]\n        faces = get_real_faces(vid_name, idxs=[0])\n        face = faces[0]['face']\n        title = f'{cluster}: {vid_name[:-4]}'\n        title_list.append(title)\n        face_list.append(face)\n        \nshow_images(face_list, title_list, hw=(2,2), rc=(rows,cols))","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:36:33.639254Z","iopub.execute_input":"2023-11-30T09:36:33.639612Z","iopub.status.idle":"2023-11-30T09:36:44.355349Z","shell.execute_reply.started":"2023-11-30T09:36:33.639561Z","shell.execute_reply":"2023-11-30T09:36:44.354200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Another set of challenges to establishing a high quality training dataset were related to the labels of the videos. Not every video labeled as fake actually had altered pixels. Furthermore, some fake videos had some modified frames, and some that were not modified. Lastly, in videos that had more than one face, not all of the faces had modified pixels.\n\nThe faces from the video below illustrate these challenges, where the face from one of the frames has been modified, but the other hasn't. This set of faces also illustrates another, more fundamental challenge that I didn't solve, that of grouping faces of the same person together in sequence. You can see on the first row with faces on the left from box 0 on frame 0 and faces on the right from box 0 on frame 180. In videos with more than one face. Time permitting, I would have run each face through a face identification model, getting an embedding back and then group faces across frames by minimizing the distance from the embedding for the faces from the first frame.","metadata":{}},{"cell_type":"code","source":"orig_name = df_box_probs[df_box_probs.index.get_level_values(0)\n                         .isin(df_test_meta.index) & (df_box_probs.index.get_level_values(1) == 1)].index[2][0]\n\nfor box_idx in range(2):\n    fake_vid_name = df_test_meta[(df_test_meta.label == 'FAKE') & (df_test_meta.original == orig_name)].index[0]\n    frame_idxs = [0, 180]\n    faces = get_fake_faces(fake_vid_name, idxs=frame_idxs, box_idx=box_idx)\n\n    face_list = []\n    title_list = []\n    for n in range(len(frame_idxs)):\n        face = faces[n]\n        keys = ['orig_face', 'face', 'mask']\n        title_list.extend([f\"fram: {face['frame_idx']}, box: {face['box_idx']} - {key[:4]}\" for key in keys])\n        face_list.extend([faces[n][key] for key in keys])\n\n    show_images(face_list, title_list, hw=(3,3), rc=(1,6))","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:36:50.546030Z","iopub.execute_input":"2023-11-30T09:36:50.546363Z","iopub.status.idle":"2023-11-30T09:36:55.370579Z","shell.execute_reply.started":"2023-11-30T09:36:50.546314Z","shell.execute_reply":"2023-11-30T09:36:55.369643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tfrecords","metadata":{}},{"cell_type":"markdown","source":"[Tfrecords](https://www.tensorflow.org/tutorials/load_data/tfrecord) are serialized records of training examples that are used to create datasets that can be read super efficiently by the TPUs from cloud storage. They are essentially serialized dicts of training examples that get read and parsed to create datasets. Tensorflow recommended creating multiple approximately 150MB files so that they could be read in parallel.\n\nThe features have to be stored as lists of either `tf.int64`, `tf.float32` or `tf.string`. The cool thing was that it was possible to store multiple frames or faces in each record and then specify how many to read back at dataset creation time. This made it possible to run both single frame and sequence models from the same dataset.\n\nI also stored the run length encoding of the mask for each of the fake video faces so that segmentation models could be run from the same set of tfrecords.\n\nTo create the tfrecords, I got all of the bounding boxes for all of the original videos using [facenet-pytorch](https://github.com/timesler/facenet-pytorch), stored them in a dataframe and then looped through each of the corresponding fake videos to grab the faces from them.\n\nI also used a modified version of `imutils` `FileVideoStream` class to to pull video frames that took either a number of frames or a list of indexes of the frames to speed things up by not reading all of the frames in each video. Another time saver was setting the bounding boxes up to use the boxes from the nearest frame for which a bounding box had previously been detected.","metadata":{}},{"cell_type":"markdown","source":"## Serialize Records","metadata":{}},{"cell_type":"markdown","source":"This is just boilerplate to create serialized features of the three permissible data types for the tfrecords.","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    if type(value) != type(list()):\n        value = [value]\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    if type(value) != type(list()):\n        value = [value]\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    \n    if type(value) != type(list()):\n        value = [value]\n\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:37:02.350772Z","iopub.execute_input":"2023-11-30T09:37:02.351138Z","iopub.status.idle":"2023-11-30T09:37:02.422532Z","shell.execute_reply.started":"2023-11-30T09:37:02.351079Z","shell.execute_reply":"2023-11-30T09:37:02.421783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This stores the records created from the `get_faces` functions in a single tfrecord example. Notice how multiple faces, masks and frame indexes are serialized.","metadata":{}},{"cell_type":"code","source":"def serialize_example(records):\n    \"\"\"\n    Creates a tf.Example message ready to be written to a file.\n    \"\"\"\n    \n    meta = records[0]\n    shape = list(meta['face'].shape)\n    \n    feature = {\n        'name':       _bytes_feature(tf.compat.as_bytes(meta['name'])),\n        'original':   _bytes_feature(tf.compat.as_bytes(meta['orig_name'])),\n        'label':      _bytes_feature(tf.compat.as_bytes(meta['label'])),\n        'label_code': _int64_feature(meta['label_code']),\n        'scale':      _float_feature(meta['scale']),\n        'shape':      _int64_feature(shape),\n        'face':       _bytes_feature([tf.io.encode_jpeg(np.array(record['face'])).numpy() for record in records]),\n        'mask':       _bytes_feature([record['mask_rle'].tobytes() for record in records]),\n        'idx':        _int64_feature([record['frame_idx'] for record in records]),\n        'zip_no':     _int64_feature(meta['zip_no']),\n        'cluster':    _int64_feature(meta['cluster'])\n    }\n\n    return tf.train.Example(features=tf.train.Features(feature=feature)).SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:37:07.350080Z","iopub.execute_input":"2023-11-30T09:37:07.350420Z","iopub.status.idle":"2023-11-30T09:37:07.431369Z","shell.execute_reply.started":"2023-11-30T09:37:07.350362Z","shell.execute_reply":"2023-11-30T09:37:07.430419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fake_records = get_fake_faces(fake_name, n_frames=8)\nfake_example = serialize_example(fake_records)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:06.295714Z","iopub.execute_input":"2023-11-30T09:38:06.296317Z","iopub.status.idle":"2023-11-30T09:38:12.973125Z","shell.execute_reply.started":"2023-11-30T09:38:06.296080Z","shell.execute_reply":"2023-11-30T09:38:12.972159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_records = get_real_faces(orig_name, n_frames=8)\nreal_example = serialize_example(real_records)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:16.709992Z","iopub.execute_input":"2023-11-30T09:38:16.710332Z","iopub.status.idle":"2023-11-30T09:38:17.951264Z","shell.execute_reply.started":"2023-11-30T09:38:16.710284Z","shell.execute_reply":"2023-11-30T09:38:17.950593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parse Examples ","metadata":{}},{"cell_type":"markdown","source":"This function is set up to return a parsing function that returns a parsed record with a specified number of frames, with or without corresponding mask.","metadata":{}},{"cell_type":"code","source":"def get_parse_face_with_mask_examples_fn(take_n_faces, incl_mask=False):\n    def parse_face_examples(examples):          \n        features = {\n            'name':       tf.io.FixedLenFeature([], tf.string),\n            'original':   tf.io.FixedLenFeature([], tf.string),\n            'label':      tf.io.FixedLenFeature([], tf.string),\n            'label_code': tf.io.FixedLenFeature([], tf.int64),\n            'scale':      tf.io.FixedLenFeature([], tf.float32),\n            'shape':     tf.io.FixedLenFeature([3], tf.int64),\n            'face':     tf.io.VarLenFeature(tf.string),\n            'mask':     tf.io.VarLenFeature(tf.string),\n            'idx':       tf.io.VarLenFeature(tf.int64),\n            'zip_no': tf.io.FixedLenFeature([], tf.int64),\n            'cluster': tf.io.FixedLenFeature([], tf.int64),\n        }\n\n        parsed = tf.io.parse_example(examples, features)\n        \n        # make sure label code is consistent with label\n        label_code = tf.cast(tf.math.equal(parsed['label'], tf.convert_to_tensor('FAKE', tf.string)), tf.int64)\n        parsed['label_code'] = label_code\n    \n        def brle_to_mask(rle_bytes):\n            mask = tf.io.decode_raw(rle_bytes, tf.int64)\n            mask = tf.cast(brle_to_dense(mask), tf.uint8)\n            mask = tf.reshape(mask, (parsed['shape'][:2]))\n            return mask\n                                     \n        face = tf.map_fn(lambda x: tf.io.decode_jpeg(x), parsed['face'].values[:take_n_faces], tf.uint8)\n        parsed['face'] = face\n\n        if incl_mask:\n            mask = tf.map_fn(lambda x: brle_to_mask(x), parsed['mask'].values[:take_n_faces], tf.uint8)\n            parsed['mask'] = mask\n\n        else:\n            del parsed['mask']\n\n        \n        idx = tf.sparse.to_dense(parsed['idx'])\n        parsed['idx'] = idx[:take_n_faces]\n\n        return parsed\n\n    return parse_face_examples","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:27.095215Z","iopub.execute_input":"2023-11-30T09:38:27.095517Z","iopub.status.idle":"2023-11-30T09:38:27.170167Z","shell.execute_reply.started":"2023-11-30T09:38:27.095468Z","shell.execute_reply":"2023-11-30T09:38:27.169366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These records have gone full cycle, from cropped faces from video frames, to serialized tfecords and then parsed back out again.","metadata":{}},{"cell_type":"code","source":"show_n = 8\nparse_fn = get_parse_face_with_mask_examples_fn(show_n, incl_mask=True)\nprint(parse_fn(fake_example).keys())\n\nfor example in [fake_example, real_example]:\n    parsed_example = parse_fn(example)\n    print(parsed_example['name'].numpy(),\n          parsed_example['label'].numpy(),\n          f\"cluster: {parsed_example['cluster'].numpy()}\")\n    show_images(parsed_example['face'].numpy()[:show_n],\n                parsed_example['idx'].numpy()[:show_n].tolist(), hw=(2,2), rc=(1,show_n))\n    show_images(parsed_example['mask'].numpy()[:show_n], hw=(2,2), rc=(1,show_n))","metadata":{"_kg_hide-input":true,"hide_input":true,"execution":{"iopub.status.busy":"2023-11-30T09:38:31.824929Z","iopub.execute_input":"2023-11-30T09:38:31.825261Z","iopub.status.idle":"2023-11-30T09:38:35.116398Z","shell.execute_reply.started":"2023-11-30T09:38:31.825211Z","shell.execute_reply":"2023-11-30T09:38:35.115396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Record Files","metadata":{}},{"cell_type":"markdown","source":"The tfrecord files are created separately for real and fake videos and for training and validation splits. It was possible to filter tfrecords into training and validation splits after they are parsed, on cluster or zip file number, for example, but that ended up being much slower than doing the splits in writing the files. The real and fake are written separately so that they can be balanced efficiently for training and also because records for the fake videos can be written by looping through fakes for each real video so that frames for each real video only have to be pulled twice, instead of for each fake video (since I create masks from thresholded differences for each face).\n\nThis function just chunks up the original video file names. It includes an argument for number of files, for testing or distributing to other machines.","metadata":{}},{"cell_type":"code","source":"def get_chunks(split, label, n_per_chunk=20, n_files=None):\n    \"\"\"Returns a chunks of original video names. Specify n_files > 0 to limit number of files written.\"\"\"\n    \n    orig_names = df_test_meta[df_test_meta.test_split == split].original.unique()\n    n_chunks = len(orig_names) // n_per_chunk  \n    orig_chunks = list(enumerate(np.array_split(np.array(orig_names),n_chunks)))\n    sample_chunks = orig_chunks[:n_files]\n\n    print(f'Total {len(orig_chunks)} chunks with videos from {len(orig_chunks[0][1])} original videos in each.')\n    print(f'sample_chunks {len(sample_chunks)} chunks with videos from {len(sample_chunks[0][1])} original videos in each.')\n    \n    return sample_chunks","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:44.867618Z","iopub.execute_input":"2023-11-30T09:38:44.867919Z","iopub.status.idle":"2023-11-30T09:38:44.932908Z","shell.execute_reply.started":"2023-11-30T09:38:44.867876Z","shell.execute_reply":"2023-11-30T09:38:44.931959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function is where the tfrecords are actually written. It loops the the original video names, pulls the high probability face boxes for each and then creates records for each fake video. It takes an argument for max number of fake videos if you want to limit the number of fake videos per original video in the records.","metadata":{}},{"cell_type":"code","source":"def write_tfrecord_files(split, label, chunks, max_fakes=None):\n    start = time.perf_counter()\n    error_vids = []\n\n    for chunk_idx, chunk in chunks:\n        filename = f'{TFRECORD_PREFIX}_{split}_{label}_{chunk_idx:05d}.tfrecords'\n        record_file = f'{TFRECORD_DIR}/{filename}'\n\n        with tf.io.TFRecordWriter(record_file) as writer:\n\n            for orig_name in tqdm(chunk):\n                if label == 'FAKE':\n                    orig_frames, orig_idxs = get_frames(orig_name, n_frames=20)\n                    fake_vid_names = df_test_meta[(df_test_meta.label == 'FAKE')\n                                                 & (df_test_meta.original == orig_name)].index[:max_fakes]\n                    \n                    for vid_name in fake_vid_names:\n                        for box_idx in df_box_probs.loc[orig_name].index:\n                            try:\n                                fake_records = get_fake_faces(vid_name,\n                                                              orig_vid=(orig_name, orig_frames, orig_idxs),\n                                                              box_idx=box_idx)\n                                writer.write(serialize_example(fake_records))\n                            except:\n                                print(vid_name)\n                                error_vids.append(vid_name)\n                else:\n                    for box_idx in df_box_probs.loc[orig_name].index:\n                        for box_idx in df_box_probs.loc[orig_name].index:\n                            try:\n                                real_records = get_real_faces(orig_name, n_frames=20, box_idx=box_idx)\n                                writer.write(serialize_example(real_records))\n                            except:\n                                print(orig_name)\n                                error_vids.append(orig_name)\n\n        print(f'{filename} written to {TFRECORD_DIR}')\n\n    duration = time.perf_counter() - start\n    n_videos = len(np.concatenate([x[1] for x in chunks]))\n\n    print(f'\\nProcessed {n_videos} original videos in {duration:0.1f} seconds - avg of {duration / n_videos:0.1f} seconds per video.')\n    print(f'Estimated duration for 19,000 original videos: {duration / n_videos * 19000 / 3600:0.1f} hours.')\n\n    total_size = 0.\n    \n    if TFRECORD_DIR[:2] == 'gs':\n        for b in bucket.list_blobs(prefix=TFRECORD_PREFIX):\n            total_size += b.size\n            \n    else:\n        for p in Path(TFRECORD_DIR).glob(f'{TFRECORD_PREFIX}*'):\n            total_size += p.lstat().st_size\n    #     print(p.name, f'{p.lstat().st_size / 1e6:.1f} MB')\n\n    print(f'\\nTotal disk space of {total_size / 1e6:0.1f} MB for {n_videos} videos - avg of {total_size / n_videos / 1e6:0.1f} MB per video.')\n    print(f'Estimated disk space for 19,000 original videos: {total_size / n_videos * 19000 / 1e9:0,.1f} GB.')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:49.037428Z","iopub.execute_input":"2023-11-30T09:38:49.037814Z","iopub.status.idle":"2023-11-30T09:38:49.113791Z","shell.execute_reply.started":"2023-11-30T09:38:49.037748Z","shell.execute_reply":"2023-11-30T09:38:49.113096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    for s in ['train', 'valid']:\n        for l in ['FAKE', 'REAL']:\n            print('\\n\\033[1m', s,l, '\\033[0m')\n            chunks = get_chunks(s, l, 10, n_files=2)\n            write_tfrecord_files(s, l, chunks)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:38:53.870000Z","iopub.execute_input":"2023-11-30T09:38:53.870304Z","iopub.status.idle":"2023-11-30T09:42:15.972630Z","shell.execute_reply.started":"2023-11-30T09:38:53.870260Z","shell.execute_reply":"2023-11-30T09:42:15.971673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"markdown","source":"These dataset functions are set up to be able to create examples with single faces or multiple faces with and without masks. Augmentation includes random horizontal flip and resize with random crop. Preprocessing includes normalization for three different sets of parameters as well as resizing. Both augmentation and preprocessing are applied to both faces and corresponding masks if they are included in the dataset.","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:42:27.505366Z","iopub.execute_input":"2023-11-30T09:42:27.505693Z","iopub.status.idle":"2023-11-30T09:42:27.609683Z","shell.execute_reply.started":"2023-11-30T09:42:27.505629Z","shell.execute_reply":"2023-11-30T09:42:27.608870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flatten_frames(example):\n    n_frames = tf.shape(example['face'])[0]\n    for k in [k for k in example.keys() if k not in ['face', 'mask', 'idx', 'shape']]:\n        example[k] = tf.repeat(example[k], n_frames)\n    example['shape'] = tf.tile(tf.expand_dims(example['shape'], axis=0), (n_frames,1))\n    return example\n\ndef get_augment_fn(zoom=1.0, incl_mask=False):\n    def augment(batch):\n        new_batch = batch.copy()\n        del batch\n\n        face = new_batch['face']\n        face = tf.image.central_crop(face, zoom)\n        shape = tf.shape(face)\n\n        flip = tf.random.uniform(()) > 0.5\n\n        if flip: \n            face = tf.image.flip_left_right(face)\n\n        resize_pix = 30\n        face = tf.image.resize(face, shape[-3:-1] + resize_pix)\n        upper = tf.random.uniform((), 0, resize_pix, tf.int32)\n        left = tf.random.uniform((), 0, resize_pix, tf.int32)\n        face = face[:, upper:shape[1]+upper, left:shape[2]+left, :]\n\n        new_batch['face'] = tf.cast(face, tf.uint8)\n\n        if incl_mask:\n            mask = tf.expand_dims(new_batch['mask'], axis=-1)\n            mask = tf.image.central_crop(mask, zoom)\n\n            if flip: \n                mask = tf.image.flip_left_right(mask)\n\n            mask = tf.image.resize(mask, shape[-3:-1] + resize_pix, 'nearest')\n            mask = mask[:, upper:shape[1]+upper, left:shape[2]+left]\n            mask = tf.reshape(mask, shape[:-1])\n\n            new_batch['mask'] = tf.cast(mask, tf.uint8)\n\n        return new_batch\n\n    return augment\n    \ndef get_preprocess_fn(norm_type, batch_size=128, image_size=(224, 224), seq=False, incl_mask=False):   \n    def vgg_norm(face):\n        mean = tf.constant([91.4953, 103.8827, 131.0912])\n        \n        return face - mean\n    \n    def dfdc_norm(face):\n        mean = tf.constant([109.733734, 92.62417, 85.35359])\n        std = tf.constant([55.618042, 55.17303, 54.191914])\n\n        return (face - mean) / std\n    \n    def imagenet_norm(face):\n        mean = tf.constant([0.485, 0.456, 0.406])\n        std = tf.constant([0.229, 0.224, 0.225])\n        \n        return (face / tf.constant(255, tf.float32) - mean) / std\n \n    norm_fn = {'vgg': vgg_norm,\n               'dfdc': dfdc_norm,\n               'imagenet': imagenet_norm,\n               None: dfdc_norm\n              }\n\n    def preprocess(batch):\n        if seq:\n            shape = tf.shape(batch['face'])\n            face = tf.map_fn(lambda x: tf.image.resize(x, image_size), batch['face'], tf.float32)\n            face = tf.reshape(face, (batch_size, shape[1], *image_size, 3))\n        else:\n            face = tf.image.resize(batch['face'], image_size)\n            face = tf.reshape(face, (batch_size, *image_size, 3))\n\n        face = norm_fn[norm_type](face)\n\n        label_code = tf.cast(batch['label_code'], tf.float32)\n        label_code = tf.reshape(label_code, (batch_size,))\n        \n        if incl_mask:\n            mask = tf.cast(tf.expand_dims(batch['mask'], axis=-1), tf.float32)\n\n            if seq:\n                mask = tf.map_fn(lambda x: tf.image.resize(x, image_size, 'nearest'), mask, tf.float32)\n                mask = tf.reshape(tf.squeeze(mask), (batch_size, shape[1], *image_size))\n            else:\n                mask = tf.image.resize(mask, image_size, 'nearest')\n                mask = tf.reshape(tf.squeeze(mask), (batch_size, *image_size))\n\n            return face,  (mask, label_code)\n        \n        return face, label_code\n    \n    return preprocess","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:42:30.311826Z","iopub.execute_input":"2023-11-30T09:42:30.312185Z","iopub.status.idle":"2023-11-30T09:42:30.404641Z","shell.execute_reply.started":"2023-11-30T09:42:30.312123Z","shell.execute_reply":"2023-11-30T09:42:30.403648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function actually creates the datasets. One cool thing was that most of the `tf.image` methods work on either three or four dimensional tensors, so by changing the order of the augmentation and batching, it was possible to use the same functions to create both single frame and sequence datasets. Another nice feature was the `tf.data.experimental.sample_from_datasets` method that takes a list of datasets and an optional set of weights to create a balanced dataset.","metadata":{}},{"cell_type":"code","source":"def get_ds(split, batch_size=32, n_frames=1, incl_mask=False, seq=False, shuffle=False):\n            \n    def get_ds_label(label):\n        return (tf.data.Dataset.list_files(f'{TFRECORD_DIR}/{TFRECORD_PREFIX}_{split}_{label}_*.tfrecords',\n                                           shuffle=shuffle)\n                .interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n                .map(get_parse_face_with_mask_examples_fn(n_frames, incl_mask), num_parallel_calls=AUTO)\n                .repeat())\n        \n    ds_real = get_ds_label('REAL')\n    ds_fake = get_ds_label('FAKE')\n    \n    ds_bal = tf.data.experimental.sample_from_datasets([ds_real, ds_fake], [1., 1.], seed=42)\n    \n    if seq:\n        if split == 'train':\n            ds_bal = ds_bal.map(get_augment_fn(incl_mask=incl_mask), num_parallel_calls=AUTO)\n        \n        ds_bal = ds_bal.batch(batch_size)\n        \n    else:\n        ds_bal = ds_bal.map(flatten_frames, num_parallel_calls=AUTO).unbatch().batch(batch_size)\n\n        if split == 'train':\n            ds_bal = ds_bal.map(get_augment_fn(incl_mask=incl_mask), num_parallel_calls=AUTO)\n        \n    return ds_bal.prefetch(AUTO)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:42:36.305231Z","iopub.execute_input":"2023-11-30T09:42:36.305568Z","iopub.status.idle":"2023-11-30T09:42:36.384752Z","shell.execute_reply.started":"2023-11-30T09:42:36.305512Z","shell.execute_reply":"2023-11-30T09:42:36.383892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TFRECORD_DIR[:2] == 'gs':\n    for b in bucket.list_blobs(prefix=TFRECORD_PREFIX):\n        print(b.name, f'{b.size / 1e6:.1f} MB')\n        \nelse:\n    for p in sorted(Path(TFRECORD_DIR).glob(f'{TFRECORD_PREFIX}*')):\n        print(p.name, f'{p.lstat().st_size / 1e6:.1f} MB')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:42:40.495051Z","iopub.execute_input":"2023-11-30T09:42:40.495370Z","iopub.status.idle":"2023-11-30T09:42:40.560736Z","shell.execute_reply.started":"2023-11-30T09:42:40.495313Z","shell.execute_reply":"2023-11-30T09:42:40.559702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_ds('train', seq=False, incl_mask=True)\nfor b in ds.take(1):\n    b=b","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:42:45.555280Z","iopub.execute_input":"2023-11-30T09:42:45.555633Z","iopub.status.idle":"2023-11-30T09:42:49.190632Z","shell.execute_reply.started":"2023-11-30T09:42:45.555573Z","shell.execute_reply":"2023-11-30T09:42:49.189617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These records have gone the full, full cycle, extracted faces from video frames, serialized to tfrecords and then parsed back out into a dataset.","metadata":{}},{"cell_type":"code","source":"show_images(b['face'].numpy(), b['name'].numpy().tolist(), hw=(2,2), rc=(1,8))\nshow_images(b['mask'].numpy(), b['label'].numpy().tolist(), hw=(2,2), rc=(1,8))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-30T09:43:06.995546Z","iopub.execute_input":"2023-11-30T09:43:06.995891Z","iopub.status.idle":"2023-11-30T09:43:08.056309Z","shell.execute_reply.started":"2023-11-30T09:43:06.995843Z","shell.execute_reply":"2023-11-30T09:43:08.055434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"## Device ","metadata":{}},{"cell_type":"markdown","source":"This allows the models to run on CPU or GPU or TPU if available.","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(TPU_NAME)\n    print('Running on TPU ', tpu.master())\nexcept:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:43:27.016037Z","iopub.execute_input":"2023-11-30T09:43:27.016397Z","iopub.status.idle":"2023-11-30T09:43:47.104474Z","shell.execute_reply.started":"2023-11-30T09:43:27.016327Z","shell.execute_reply":"2023-11-30T09:43:47.103589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Single Frame CNN ","metadata":{}},{"cell_type":"markdown","source":"This is just a quick example to demonstrate how the datasets can be constructed to train a simple single frame cnn model. To train it on a TPU, the tfrecord files would need to be in cloud storage. There also aren't any callbacks here, most notably for saving model checkpoints. The model and checkpoint directories would also need to be on cloud storage in order to work with TPUs. The batch sizes could obviously be a lot larger on TPUs. I was running batch sizes as large as 2000 in some models.","metadata":{}},{"cell_type":"code","source":"image_size = (260, 260)\nbatch_size = 16\nseq = False\nincl_mask = False\n\nds_train = get_ds('train', batch_size=batch_size, n_frames=1,\n                      seq=seq, incl_mask=incl_mask, shuffle=True)\nds_valid = get_ds('valid', batch_size=batch_size, n_frames=1,\n                      seq=seq, incl_mask=incl_mask)\n\npreprocess = get_preprocess_fn('dfdc', batch_size=batch_size,\n                               image_size=image_size, seq=seq, incl_mask=incl_mask)\n\nds_train_fit = ds_train.map(preprocess, num_parallel_calls=AUTO)\nds_valid_fit = ds_valid.map(preprocess, num_parallel_calls=AUTO)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:43:56.464875Z","iopub.execute_input":"2023-11-30T09:43:56.465229Z","iopub.status.idle":"2023-11-30T09:43:57.804557Z","shell.execute_reply.started":"2023-11-30T09:43:56.465169Z","shell.execute_reply":"2023-11-30T09:43:57.803736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    opt = tf.keras.optimizers.Adam()\n    loss_fn = tf.keras.losses.BinaryCrossentropy()\n    metrics = [tf.keras.metrics.BinaryAccuracy()]\n    \n    cnn = efn.EfficientNetB2(weights=None,include_top=False,pooling='avg', input_shape=(*image_size, 3))\n    \n    model = tf.keras.Sequential([\n        cnn,\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\"),\n        tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\"),\n        tf.keras.layers.Dense(1, activation='sigmoid')\n    ])\n    \n    model.compile(loss=loss_fn, optimizer=opt, metrics=metrics)\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:44:04.070359Z","iopub.execute_input":"2023-11-30T09:44:04.070664Z","iopub.status.idle":"2023-11-30T09:44:10.942991Z","shell.execute_reply.started":"2023-11-30T09:44:04.070620Z","shell.execute_reply":"2023-11-30T09:44:10.942264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(ds_train_fit,\n                    steps_per_epoch=10,\n                    epochs=2,\n                    validation_data=ds_valid_fit,\n                    validation_steps=2\n                   )","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:44:16.660207Z","iopub.execute_input":"2023-11-30T09:44:16.660496Z","iopub.status.idle":"2023-11-30T09:44:45.355902Z","shell.execute_reply.started":"2023-11-30T09:44:16.660455Z","shell.execute_reply":"2023-11-30T09:44:45.355167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('my_cnn_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:46:56.866199Z","iopub.execute_input":"2023-11-30T09:46:56.866562Z","iopub.status.idle":"2023-11-30T09:46:58.025253Z","shell.execute_reply.started":"2023-11-30T09:46:56.866511Z","shell.execute_reply":"2023-11-30T09:46:58.024548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_curves(history)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:47:28.425045Z","iopub.execute_input":"2023-11-30T09:47:28.425364Z","iopub.status.idle":"2023-11-30T09:47:28.801828Z","shell.execute_reply.started":"2023-11-30T09:47:28.425320Z","shell.execute_reply":"2023-11-30T09:47:28.800486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sequence","metadata":{}},{"cell_type":"markdown","source":"Likewise, this is also a simple implementation to demonstrate how the datasets can be configured from the same tfrecord files to train a sequence model. This model starts with a pretrained resnet50 model trained on the vggface2 dataset from [WeidiXie/\nKeras-VGGFace2-ResNet50](https://github.com/WeidiXie/Keras-VGGFace2-ResNet50), unfreezing the weights in the last convolution block.e","metadata":{}},{"cell_type":"code","source":"batch_size = 8\nimage_size = (224, 224)\nn_frames = 16\nseq = True\nincl_mask = False\n\nds_train = get_ds('train', batch_size=batch_size, n_frames=n_frames,\n                      seq=seq, incl_mask=incl_mask, shuffle=True)\nds_valid = get_ds('valid', batch_size=batch_size, n_frames=n_frames,\n                      seq=seq, incl_mask=incl_mask)\n\npreprocess = get_preprocess_fn('vgg', batch_size=batch_size, image_size=image_size,\n                               seq=seq, incl_mask=incl_mask)\n\nds_train_fit = ds_train.map(preprocess, num_parallel_calls=AUTO)\nds_valid_fit = ds_valid.map(preprocess, num_parallel_calls=AUTO)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:47:40.265034Z","iopub.execute_input":"2023-11-30T09:47:40.265382Z","iopub.status.idle":"2023-11-30T09:47:40.794762Z","shell.execute_reply.started":"2023-11-30T09:47:40.265324Z","shell.execute_reply":"2023-11-30T09:47:40.793844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    \n    opt = tf.keras.optimizers.Adam(lr=1e-4)\n    loss_fn = tf.keras.losses.BinaryCrossentropy()\n    metrics = [tf.keras.metrics.BinaryAccuracy()]    \n\n    cnn = Vggface2_ResNet50(mode='train')\n    cnn.load_weights(str(DATASET_DIR/'vggface2_Keras/model/resnet50_softmax_dim512/weights.h5'))\n    inputs = tf.keras.layers.Input(shape=(*image_size, 3))\n    cnn = tf.keras.Model(cnn.get_layer('base_input').input, outputs=cnn.get_layer('dim_proj').output)\n\n    cnn.trainable = True\n    for l in cnn.layers[:-13]:\n        l.trainable = False\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Input((n_frames, *image_size, 3)),\n        tf.keras.layers.TimeDistributed(cnn),\n        tf.keras.layers.LSTM(256),\n        tf.keras.layers.Dense(128, activation='selu', kernel_initializer='lecun_normal'),\n        tf.keras.layers.Dense(1, activation='sigmoid'),\n    ])\n    \n    model.compile(loss=loss_fn, optimizer=opt, metrics=metrics)\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:47:44.425416Z","iopub.execute_input":"2023-11-30T09:47:44.425748Z","iopub.status.idle":"2023-11-30T09:47:54.939224Z","shell.execute_reply.started":"2023-11-30T09:47:44.425690Z","shell.execute_reply":"2023-11-30T09:47:54.938213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(ds_train_fit,\n                    steps_per_epoch=10,\n                    epochs=2,\n                    validation_data=ds_valid_fit,\n                    validation_steps=2\n                   )","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:48:00.545363Z","iopub.execute_input":"2023-11-30T09:48:00.545660Z","iopub.status.idle":"2023-11-30T09:48:22.894172Z","shell.execute_reply.started":"2023-11-30T09:48:00.545617Z","shell.execute_reply":"2023-11-30T09:48:22.893168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('resnet.h5')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:48:47.840170Z","iopub.execute_input":"2023-11-30T09:48:47.840466Z","iopub.status.idle":"2023-11-30T09:48:48.397168Z","shell.execute_reply.started":"2023-11-30T09:48:47.840425Z","shell.execute_reply":"2023-11-30T09:48:48.396459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_curves(history)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:48:58.454831Z","iopub.execute_input":"2023-11-30T09:48:58.455164Z","iopub.status.idle":"2023-11-30T09:48:58.821829Z","shell.execute_reply.started":"2023-11-30T09:48:58.455110Z","shell.execute_reply":"2023-11-30T09:48:58.820642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Segmentation","metadata":{}},{"cell_type":"markdown","source":"This segmentation model uses a Unet from the excellent [qubvel/\nsegmentation_models](https://github.com/qubvel/segmentation_models) repo, using the EfficientnetB2 cnn as the encoder. It also adds a layer after the segmentation output to calculate binary classification loss and then weights the segmentation and binary classification losses 4 to 1.\n\nI train this model for few epochs on the toy tfrecords and then load weights from a checkpoint after more extensive training to make predictions in the next section.","metadata":{}},{"cell_type":"code","source":"batch_size = 16\nimage_size = (256, 256)\nn_frames = 1\nencoder_weights = 'imagenet'\nseq = False\nincl_mask = True\n\nds_train = get_ds('train', batch_size=batch_size, n_frames=n_frames,\n                      seq=seq, incl_mask=incl_mask, shuffle=True)\nds_valid = get_ds('valid', batch_size=batch_size, n_frames=n_frames,\n                      seq=seq, incl_mask=incl_mask)\n\npreprocess = get_preprocess_fn(encoder_weights, batch_size=batch_size, image_size=image_size,\n                               seq=seq, incl_mask=incl_mask)\n\nds_train_fit = ds_train.map(preprocess, num_parallel_calls=AUTO)\nds_valid_fit = ds_valid.map(preprocess, num_parallel_calls=AUTO)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:49:18.150508Z","iopub.execute_input":"2023-11-30T09:49:18.150805Z","iopub.status.idle":"2023-11-30T09:49:19.536859Z","shell.execute_reply.started":"2023-11-30T09:49:18.150762Z","shell.execute_reply":"2023-11-30T09:49:19.536085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    class SegClassModel(tf.keras.Model):\n\n        def __init__(self):\n            super(SegClassModel, self).__init__()\n            self.seg_model = sm.Unet('efficientnetb2', encoder_weights=None, encoder_freeze=False)\n            self.pooling = tf.keras.layers.GlobalAvgPool2D()\n            self.flatten = tf.keras.layers.Flatten()\n            self.dense0 = tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\")\n            self.dense1 = tf.keras.layers.Dense(512, activation='selu', kernel_initializer=\"lecun_normal\")\n            self.final = tf.keras.layers.Dense(1, activation='sigmoid', name='class_output')\n\n        def call(self, inputs):\n            seg_output = self.seg_model(inputs)\n            class_output = self.pooling(seg_output)\n            class_output = self.flatten(class_output)\n            class_output = self.dense0(class_output)\n            class_output = self.dense1(class_output)\n            class_output = self.final(class_output)\n\n            return seg_output, class_output\n    \n    opt = tf.keras.optimizers.Adam(lr=1e-5)\n    seg_loss_fn = sm.losses.bce_jaccard_loss\n    class_loss_fn = tf.keras.losses.BinaryCrossentropy()\n    \n    seg_metrics = [sm.metrics.iou_score]\n    class_metrics = [tf.keras.metrics.BinaryAccuracy()]\n    \n    model = SegClassModel()\n\n    model.compile(loss=[seg_loss_fn, class_loss_fn],\n                  loss_weights=[4., 1.],\n                  optimizer=opt,\n                  metrics=[seg_metrics, class_metrics])\n   \n    model.build((batch_size, *image_size, 3))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:49:23.815398Z","iopub.execute_input":"2023-11-30T09:49:23.815696Z","iopub.status.idle":"2023-11-30T09:49:30.216104Z","shell.execute_reply.started":"2023-11-30T09:49:23.815652Z","shell.execute_reply":"2023-11-30T09:49:30.215347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(ds_train_fit,\n                    steps_per_epoch=10,\n                    epochs=2,\n                    validation_data=ds_valid_fit,\n                    validation_steps=2\n                   )","metadata":{"execution":{"iopub.status.busy":"2023-11-30T09:49:36.104997Z","iopub.execute_input":"2023-11-30T09:49:36.105310Z","iopub.status.idle":"2023-11-30T09:50:11.786983Z","shell.execute_reply.started":"2023-11-30T09:49:36.105267Z","shell.execute_reply":"2023-11-30T09:50:11.786283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_curves(history)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load weights from checkpoint.","metadata":{}},{"cell_type":"code","source":"cp_dir = f'{DATASET_DIR}/seg_ckpt'\ncp_filename = f'{cp_dir}/cp-0023.ckpt'\nwith strategy.scope():\n    model.load_weights(cp_filename)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"markdown","source":"For the sake of simplicity, I just use the validation set from the segmentation example above, but extend out the number of frames pulled for each video to 16 in order to improve prediction accuracy at the video level.","metadata":{}},{"cell_type":"code","source":"batch_size = 16\nimage_size = (256, 256)\nn_frames = 16\nencoder_weights = 'imagenet'\nseq = False\nincl_mask = True\n\nds_pred = get_ds('valid', batch_size=batch_size, n_frames=n_frames,\n                      seq=seq, incl_mask=incl_mask).take(32)\n\npreprocess_pred = get_preprocess_fn(encoder_weights, batch_size=batch_size, image_size=image_size,\n                               seq=seq, incl_mask=incl_mask)\n\nds_pred_pp = ds_pred.map(preprocess_pred)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T10:23:54.543525Z","iopub.execute_input":"2023-11-30T10:23:54.543901Z","iopub.status.idle":"2023-11-30T10:23:55.196801Z","shell.execute_reply.started":"2023-11-30T10:23:54.543850Z","shell.execute_reply":"2023-11-30T10:23:55.196070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure example order is deterministic so we can line up training data with predictions\nassert np.array_equal(np.concatenate([b['name'] for b in ds_pred.as_numpy_iterator()]),\n               np.concatenate([b['name'] for b in ds_pred.as_numpy_iterator()]))","metadata":{"execution":{"iopub.status.busy":"2023-11-30T10:24:03.230430Z","iopub.execute_input":"2023-11-30T10:24:03.230772Z","iopub.status.idle":"2023-11-30T10:24:04.614629Z","shell.execute_reply.started":"2023-11-30T10:24:03.230713Z","shell.execute_reply":"2023-11-30T10:24:04.613804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(ds_pred_pp)\nname_list = []\nface_list = []\nframe_list = []\nmask_list = []\nfor b in ds_pred.as_numpy_iterator():\n    face_list.extend(b['face'].squeeze())\n    name_list.extend(b['name'].squeeze())\n    if not seq:\n        frame_list.extend(b['idx'].squeeze())\n    if incl_mask:\n        b_pp = preprocess_pred(b)\n        mask_list.extend(b_pp[1][0].numpy().squeeze())","metadata":{"execution":{"iopub.status.busy":"2023-11-30T10:24:16.292592Z","iopub.execute_input":"2023-11-30T10:24:16.293037Z","iopub.status.idle":"2023-11-30T10:24:23.339906Z","shell.execute_reply.started":"2023-11-30T10:24:16.292958Z","shell.execute_reply":"2023-11-30T10:24:23.339142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = pd.DataFrame({'name': [n.decode() for n in name_list]})\n\nif incl_mask:\n    df_pred['mask_pred'] = [m.squeeze() for m in predictions[0]]\n    df_pred['mask_actual'] = [m.squeeze() for m in mask_list]\n    df_pred['iou'] = df_pred.apply(iou, axis=1)\n    df_pred['prob'] = [p.squeeze() for p in predictions[1]]\nelse:\n    def_pred['prob'] = predictions.squeeze()\n    \nif not seq:\n    df_pred['frame'] = frame_list\n\ndf_pred['face'] = face_list\ndf_pred['actual'] = df_pred.name.map(df_meta.label_code)\ndf_pred['label'] = df_pred.name.map(df_meta.label)\ndf_pred['error'] = np.square(df_pred.actual - df_pred.prob)\ndf_pred['prob_thresh'] = (df_pred.prob > 0.5).astype(int)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Face level classification ","metadata":{}},{"cell_type":"markdown","source":"This just takes every face that was predicted and calculates losses and classification metrics at the face level. The numbers here are obviously terrible since I couldn't make the underlying model work, but the predictions below, where the model got it right are pretty cool.","metadata":{}},{"cell_type":"code","source":"print_pred_metrics(df_pred.actual, df_pred.prob_thresh, df_pred.prob)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here are some examples of where the model got it right.","metadata":{}},{"cell_type":"code","source":"for r in df_pred[df_pred.actual == 1].sort_values('error').to_records()[:5]:\n    show_pred(r)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Video level classification ","metadata":{}},{"cell_type":"markdown","source":"This section would get used when running models on individual faces predictions for multiple faces for each video being made. It aggregates together the frames by video and then determines whether a video should be classified as fake if it had a specified number of frames with a predicted probability over a specified frame level probability.","metadata":{}},{"cell_type":"code","source":"vid_prob_thresh = 0.8\nframe_thresh = 3\ndf_pred['vid_prob_thresh'] = (df_pred.prob > vid_prob_thresh).astype(int)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred_vid = df_pred.groupby(['name']).agg({'actual': 'max', 'vid_prob_thresh': 'sum'})\ndf_pred_vid['vid_frame_thresh'] = (df_pred_vid.vid_prob_thresh > frame_thresh).astype(int)\ndf_pred_vid['vid_prob'] = (df_pred_vid.vid_prob_thresh / (frame_thresh * 1.5)).clip(0.05, 0.95)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_pred_metrics(df_pred_vid.actual, df_pred_vid.vid_frame_thresh, df_pred_vid.vid_prob)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Commit","metadata":{}},{"cell_type":"markdown","source":"### Update Dataset","metadata":{}},{"cell_type":"code","source":"if False:\n    if not KAGGLE:\n        print_output(run_command(f'kaggle d version -r tar -p {DATASET_DIR} -m \"added dependencies\"'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save Notebook","metadata":{}},{"cell_type":"code","source":"%%javascript\nIPython.notebook.save_notebook()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Commit Kernel","metadata":{}},{"cell_type":"code","source":"if True:\n    if not KAGGLE:\n\n        data = {'id': 'calebeverett/dfdc-novice-approach',\n                      'title': 'dfdc-novice-approach',\n                      'code_file': 'dfdc-novice-approach.ipynb',\n                      'language': 'python',\n                      'kernel_type': 'notebook',\n                      'is_private': 'false',\n                      'enable_gpu': 'false',\n                      'enable_internet': 'false',\n                      'dataset_sources': ['calebeverett/dfdc-na'],\n                      'competition_sources': ['deepfake-detection-challenge'],\n                     ' kernel_sources': []}\n        \n        with open('kernel-metadata.json', 'w') as f:\n            json.dump(data, f)\n\n        print_output(run_command('kaggle k push'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom tensorflow.keras.models import load_model\n\n# Load the pre-trained model\nmodel = load_model('/kaggle/working/my_cnn_model.h5')\n\n# Define the preprocess function\ndef preprocess(frame):\n    # Resize the frame to match the input size of your model\n    frame = cv2.resize(frame, (260, 260))\n\n    # Convert the frame to float32\n    frame = frame.astype(np.float32)\n\n    # Perform mean subtraction (if applicable)\n    # Replace 'mean_values' with the actual mean values used during training\n    mean_values = [123.68, 116.78, 103.94]  # Example mean values for RGB channels\n    frame -= mean_values\n\n    # Normalize the pixel values to the range [0, 1]\n    frame = frame / 255.0\n\n    # You might need additional preprocessing steps based on how the model was trained\n    # For example, channel-wise normalization, data augmentation, etc.\n\n    return frame\n\n# Load the video file\nvideo_path = '/kaggle/input/deepfake-detection-challenge/test_videos/aassnaulhq.mp4'\ncap = cv2.VideoCapture(video_path)\n\n# Get video details\nfps = cap.get(cv2.CAP_PROP_FPS)\nwidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\nheight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n\n# Create VideoWriter object to save the output video\noutput_path = '/kaggle/working/output_video.mp4'\nfourcc = cv2.VideoWriter_fourcc(*'mp4v')\nout = cv2.VideoWriter(output_path, fourcc, fps, (width, height))\n\n# Process each frame in the video\nwhile True:\n    ret, frame = cap.read()\n    if not ret:\n        break\n\n    # Preprocess the frame\n    preprocessed_frame = preprocess(frame)\n\n    # Make predictions using the model\n    # Replace this with your actual prediction logic based on your model\n    prediction = model.predict(np.expand_dims(preprocessed_frame, axis=0))[0]\n\n    # Visualize or post-process the prediction as needed\n    # Replace this with your actual post-processing logic\n\n    # Write the frame to the output video\n    out.write(frame)\n\n# Release video capture and writer objects\ncap.release()\nout.release()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T10:32:40.739117Z","iopub.execute_input":"2023-11-30T10:32:40.739466Z","iopub.status.idle":"2023-11-30T10:33:33.902448Z","shell.execute_reply.started":"2023-11-30T10:32:40.739420Z","shell.execute_reply":"2023-11-30T10:33:33.901519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom tensorflow.keras.models import load_model\n\n# Load the pre-trained model\nmodel = load_model('/kaggle/working/my_cnn_model.h5')\n\n# Define the preprocess function\ndef preprocess(frame):\n    # Resize the frame to match the input size of your model\n    frame = cv2.resize(frame, (260, 260))\n\n    # Convert the frame to float32\n    frame = frame.astype(np.float32)\n\n    # Perform mean subtraction (if applicable)\n    # Replace 'mean_values' with the actual mean values used during training\n    mean_values = [123.68, 116.78, 103.94]  # Example mean values for RGB channels\n    frame -= mean_values\n\n    # Normalize the pixel values to the range [0, 1]\n    frame = frame / 255.0\n\n    # You might need additional preprocessing steps based on how the model was trained\n    # For example, channel-wise normalization, data augmentation, etc.\n\n    return frame\n\n# Load the video file\nvideo_path = '/kaggle/input/deepfake-detection-challenge/test_videos/ahjnxtiamx.mp4'\ncap = cv2.VideoCapture(video_path)\n\n# Get video details\nfps = cap.get(cv2.CAP_PROP_FPS)\nwidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\nheight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n\n# Create VideoWriter object to save the output video\noutput_path = '/kaggle/working/output_videoo.mp4'\nfourcc = cv2.VideoWriter_fourcc(*'mp4v')\nout = cv2.VideoWriter(output_path, fourcc, fps, (width, height))\n\n# Process each frame in the video\nwhile True:\n    ret, frame = cap.read()\n    if not ret:\n        break\n\n    # Preprocess the frame\n    preprocessed_frame = preprocess(frame)\n\n    # Make predictions using the model\n    # Replace this with your actual prediction logic based on your model\n    prediction = model.predict(np.expand_dims(preprocessed_frame, axis=0))[0]\n\n    # Get the prediction result (assuming binary classification, adjust as needed)\n    result = \"Real\" if prediction[0] > 0.5 else \"Fake\"\n\n    # Display the prediction result on the frame\n    cv2.putText(frame, f\"Prediction: {result}\", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)\n\n    # Write the frame to the output video\n    out.write(frame)\n\n# Release video capture and writer objects\ncap.release()\nout.release()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T10:36:26.189557Z","iopub.execute_input":"2023-11-30T10:36:26.189954Z","iopub.status.idle":"2023-11-30T10:37:18.899806Z","shell.execute_reply.started":"2023-11-30T10:36:26.189887Z","shell.execute_reply":"2023-11-30T10:37:18.898794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom tensorflow.keras.models import load_model\n\ndef predict_fake_or_real(video_path):\n    # Load the pre-trained model\n    model = load_model('/kaggle/working/my_cnn_model.h5')\n\n    # Define the preprocess function\n    def preprocess(frame):\n        # Resize the frame to match the input size of your model\n        frame = cv2.resize(frame, (260, 260))\n\n        # Convert the frame to float32\n        frame = frame.astype(np.float32)\n\n        # Perform mean subtraction (if applicable)\n        # Replace 'mean_values' with the actual mean values used during training\n        mean_values = [123.68, 116.78, 103.94]  # Example mean values for RGB channels\n        frame -= mean_values\n\n        # Normalize the pixel values to the range [0, 1]\n        frame = frame / 255.0\n\n        # You might need additional preprocessing steps based on how the model was trained\n        # For example, channel-wise normalization, data augmentation, etc.\n\n        return frame\n\n    # Open the video file\n    cap = cv2.VideoCapture(video_path)\n\n    # Variables to keep track of frame counts\n    total_frames = 0\n    fake_frames = 0\n\n    # Process each frame in the video\n    while True:\n        ret, frame = cap.read()\n        if not ret:\n            break\n\n        # Preprocess the frame\n        preprocessed_frame = preprocess(frame)\n\n        # Make predictions using the model\n        # Replace this with your actual prediction logic based on your model\n        prediction = model.predict(np.expand_dims(preprocessed_frame, axis=0))[0]\n\n        # Update frame counts\n        total_frames += 1\n        fake_frames += prediction[0]  # Assuming the first element is the \"Fake\" probability\n\n    # Release video capture object\n    cap.release()\n\n    # Calculate percentages\n    real_percentage = (total_frames - fake_frames) / total_frames * 100\n    fake_percentage = fake_frames / total_frames * 100\n\n    # Determine the class with the higher percentage\n    if real_percentage > fake_percentage:\n        return \"Real\"\n    else:\n        return \"Fake\"\n\n# Example usage:\nvideo_file_path = '/kaggle/input/deepfake-detection-challenge/test_videos/bvpeerislp.mp4'\nresult = predict_fake_or_real(video_file_path)\nprint(f\"Output: {result}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T11:02:48.707121Z","iopub.execute_input":"2023-11-30T11:02:48.707520Z","iopub.status.idle":"2023-11-30T11:03:43.802390Z","shell.execute_reply.started":"2023-11-30T11:02:48.707457Z","shell.execute_reply":"2023-11-30T11:03:43.801646Z"},"trusted":true},"execution_count":null,"outputs":[]}]}