{"cells":[{"metadata":{},"cell_type":"markdown","source":"All places where it is stated the code is not ours, it comes from https://www.kaggle.com/humananalog/binary-image-classifier-training-demo unless otherwise noted"},{"metadata":{},"cell_type":"markdown","source":"Import all libraries needed for the project"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys, random\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm.notebook import tqdm\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom torchvision.transforms import Normalize\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom concurrent.futures import ThreadPoolExecutor\nimport torchvision.models as models\n#https://www.kaggle.com/humananalog/deepfakes-inference-demo\nsys.path.insert(0, \"/kaggle/input/deepfakes-inference-demo\")\nfrom helpers.read_video_1 import VideoReader\nfrom helpers.face_extract_1 import FaceExtractor\n#https://www.kaggle.com/humananalog/blazeface-pytorch\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\nfrom blazeface import BlazeFace","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set up files and hyperparemters for model (not our code, but added comments)"},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n#The size of the images to bed fed in to the network\nimage_size = 224\nbatch_size = 64\ninput_size = 224\n\n#mean and std of the RGB channels in training set, precomputed\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\n#All the large image networks such as Resnet expect normalized data\nnormalize_transform = Normalize(mean, std)\n\n\n#Here is where you set up the path to import the faces_224 data and metadata\ncrops_dir = \"../input/deepfake-faces/faces_224\"\nmetadata_df = pd.read_csv(\"../input/deepfake-faces/metadata.csv\")\ntest_dir = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos/\"\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data augmentations to be used during training to increase invariance of model to non-class changing effects"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Vertical flip with 25% chance\ndef random_vflip(img, p=0.25):\n    \"\"\"Random horizontal flip.\"\"\"\n    if random.random() < p:\n        return cv2.flip(img, 0)\n    else:\n        return img\n\n#90 degree rotate with 25% chance\ndef random_rotate(img, p=0.25):\n    if random.random() < p:\n        return cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)\n    else:\n        return img\n    \n#Horizontal flip with 25% chance\ndef random_hflip(img, p=0.25):\n    \"\"\"Random horizontal flip.\"\"\"\n    if random.random() < p:\n        return cv2.flip(img, 1)\n    else:\n        return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"All the following is not our code (except a small piece which is noted), within this block is the code to load the images, create a PyTorch dataset for training, validation, and testing, code for making even splits between real and fake videos (needed due to much higher prevelance of fake videos), evaluate a model based on binary cross entropy, and code to train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_and_label(filename, cls, crops_dir, image_size, augment):\n    \"\"\"Loads an image into a tensor. Also returns its label.\"\"\"\n    img = cv2.imread(os.path.join(crops_dir, filename))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    if augment:\n        #our code\n        img = random_hflip(img)\n        img = random_vflip(img)\n        img = random_rotate(img)\n\n    img = cv2.resize(img, (image_size, image_size))\n\n    img = torch.tensor(img).permute((2, 0, 1)).float().div(255)\n    img = normalize_transform(img)\n\n    target = 1 if cls == \"FAKE\" else 0\n    return img, target\n\nclass VideoDataset(Dataset):\n    \"\"\"Face crops dataset.\n\n    Arguments:\n        crops_dir: base folder for face crops\n        df: Pandas DataFrame with metadata\n        split: if \"train\", applies data augmentation\n        image_size: resizes the image to a square of this size\n        sample_size: evenly samples this many videos from the REAL\n            and FAKE subfolders (None = use all videos)\n        seed: optional random seed for sampling\n    \"\"\"\n    def __init__(self, crops_dir, df, split, image_size, sample_size=None, seed=None):\n        self.crops_dir = crops_dir\n        self.split = split\n        self.image_size = image_size\n        \n        if sample_size is not None:\n            real_df = df[df[\"label\"] == \"REAL\"]\n            fake_df = df[df[\"label\"] == \"FAKE\"]\n            sample_size = np.min(np.array([sample_size, len(real_df), len(fake_df)]))\n            print(\"%s: sampling %d from %d real videos\" % (split, sample_size, len(real_df)))\n            print(\"%s: sampling %d from %d fake videos\" % (split, sample_size, len(fake_df)))\n            real_df = real_df.sample(sample_size, random_state=seed)\n            fake_df = fake_df.sample(sample_size, random_state=seed)\n            self.df = pd.concat([real_df, fake_df])\n        else:\n            self.df = df\n\n        num_real = len(self.df[self.df[\"label\"] == \"REAL\"])\n        num_fake = len(self.df[self.df[\"label\"] == \"FAKE\"])\n        print(\"%s dataset has %d real videos, %d fake videos\" % (split, num_real, num_fake))\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        filename = row[\"videoname\"][:-4] + \".jpg\"\n        cls = row[\"label\"]\n        return load_image_and_label(filename, cls, self.crops_dir, \n                                    self.image_size, self.split == \"train\")\n    def __len__(self):\n        return len(self.df)\n\n    \ndef make_splits(crops_dir, metadata_df, frac):\n    # Make a validation split. Sample a percentage of the real videos, \n    # and also grab the corresponding fake videos.\n    real_rows = metadata_df[metadata_df[\"label\"] == \"REAL\"]\n    real_df = real_rows.sample(frac=frac, random_state=666)\n    fake_df = metadata_df[metadata_df[\"original\"].isin(real_df[\"videoname\"])]\n    val_df = pd.concat([real_df, fake_df])\n\n    # The training split is the remaining videos.\n    train_df = metadata_df.loc[~metadata_df.index.isin(val_df.index)]\n\n    return train_df, val_df\n\ndef create_data_loaders(crops_dir, metadata_df, image_size, batch_size, num_workers):\n    train_df, val_df = make_splits(crops_dir, metadata_df, frac=0.05)\n\n    train_dataset = VideoDataset(crops_dir, train_df, \"train\", image_size, sample_size=10000)\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, \n                              num_workers=num_workers, pin_memory=True)\n\n    val_dataset = VideoDataset(crops_dir, val_df, \"val\", image_size, sample_size=500, seed=1234)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, \n                            num_workers=num_workers, pin_memory=True)\n\n    return train_loader, val_loader\n\ntrain_loader, val_loader = create_data_loaders(crops_dir, metadata_df, image_size, \n                                               batch_size, num_workers=2)\n\ndef evaluate(net, data_loader, device, silent=False):\n    net.train(False)\n\n    bce_loss = 0\n    total_examples = 0\n\n    with tqdm(total=len(data_loader), desc=\"Evaluation\", leave=False, disable=silent) as pbar:\n        for batch_idx, data in enumerate(data_loader):\n            with torch.no_grad():\n                batch_size = data[0].shape[0]\n                x = data[0].to(device)\n                y_true = data[1].to(device).float()\n\n                y_pred = net(x)\n                y_pred = y_pred.squeeze()\n\n                bce_loss += F.binary_cross_entropy_with_logits(y_pred, y_true).item() * batch_size\n\n            total_examples += batch_size\n            pbar.update()\n\n    bce_loss /= total_examples\n\n    if silent:\n        return bce_loss\n    else:\n        print(\"BCE: %.4f\" % (bce_loss))\n        \n\ndef fit(epochs):\n    global history, iteration, epochs_done, lr\n\n    with tqdm(total=len(train_loader), leave=False) as pbar:\n        for epoch in range(epochs):\n            pbar.reset()\n            pbar.set_description(\"Epoch %d\" % (epochs_done + 1))\n            \n            bce_loss = 0\n            total_examples = 0\n\n            net.train(True)\n\n            for batch_idx, data in enumerate(train_loader):\n                batch_size = data[0].shape[0]\n                x = data[0].to(gpu)\n                y_true = data[1].to(gpu).float()\n                \n                optimizer.zero_grad()\n\n                y_pred = net(x)\n                y_pred = y_pred.squeeze()\n                \n                loss = F.binary_cross_entropy_with_logits(y_pred, y_true)\n                loss.backward()\n                optimizer.step()\n                \n                batch_bce = loss.item()\n                bce_loss += batch_bce * batch_size\n                history[\"train_bce\"].append(batch_bce)\n\n                total_examples += batch_size\n                iteration += 1\n                pbar.update()\n\n            bce_loss /= total_examples\n            epochs_done += 1\n\n            print(\"Epoch: %3d, train BCE: %.4f\" % (epochs_done, bce_loss))\n\n            val_bce_loss = evaluate(net, val_loader, device=gpu, silent=True)\n            history[\"val_bce\"].append(val_bce_loss)\n            \n            print(\"              val BCE: %.4f\" % (val_bce_loss))\n\n            print(\"\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The model"},{"metadata":{},"cell_type":"markdown","source":"Load ResNext pretrained on imagenet from torchvision.  Change last layer from linear(2048,1000) to linear(2048,1) to fit our goal of using this architecture as a binary classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.resnext50_32x4d(pretrained=True)\n#model = models.wide_resnet50_2(pretrained=True)\nmodel.fc = nn.Linear(2048, 1)\nnet = model.to(gpu)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Not our code, function to freeze model up to a certain layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"def freeze_until(net, param_name):\n    found_name = False\n    for name, params in net.named_parameters():\n        if name == param_name:\n            found_name = True\n        params.requires_grad = found_name","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Not our code but tuned hyperparameters, freeze model up to certain layer, evaluate baseline, set learning rate and weight decay, train for n epochs and plot training and validation BCE."},{"metadata":{"trusted":true},"cell_type":"code","source":"#to do, play with freezing\nfreeze_until(net, \"layer4.0.conv1.weight\")\n#freeze_until(net, \"layer3.0.conv1.weight\")\n#freeze_until(net, \"fc.weight\")\n\n#print all unfreezed layers\nprint([k for k,v in net.named_parameters() if v.requires_grad])\n\nevaluate(net, val_loader, device=gpu)\n\nlr = 0.01\nwd = 0.0\n\nhistory = { \"train_bce\": [], \"val_bce\": [] }\niteration = 0\nepochs_done = 0\n\noptimizer = torch.optim.Adam(net.parameters(), lr=.05, weight_decay=wd)\n\nfit(3)\n\noptimizer = torch.optim.Adam(net.parameters(), lr=.01, weight_decay=wd)\n\nfit(3)\n\noptimizer = torch.optim.Adam(net.parameters(), lr=.001, weight_decay=wd)\n\nfit(3)\n\noptimizer = torch.optim.Adam(net.parameters(), lr=.0001, weight_decay=wd)\n\nfit(3)\n\nplt.plot(history[\"train_bce\"])\nplt.plot(history[\"val_bce\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history[\"val_bce\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Not our code, extracts 17 faces from each video using blazeface, runs faces through model and takes average"},{"metadata":{"trusted":true},"cell_type":"code","source":"facedet = BlazeFace().to(gpu)\nfacedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\nfacedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n_ = facedet.train(False)\n\nframes_per_video = 17\n\nvideo_reader = VideoReader()\nvideo_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)\nface_extractor = FaceExtractor(video_read_fn, facedet)\n\ndef isotropically_resize_image(img, size, resample=cv2.INTER_AREA):\n    h, w = img.shape[:2]\n    if w > h:\n        h = h * size // w\n        w = size\n    else:\n        w = w * size // h\n        h = size\n\n    resized = cv2.resize(img, (w, h), interpolation=resample)\n    return resized\n\n\ndef make_square_image(img):\n    h, w = img.shape[:2]\n    size = max(h, w)\n    t = 0\n    b = size - h\n    l = 0\n    r = size - w\n    return cv2.copyMakeBorder(img, t, b, l, r, cv2.BORDER_CONSTANT, value=0)\n\n\ndef predict_on_video(video_path, batch_size):\n    try:\n        # Find the faces for N frames in the video.\n        faces = face_extractor.process_video(video_path)\n\n        # Only look at one face per frame.\n        face_extractor.keep_only_best_face(faces)\n        \n        if len(faces) > 0:\n            # NOTE: When running on the CPU, the batch size must be fixed\n            # or else memory usage will blow up. (Bug in PyTorch?)\n            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n\n            # If we found any faces, prepare them for the model.\n            n = 0\n            for frame_data in faces:\n                for face in frame_data[\"faces\"]:\n                    # Resize to the model's required input size.\n                    # We keep the aspect ratio intact and add zero\n                    # padding if necessary.                    \n                    resized_face = isotropically_resize_image(face, input_size)\n                    resized_face = make_square_image(resized_face)\n\n                    if n < batch_size:\n                        x[n] = resized_face\n                        n += 1\n                    else:\n                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n                    \n\n            if n > 0:\n                x = torch.tensor(x, device=gpu).float()\n\n                # Preprocess the images.\n                x = x.permute((0, 3, 1, 2))\n\n                for i in range(len(x)):\n                    x[i] = normalize_transform(x[i] / 255.)\n\n                # Make a prediction, then take the average.\n                with torch.no_grad():\n                    y_pred = net(x)\n                    y_pred = torch.sigmoid(y_pred.squeeze())\n                    return y_pred[:n].mean().item()\n\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n\n    return 0.5\n\ndef predict_on_video_set(videos, num_workers):\n    def process_file(i):\n        filename = videos[i]\n        y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)\n        return y_pred\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        predictions = ex.map(process_file, range(len(videos)))\n\n    return list(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following is the code to run the actual pipeline (besides the model), we wrote this besides the function predict_on_video_set which it calls and some other small details.  In the end it gives an accuracy using the actual videos as input on 2X test videos (X real, X fake).  The system is supposed to predict 1 if real and 0 if fake.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"def h(name):\n    if name == \"FAKE\":\n        return 1.0\n    else:\n        return 0.0\n\nc = 30\ni = 0\nj = 0\nright = 0\nlab = []\npred = []\nwhile j < c:\n    a = test_videos[i:i+1][0]\n    if len(np.argwhere(metadata_df[\"videoname\"]==a)) == 0:\n        i += 1\n        continue\n    a = np.argwhere(metadata_df[\"videoname\"]==a).item()\n    a = metadata_df.iloc[a, :]\n    if a[\"label\"] == \"FAKE\":\n        i += 1\n        continue\n    predictions = predict_on_video_set(test_videos[i:i+1], num_workers=4)\n    #print(a[\"videoname\"], a[\"label\"], predictions[0])\n    lab.append(h(a[\"label\"]))\n    pred.append(predictions[0])\n    if np.rint(predictions[0]) == h(a[\"label\"]):\n        right +=1\n    i += 1\n    j += 1\n    print(j)\ni = 0\nj = 0\nwhile j < c:\n    print(j)\n    a = test_videos[i:i+1][0]\n    if len(np.argwhere(metadata_df[\"videoname\"]==a)) == 0:\n        i+=1\n        continue\n    a = np.argwhere(metadata_df[\"videoname\"]==a).item()\n    a = metadata_df.iloc[a, :]\n    if a[\"label\"] == \"REAL\":\n        i+=1\n        continue\n    predictions = predict_on_video_set(test_videos[i:i+1], num_workers=4)\n    #print(a[\"videoname\"], a[\"label\"], predictions[0])\n    lab.append(h(a[\"label\"]))\n    pred.append(predictions[0])\n    if np.rint(predictions[0]) == h(a[\"label\"]):\n        right +=1\n    i += 1\n    j += 1\n    print(j)\nprint(\"accuracy\", right/(c*2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(pred)):\n    a = 0.0\n    if pred[i] >= .1:\n        a = 1.0\n    if a == lab[i]:\n        #print(pred[i], t, lab[i])\n        right +=1.0\nprint(best_right/len(pred), best_t)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}