{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/deepfake-detection-jph/facenet_pytorch-2.0.1-py3-none-any.whl\n!cp -r /kaggle/input/pretrainedmodels /kaggle/working/; cd /kaggle/working/pretrainedmodels/pretrained-models.pytorch ; pip install .\n!pip install /kaggle/input/deepfake-detection-jph/efficientnet_pytorch-0.6.1/efficientnet_pytorch-0.6.1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install facenet-pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Data Exploration**\n* First explore metadata files -- the json files\n* Then explore the video files -- samples of fake and real videos\n* Explore videos from the same original videos\n* Perform facial extraction and identifying different landmarks on the face -- nose, eyes, mouth."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport zipfile\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport seaborn as sns\nimport torch\nimport matplotlib.pyplot as plt\n# from tqdm import tqdm_notebook\n%matplotlib inline \n# from google.colab.patches import cv2_imshow\nfrom IPython.display import HTML #imports to play videos\nfrom base64 import b64encode \nimport cv2 as cv\nfrom skimage.measure import compare_ssim\nimport glob\nimport time\nfrom PIL import Image\nfrom facenet_pytorch import MTCNN, InceptionResnetV1, extract_face\nfrom tqdm import tqdm\n\n\nimport math\nimport pickle\nfrom functools import partial\nfrom collections import defaultdict\n\nfrom PIL import Image\nfrom glob import glob\n\nimport cv2\nimport skimage.measure\nimport albumentations as A\nfrom tqdm.notebook import tqdm \nfrom albumentations.pytorch import ToTensor \n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torchvision.models.video import mc3_18, r2plus1d_18\n\nfrom facenet_pytorch import MTCNN\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_FOLDER = \"../input/deepfake-detection-challenge\" #loading the data\nTRAIN_SAMPLE_FOLDER = \"train_sample_videos\"\nTEST_FOLDER = \"test_videos\"\n# print(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\n# print(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # adding face detection resources\nFACE_DETECTION_FOLDER = '../input/haarcascades/haar-cascades-for-face-detection'\nprint(f\"Face detection resources: {os.listdir(FACE_DETECTION_FOLDER)}\")    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Meta data exploration**\n* Missing data\n* Unique videos\n* Most frequent originals"},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking file type\ntrain_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\next_dict = []\nfor file in train_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#repeating the same for test sample folder\ntest_list = list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))\next_dict = []\nfor file in test_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking the json file\njson_file = [file for file in train_list if  file.endswith('json')][0]\nprint(f\"JSON file: {json_file}\")\n#reading the json file\ndef get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)\nmeta_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking for missing values\ndef missing_data(data):\n    total = data.isnull().sum()\n    percent = (data.isnull().sum()/data.isnull().count()*100)\n    tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    types = []\n    for col in data.columns:\n        dtype = str(data[col].dtype)\n        types.append(dtype)\n    tt['Types'] = types\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_data(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_data(meta_train_df.loc[meta_train_df.label == 'REAL'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unique_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Totals']\n    uniques = []\n    for col in data.columns:\n        unique = data[col].nunique() #collect all unique instances\n        uniques.append(unique)\n    tt['Uniques'] = uniques\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_values(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def most_frequent_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Total']\n    items = []\n    vals = []\n    for col in data.columns:\n        itm = data[col].value_counts().index[0]\n        val = data[col].value_counts().values[0]\n        items.append(itm)\n        vals.append(val)\n    tt['Most frequent item'] = items\n    tt['Frequence'] = vals\n    tt['Percent from total'] = np.round(vals / total * 100, 3)\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"most_frequent_values(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"most_frequent_values(meta_train_df.loc[meta_train_df.label == 'FAKE'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data distribution visualisations\ndef plot_count(feature, title, df, size=1):\n  '''\n    Plot count of classes / feature\n    param: feature - the feature to analyze\n    param: title - title to add to the graph\n    param: df - dataframe from which we plot feature's classes distribution \n    param: size - default 1.\n  '''  \n  f, ax = plt.subplots(1,1, figsize=(4*size,4))\n  total = float(len(df))\n  g =  sns.countplot(df[feature], order = df[feature].value_counts().index[:20], palette='Set3')\n  g.set_title(\"Number and percentage of {}\".format(title)) \n  if(size > 2):\n    plt.xticks(rotation=90, size=8)\n  for p in ax.patches:\n     height = p.get_height()\n     ax.text(p.get_x()+ p.get_width()/2.,height + 3,'{:1.2f}%'.format(100*height/total),ha=\"center\")\n\n  plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count('split','split(train)',meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count('label','label(train)',meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking if files in metadata is same as files in folder\nmeta = np.array(list(meta_train_df.index))\nstorage = np.array([file for file in train_list if  file.endswith('mp4')])\nprint(f\"Metadata: {meta.shape[0]}, Folder: {storage.shape[0]}\")\nprint(f\"Files in metadata and not in folder: {np.setdiff1d(meta,storage,assume_unique=False).shape[0]}\")\nprint(f\"Files in folder and not in metadata: {np.setdiff1d(storage,meta,assume_unique=False).shape[0]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='FAKE'].sample(3).index)\nfake_train_sample_video","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Video data exploration**\n* Displaying frames from fake and real videos\n* Displaying videos with same original\n* Viewing the test video files\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image_from_video(video_path):\n    '''\n    input: video_path - path for video\n    process:\n    1. perform a video capture from the video\n    2. read the image\n    3. display the image\n    '''\n    capture_img = cv.VideoCapture(video_path)\n    ret, frame = capture_img.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n    ax.imshow(frame)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for video_file in fake_train_sample_video:\n  display_image_from_video(os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER,video_file))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='REAL'].sample(3).index) #viewing the real videos\nreal_train_sample_video\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for video in real_train_sample_video:\n  display_image_from_video(os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER,video))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for video in real_train_sample_video:\n  display_image_from_video(os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER,video))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image_from_video_list(video_path_list, video_folder=TRAIN_SAMPLE_FOLDER):\n    '''\n    input: video_path_list - path for video\n    process:\n    0. for each video in the video path list\n        1. perform a video capture from the video\n        2. read the image\n        3. display the image\n    '''\n    plt.figure()\n    fig, ax = plt.subplots(2,3,figsize=(16,8))\n    #we only show images extracted from first 6 videos\n    for i, video_file in enumerate(video_path_list[0:6]):\n      video_path = os.path.join(DATA_FOLDER, video_folder, video_file)\n      capture_img = cv.VideoCapture(video_path)\n      ret, frame = capture_img.read()\n      frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n      ax[i//3, i%3].imshow(frame)\n      ax[i//3, i%3].set_title(f\"Video: {video_file}\")\n      ax[i//3, i%3].axis('on')\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='meawmsgiti.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video) #videos with same original","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real_videos_df = meta_train_df.loc[meta_train_df.label=='REAL']\nfake_videos_df = meta_train_df.loc[meta_train_df.label=='FAKE']\n# print(real_videos_df)\n#all real videos have no original\n# print(real_videos_df.loc[real_videos_df.split=='test'])\nprint(fake_videos_df.loc[fake_videos_df.split=='train'])\n# list_of_originals = fake_videos_df.loc[fake_videos_df.original].index\n# print(list_of_originals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(meta_train_df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='atvmxvwyns.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='qeumxirsme.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='kgbkktcjxf.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='dzyuwjkjui.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#looking at test videos \ntest_videos = pd.DataFrame(list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER))), columns=['video'])\ntest_videos.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndisplay_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[0].video))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image_from_video_list(test_videos.sample(6).video, TEST_FOLDER)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[0].video))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Face detection**\n* The class ObjectDetector uses CascadeClassifier to identify the following:\n* Frontal face: green rectangle\n* Eye: red circle\n* Smile: red rectangle\n* Profile face: blue rectangle\n* MTCNN is also used for facial extraction and recognition\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_frames(filename):\n  #Get all frames from video\n  #Input: filename - video filename\n  #Output: 50 frames in the video\n  num_frames = 50\n  frames = []\n  for i in range(0, num_frames):\n    vidcap = cv.VideoCapture(filename)\n    ret,frame = vidcap.read()\n    frames.append(frame)\n    return frames\n  print(len(frames))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_similarity_scores(frames):\n  #get similarity scores btwn frames\n  scores = []\n  for i in range (1, len(frames)):\n    frame = frames[i]\n    prev_frame = frames[i-1]\n\n    if frame.shape[0] != prev_frame.shape[0]:\n      if frame.shape[0] > prev_frame.shape[0]:\n        frame = frame[:prev_frame.shape[0], :prev_frame.shape[0], :]\n      else:\n        prev_frame = prev_frame[:frame>shape[0], :frame.shape[0], :]\n\n    (score, diff) = compare_ssim(frame, prev_frame, full=True, multichannel=True)\n    #compare_ssim is a similaririty func to compare two images\n    scores.append(score)\n    return scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#face detection\nclass ObjectDetector():\n  #Class for Object Detection\n  def __init__(self, object_cascade_path):\n    #  param: object_cascade_path - path for the *.xml defining the parameters for {face, eye, smile, profile}\n    #     detection algorithm\n    #     source of the haarcascade resource is: https://github.com/opencv/opencv/tree/master/data/haarcascades\n    self.objectCascade=cv.CascadeClassifier(object_cascade_path)\n  def detect(self, image, scale_factor=1.3, min_neighbours=5, min_size=(20,20)):\n    # Function return rectangle coordinates of object for given image\n    #     param: image - image to process\n    #     param: scale_factor - scale factor used for object detection\n    #     param: min_neighbors - minimum number of parameters considered during object detection\n    #     param: min_size - minimum size of bounding box for object detected\n    \n    rects=self.objectCascade.detectMultiScale(image, scaleFactor=scale_factor, minNeighbors=min_neighbours, minSize=min_size)\n    return rects\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load resources for frontal face, eye, smile and profile detection\nfrontal_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_frontalface_default.xml')\neye_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_eye.xml')\nprofile_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_profileface.xml')\nsmile_cascade_path = os.path.join(FACE_DETECTION_FOLDER,'haarcascade_smile.xml')\n\n#Detector object created\n# frontal face\nfd=ObjectDetector(frontal_cascade_path)\n# eye\ned=ObjectDetector(eye_cascade_path)\n# profile face\npd=ObjectDetector(profile_cascade_path)\n# smile\nsd=ObjectDetector(smile_cascade_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for each object use a diffrent color\ndef detect_objects(image, scale_factor, min_neighbours, min_size):\n  # Objects detection function\n  #   Identify frontal face, eyes, smile and profile face and display the detected objects over the image\n  #   param: image - the image extracted from the video\n  #   param: scale_factor - scale factor parameter for `detect` function of ObjectDetector object\n  #   param: min_neighbors - min neighbors parameter for `detect` function of ObjectDetector object\n  #   param: min_size - minimum size parameter for f`detect` function of ObjectDetector object\n  image_gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)\n  \n  #for eyes\n  eyes = ed.detect(image_gray, scale_factor=scale_factor,min_neighbours=min_neighbours,min_size=(int(min_size[0]/2), int(min_size[1]/2)))\n\n  for x, y, w, h in eyes:\n    cv.circle(image,(int(x+w/2), int(y+h/2)), (int((w+h)/4)),(0, 0, 255),3)\n\n\n  # deactivated due to many false positive\n  # smiles=sd.detect(image_gray,\n  #               scale_factor=scale_factor,\n  #               min_neighbours=min_neighbours,\n  #               min_size=(int(min_size[0]/2), int(min_size[1]/2)))\n\n  # for x, y, w, h in smiles:\n  #     #detected smiles shown in color image\n  #     cv.rectangle(image,(x,y),(x+w, y+h),(0, 255,255),3)\n  profiles=pd.detect(image_gray, scale_factor=scale_factor, min_neighbours=min_neighbours, min_size=min_size)\n\n  for x, y, w, h in profiles:\n      #detected profiles shown in color image\n      cv.rectangle(image,(x,y),(x+w, y+h),(255, 0,0),3)\n\n  \n  faces=fd.detect(image_gray,\n                   scale_factor=scale_factor,\n                  min_neighbours=min_neighbours,\n                   min_size=min_size)\n\n  for x, y, w, h in faces:\n      #detected faces shown in color image\n      cv.rectangle(image,(x,y),(x+w, y+h),(0, 255,0),3)\n\n  fig = plt.figure(figsize=(10,10))\n  ax = fig.add_subplot(111)\n  image = cv.cvtColor(image, cv.COLOR_BGR2RGB)\n  ax.imshow(image)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_image_objects(video_file, video_set_folder=TRAIN_SAMPLE_FOLDER):\n  # Extract one image from the video and then perform face/eyes/smile/profile detection on the image\n  #   param: video_file - the video from which to extract the image from which we extract the face\n  video_path = os.path.join(DATA_FOLDER,video_set_folder, video_file)\n  capture_image = cv.VideoCapture(video_path)\n  ret, frame = capture_image.read()\n  detect_objects(image=frame, \n            scale_factor=1.3, \n            min_neighbours=5, \n            min_size=(50, 50))  \n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nsame_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='kgbkktcjxf.mp4'].index)\nfor video_file in same_original_fake_train_sample_video[1:4]:\n    print(video_file)\n    extract_image_objects(video_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_subsample_video = list(meta_train_df.sample(3).index)\nfor video_file in train_subsample_video:\n    print(video_file)\n    extract_image_objects(video_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subsample_test_videos = list(test_videos.sample(3).video)\nfor video_file in subsample_test_videos:\n    print(video_file)\n    extract_image_objects(video_file, TEST_FOLDER)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#playing videos function\ndef play_video(video_file, subset=TRAIN_SAMPLE_FOLDER):\n  # Display video\n  #   param: video_file - the name of the video file to display\n  #   param: subset - the folder where the video file is located (can be TRAIN_SAMPLE_FOLDER or TEST_Folder)\n   video_url = open(os.path.join(DATA_FOLDER, subset,video_file),'rb').read()\n   data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n   return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fake_videos = list(meta_train_df.loc[meta_train_df.label=='FAKE'].index)\nreal_videos = list(meta_train_df.loc[meta_train_df.label=='REAL'].index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"play_video(fake_videos[150])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tryring smthing with facenet_pytorch\n#loading face detector\nmtcnn = MTCNN(margin=14, keep_all=True, factor=0.5, select_largest=False,device='cuda:0',post_process=False)\n#loading facial recognition model\nresnet = InceptionResnetV1(pretrained='vggface2').eval()\n#setting torch device to use\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nprint(f'Running on device: {device}')\n# help(MTCNN)\n# help(InceptionResnetV1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"v_cap = cv.VideoCapture('../input/deepfake-detection-challenge/train_sample_videos/avibnnhwhp.mp4')\nsuccess, frame = v_cap.read()\nframe = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\nframe = Image.fromarray(frame)\n\nplt.figure(figsize=(12, 8))\nplt.imshow(frame)\nplt.axis('off')\nplt.show()\n\n# Detect face\nfaces = mtcnn(frame)\nprint(faces.size())\n\n# plt.figure(figsize=(4,3))\n# plt.imshow(faces)\n# plt.axis('off')\n# plt.show()\n\n# Visualize\nfig, axes = plt.subplots(1, len(faces))\nfor face, ax in zip(faces, axes):\n    ax.imshow(face.permute(1, 2, 0).int().numpy())\n    ax.axis('off')\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Feature Enginiering and modeling**\n* Extracting frames efficiently while considering feature progression in the frames\n* Extracting features out of the frames -- features to identify fake videos characteristics since fake videos are more in the dataset\n* Create, train and validate the models\n* Make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nINPUT_DIR = \"../input/deepfake-detection-challenge/test_videos\"\nPRETRAINED_MODELS_3D = [{'type':'i3d',\n                         'path':\"../input/deepfake-detection-jph/j3d_e1_l0.1374.model\"},\n                        {'type':'res34',\n                         'path':\"../input/deepfake-detection-jph/res34_1cy_minaug_nonorm_e4_l0.1794.model\"},\n                        {'type':'mc3_112',\n                         'path':\"../input/deepfake-detection-jph/mc3_18_112_1cy_lilaug_nonorm_e9_l0.1905.model\"},\n                        {'type':'mc3_224',\n                         'path':\"../input/deepfake-detection-jph/mc3_18_112t224_1cy_lilaug_nonorm_e7_l0.1901.model\"},\n                        {'type':'r2p1_112',\n                         'path':\"../input/deepfake-detection-jph/r2p1_18_8_112tr_112te_e12_l0.1741.model\"},\n                        {'type':'i3d',\n                         'path':\"../input/deepfake-detection-jph/i3dcutmix_e11_l0.1612.model\"},\n                        {'type':'r2p1_112',\n                         'path':\"../input/deepfake-detection-jph/r2plus1dcutmix_112_e10_l0.1608.model\"}]\n\n# Face detection\nMAX_FRAMES_TO_LOAD = 100\nMIN_FRAMES_FOR_FACE = 30\nMAX_FRAMES_FOR_FACE = 100\nFACE_FRAMES = 10\nMAX_FACES_HIGHTHRESH = 5\nMAX_FACES_LOWTHRESH = 1\nFACEDETECTION_DOWNSAMPLE = 0.25\nMTCNN_THRESHOLDS = (0.8, 0.8, 0.9)  # Default [0.6, 0.7, 0.7]\nMTCNN_THRESHOLDS_RETRY = (0.5, 0.5, 0.5)\nMMTNN_FACTOR = 0.71  # Default 0.709 p\nTWO_FRAME_OVERLAP = False\n\n# Inference\nPROB_MIN, PROB_MAX = 0.001, 0.999\nREVERSE_PROBS = True\nDEFAULT_MISSING_PRED = 0.5\nUSE_FACE_FUNCTION = np.mean\n\n# 3D inference\nRATIO_3D = 1\nOUTPUT_FACE_SIZE = (256, 256)\nPRE_INFERENCE_CROP = (224, 224)\n\n# 2D\nRATIO_2D = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_video(filename, every_n_frames=10, specific_frames=None, to_rgb=True, rescale=None, inc_pil=False, max_frames=100):\n    \"\"\"Loads a video.\n    Called by:\n    \n    1) The finding faces algorithm where it pulls a frame every FACE_FRAMES frames up to MAX_FRAMES_TO_LOAD at a scale of FACEDETECTION_DOWNSAMPLE, and then half that if there's a CUDA memory error.\n    \n    2) The inference loop where it pulls EVERY frame up to a certain amount which it the last needed frame for each face for that video\"\"\"\n    \n    assert every_n_frames or specific_frames, \"Must supply either every n_frames or specific_frames\"\n    assert bool(every_n_frames) != bool(specific_frames), \"Supply either 'every_n_frames' or 'specific_frames', not both\"\n    \n    cap = cv2.VideoCapture(filename)\n    n_frames_in = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    width_in = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n    height_in = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n    \n    if rescale:\n        rescale = rescale * 1920./np.max((width_in, height_in))\n    \n    width_out = int(width_in*rescale) if rescale else width_in\n    height_out = int(height_in*rescale) if rescale else height_in\n    \n    if max_frames:\n        n_frames_in = min(n_frames_in, max_frames)\n    \n    if every_n_frames:\n        specific_frames = list(range(0,n_frames_in,every_n_frames))\n    \n    n_frames_out = len(specific_frames)\n    \n    out_pil = []\n\n    out_video = np.empty((n_frames_out, height_out, width_out, 3), np.dtype('uint8'))\n\n    i_frame_in = 0\n    i_frame_out = 0\n    ret = True\n\n    while (i_frame_in < n_frames_in and ret):\n        \n        try:\n            try:\n        \n                if every_n_frames == 1:\n                    ret, frame_in = cap.read()  # Faster if reading all frames\n                else:\n                    ret = cap.grab()\n\n                    if i_frame_in not in specific_frames:\n                        i_frame_in += 1\n                        continue\n\n                    ret, frame_in = cap.retrieve()\n                    \n#                 print(f\"Reading frame {i_frame_in}\")\n\n                if rescale:\n                    frame_in = cv2.resize(frame_in, (width_out, height_out))\n                if to_rgb:\n                    frame_in = cv2.cvtColor(frame_in, cv2.COLOR_BGR2RGB)\n                \n            except Exception as e:\n                print(f\"Error for frame {i_frame_in} for video {filename}: {e}; using 0s\")\n                frame_in = np.zeros((height_out, width_out, 3))\n\n        \n            out_video[i_frame_out] = frame_in\n            i_frame_out += 1\n\n            if inc_pil:\n                try:  # https://www.kaggle.com/zaharch/public-test-errors\n                    pil_img = Image.fromarray(frame_in)\n                except Exception as e:\n                    print(f\"Using a blank frame for video {filename} frame {i_frame_in} as error {e}\")\n                    pil_img = Image.fromarray(np.zeros((224,224,3), dtype=np.uint8))  # Use a blank frame\n                out_pil.append(pil_img)\n\n            i_frame_in += 1\n            \n        except Exception as e:\n            print(f\"Error for file {filename}: {e}\")\n\n    cap.release()\n    \n    if inc_pil:\n        return out_video, out_pil, rescale\n    else:\n        return out_video, rescale\n\ndef get_roi_for_each_face(faces_by_frame, probs, video_shape, temporal_upsample, upsample=1):\n    \"\"\"Function to find faces in frame through time and find the probability of a face appearing in a \n       frame. Returns an array of probabilities for faces and a boolean to show if a face is in a frame\n       or not.\n    \"\"\"\n    # Create boolean face array\n    frames_video, rows_video, cols_video, channels_video = video_shape\n    frames_video = math.ceil(frames_video)\n    boolean_face_3d = np.zeros((frames_video, rows_video, cols_video), dtype=np.bool)  # Remove colour channel\n    proba_face_3d = np.zeros((frames_video, rows_video, cols_video)).astype('float32')\n    for i_frame, faces in enumerate(faces_by_frame):\n        if faces is not None:  # May not be a face in the frame\n            for i_face, face in enumerate(faces):\n                left, top, right, bottom = face\n                boolean_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = True\n                proba_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = probs[i_frame][i_face]\n                \n    # Replace blank frames if face(s) in neighbouring frames with overlap\n    for i_frame, frame in enumerate(boolean_face_3d):\n        if i_frame == 0 or i_frame == frames_video-1:  # Can't do this for 1st or last frame\n            continue\n        if True not in frame:\n            if TWO_FRAME_OVERLAP:\n                if i_frame > 1:\n                    pre_overlap = boolean_face_3d[i_frame-1] | boolean_face_3d[i_frame-2]\n                else:\n                    pre_overlap = boolean_face_3d[i_frame-1]\n                if i_frame < frames_video-2:\n                    post_overlap = boolean_face_3d[i_frame+1] | boolean_face_3d[i_frame+2]\n                else:\n                    post_overlap = boolean_face_3d[i_frame+1]\n                neighbour_overlap = pre_overlap & post_overlap\n            else:\n                neighbour_overlap = boolean_face_3d[i_frame-1] & boolean_face_3d[i_frame+1]\n            boolean_face_3d[i_frame] = neighbour_overlap\n\n    # Find faces through time\n    id_face_3d, n_faces = skimage.measure.label(boolean_face_3d, return_num=True)\n    region_labels, counts = np.unique(id_face_3d, return_counts=True)\n    # Get rid of background=0\n    region_labels, counts = region_labels[1:], counts[1:]\n    ###################\n    # DESCENDING SIZE #\n    ###################\n    descending_size = np.argsort(counts)[::-1]\n    labels_by_size = region_labels[descending_size]\n    ####################\n    # DESCENDING PROBS #\n    ####################\n    probs = [np.mean(proba_face_3d[id_face_3d == i_face]) for i_face in region_labels]\n    descending_probs = np.argsort(probs)[::-1]\n    labels_by_probs = region_labels[descending_probs]\n    # Iterate over faces in video\n    rois = [] ; face_maps = []\n    for i_face in labels_by_probs:#labels_by_size:\n        # Find the first and last frame containing the face\n        frames = np.where(np.any(id_face_3d == i_face, axis=(1, 2)) == True)\n        starting_frame, ending_frame = frames[0].min(), frames[0].max()\n\n        # Iterate over the frames with faces in and find the min/max cols/rows (bounding box)\n        cols, rows = [], []\n        for i_frame in range(starting_frame, ending_frame + 1):\n            rs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=1) == True)\n            rows.append((rs[0].min(), rs[0].max()))\n            cs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=0) == True)\n            cols.append((cs[0].min(), cs[0].max()))\n        frame_from, frame_to = starting_frame*temporal_upsample, ((ending_frame+1)*temporal_upsample)-1\n        rows_from, rows_to = np.array(rows)[:, 0].min(), np.array(rows)[:, 1].max()\n        cols_from, cols_to = np.array(cols)[:, 0].min(), np.array(cols)[:, 1].max()\n        \n        frame_to = min(frame_to, frame_from + MAX_FRAMES_FOR_FACE)\n        \n        if frame_to - frame_from >= MIN_FRAMES_FOR_FACE:\n            tmp_face_map = id_face_3d.copy()\n            tmp_face_map[tmp_face_map != i_face] = 0\n            tmp_face_map[tmp_face_map == i_face] = 1\n            face_maps.append(tmp_face_map[frame_from//temporal_upsample:frame_to//temporal_upsample+1])\n            rois.append(((frame_from, frame_to),\n                         (int(rows_from*upsample), int(rows_to*upsample)),\n                         (int(cols_from*upsample), int(cols_to*upsample))))\n            \n    return np.array(rois), face_maps\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_center(bbox):\n    x1, y1, x2, y2 = bbox\n    return (x1+x2)/2, (y1+y2)/2\n\n\ndef get_coords(faces_roi):\n    coords = np.argwhere(faces_roi == 1)\n    #print(coords)\n    if coords.shape[0] == 0:\n        return None\n    y1, x1 = coords[0]\n    y2, x2 = coords[-1]\n    return x1, y1, x2, y2\n\n\ndef interpolate_center(c1, c2, length):\n    x1, y1 = c1\n    x2, y2 = c2\n    xi, yi = np.linspace(x1, x2, length), np.linspace(y1, y2, length)\n    return np.vstack([xi, yi]).transpose(1,0)\n\n\ndef get_faces(faces_roi, upsample): \n    all_faces = []\n    rows = faces_roi[0].shape[1]\n    cols = faces_roi[0].shape[2]\n    for i in range(len(faces_roi)):\n        faces = np.asarray([get_coords(faces_roi[i][j]) for j in range(len(faces_roi[i]))])\n        if faces[0] is None:  faces[0] = faces[1]\n        if faces[-1] is None: faces[-1] = faces[-2]\n        if None in faces:\n            #print(faces)\n            raise Exception('This should not have happened ...')\n        all_faces.append(faces)\n\n    extracted_faces = []\n    for face in all_faces:\n        # Get max dim size\n        max_dim = np.concatenate([face[:,2]-face[:,0],face[:,3]-face[:,1]])\n        max_dim = np.percentile(max_dim, 90)\n        # Enlarge by 1.2\n        max_dim = int(max_dim * 1.2)\n        # Get center coords\n        centers = np.asarray([get_center(_) for _ in face])\n        # Interpolate\n        centers = np.vstack([interpolate_center(centers[i], centers[i+1], length=10) for i in range(len(centers)-1)]).astype('int')\n        x1y1 = centers - max_dim // 2\n        x2y2 = centers + max_dim // 2 \n        x1, y1 = x1y1[:,0], x1y1[:,1]\n        x2, y2 = x2y2[:,0], x2y2[:,1]\n        # If x1 or y1 is negative, turn it to 0\n        # Then add to x2 y2 or y2\n        x2[x1 < 0] -= x1[x1 < 0]\n        y2[y1 < 0] -= y1[y1 < 0]\n        x1[x1 < 0] = 0\n        y1[y1 < 0] = 0\n        # If x2 or y2 is too big, turn it to max image shape\n        # Then subtract from y1\n        y1[y2 > rows] += rows - y2[y2 > rows]\n        x1[x2 > cols] += cols - x2[x2 > cols]\n        y2[y2 > rows] = rows\n        x2[x2 > cols] = cols\n        vidface = np.asarray([[x1[_],y1[_],x2[_],y2[_]] for _,c in enumerate(centers)])\n        vidface = (vidface*upsample).astype('int')\n        extracted_faces.append(vidface)\n\n    return extracted_faces\n\ndef detect_face_with_mtcnn(mtcnn_model, pil_frames, facedetection_upsample, video_shape, face_frames):\n    \"\"\"Function that uses MTCNN and get_roi_for_each_face for facial detection and extraction.\"\"\"\n    boxes, _probs = mtcnn_model.detect(pil_frames, landmarks=False)\n    faces, faces_roi = get_roi_for_each_face(faces_by_frame=boxes, probs=_probs, video_shape=video_shape, temporal_upsample=face_frames, upsample=facedetection_upsample)\n    coords = [] if len(faces_roi) == 0 else get_faces(faces_roi, upsample=facedetection_upsample)\n    return faces, coords\n\ndef face_detection_wrapper(mtcnn_model, videopath, every_n_frames, facedetection_downsample, max_frames_to_load):\n    \"\"\"Function that loads frames for facial recognition using the above function. The frames can be upsamples/downsamples\n       to get the most out of the frames.\"\"\"\n    video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample, inc_pil=True, max_frames=max_frames_to_load)\n    if len(pil_frames):\n        try:\n            faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn, \n                                                   pil_frames=pil_frames, \n                                                   facedetection_upsample=1/rescale, \n                                                   video_shape=video.shape, \n                                                   face_frames=every_n_frames)\n        except RuntimeError:  # Out of CUDA RAM\n            print(f\"Failed to process {videopath} ! Downsampling x2 ...\")\n            video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample/2, inc_pil=True, max_frames=max_frames_to_load)\n\n            try:\n                faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn, \n                                       pil_frames=pil_frames, \n                                       facedetection_upsample=1/rescale, \n                                       video_shape=video.shape, \n                                       face_frames=every_n_frames)\n            except RuntimeError:\n                print(f\"Failed on downsample ! Skipping...\")\n                return [], []\n                \n    else:\n        print('Failed to fetch frames ! Skipping ...')\n        return [], []\n        \n    if len(faces) == 0:\n        print('Failed to find faces ! Upsampling x2 ...')\n        try:\n            video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample*2, inc_pil=True, max_frames=max_frames_to_load)\n            faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn, \n                                                   pil_frames=pil_frames, \n                                                   facedetection_upsample=1/rescale, \n                                                   video_shape=video.shape, \n                                                   face_frames=every_n_frames)\n        except Exception as e:\n            print(e)\n            return [], []\n    \n    return faces, coords\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"videopaths = sorted(glob(os.path.join(INPUT_DIR, \"*.mp4\")))\nprint(f'Found {len(videopaths)} videos !')\n\nmtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False, device='cuda:0', thresholds=MTCNN_THRESHOLDS, factor=MMTNN_FACTOR)\n\nfaces_by_videopath = {}\ncoords_by_videopath = {}\n\nfor i_video, videopath in enumerate(tqdm(videopaths)):\n    faces, coords = face_detection_wrapper(mtcnn, videopath, every_n_frames=FACE_FRAMES, facedetection_downsample=FACEDETECTION_DOWNSAMPLE, max_frames_to_load=MAX_FRAMES_TO_LOAD)\n            \n    if len(faces):\n        faces_by_videopath[videopath]  = faces[:MAX_FACES_HIGHTHRESH]\n        coords_by_videopath[videopath] = coords[:MAX_FACES_HIGHTHRESH]\n    else:\n        print(f\"Found no faces for {videopath} !\")\n\n        \ndel mtcnn\nimport gc\ngc.collect()\n\nvideopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}\nprint(f\"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False ,device='cuda:0', thresholds=MTCNN_THRESHOLDS_RETRY, factor=MMTNN_FACTOR)\n\nfor i_video, videopath in enumerate(tqdm(videopaths_missing_faces)):\n    faces, coords = face_detection_wrapper(mtcnn, \n                                           videopath, \n                                           every_n_frames=FACE_FRAMES, \n                                           facedetection_downsample=FACEDETECTION_DOWNSAMPLE, \n                                           max_frames_to_load=MAX_FRAMES_TO_LOAD)\n            \n    if len(faces):\n        faces_by_videopath[videopath]  = faces[:MAX_FACES_LOWTHRESH]\n        coords_by_videopath[videopath] = coords[:MAX_FACES_LOWTHRESH]\n    else:\n        print(f\"Found no faces for {videopath} !\")\n\ndel mtcnn\nimport gc\ngc.collect()\n\nfaces_by_videopath = dict(sorted(faces_by_videopath.items(), key=lambda x: x[0]))\nvideopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}\nprint(f\"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"faces_by_videopath = dict(sorted(faces_by_videopath.items(), key=lambda x: x[0]))\nvideopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}\nprint(f\"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **3D CNN Models**\n* Use an ensemble of models that will perform best"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MaxPool3dSamePadding(nn.MaxPool3d):\n\n    def compute_pad(self, dim, s):\n        if s % self.stride[dim] == 0:\n            return max(self.kernel_size[dim] - self.stride[dim], 0)\n        else:\n            return max(self.kernel_size[dim] - (s % self.stride[dim]), 0)\n\n    def forward(self, x):\n        # compute 'same' padding\n        (batch, channel, t, h, w) = x.size()\n        # print t,h,w\n        out_t = np.ceil(float(t) / float(self.stride[0]))\n        out_h = np.ceil(float(h) / float(self.stride[1]))\n        out_w = np.ceil(float(w) / float(self.stride[2]))\n        # print out_t, out_h, out_w\n        pad_t = self.compute_pad(0, t)\n        pad_h = self.compute_pad(1, h)\n        pad_w = self.compute_pad(2, w)\n        # print pad_t, pad_h, pad_w\n\n        pad_t_f = pad_t // 2\n        pad_t_b = pad_t - pad_t_f\n        pad_h_f = pad_h // 2\n        pad_h_b = pad_h - pad_h_f\n        pad_w_f = pad_w // 2\n        pad_w_b = pad_w - pad_w_f\n\n        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)\n        # print x.size()\n        # print pad\n        x = F.pad(x, pad)\n        return super(MaxPool3dSamePadding, self).forward(x)\n\n\nclass Unit3D(nn.Module):\n\n    def __init__(self, in_channels,\n                 output_channels,\n                 kernel_shape=(1, 1, 1),\n                 stride=(1, 1, 1),\n                 padding=0,\n                 activation_fn=F.relu,\n                 use_batch_norm=True,\n                 use_bias=False,\n                 name='unit_3d'):\n\n        \"\"\"Initializes Unit3D module.\"\"\"\n        super(Unit3D, self).__init__()\n\n        self._output_channels = output_channels\n        self._kernel_shape = kernel_shape\n        self._stride = stride\n        self._use_batch_norm = use_batch_norm\n        self._activation_fn = activation_fn\n        self._use_bias = use_bias\n        self.name = name\n        self.padding = padding\n\n        self.conv3d = nn.Conv3d(in_channels=in_channels,\n                                out_channels=self._output_channels,\n                                kernel_size=self._kernel_shape,\n                                stride=self._stride,\n                                padding=0,\n                                # we always want padding to be 0 here. We will dynamically pad based on input size in forward function\n                                bias=self._use_bias)\n\n        if self._use_batch_norm:\n            self.bn = nn.BatchNorm3d(self._output_channels, eps=0.001, momentum=0.01)\n\n    def compute_pad(self, dim, s):\n        if s % self._stride[dim] == 0:\n            return max(self._kernel_shape[dim] - self._stride[dim], 0)\n        else:\n            return max(self._kernel_shape[dim] - (s % self._stride[dim]), 0)\n\n    def forward(self, x):\n        # compute 'same' padding\n        (batch, channel, t, h, w) = x.size()\n        # print t,h,w\n        out_t = np.ceil(float(t) / float(self._stride[0]))\n        out_h = np.ceil(float(h) / float(self._stride[1]))\n        out_w = np.ceil(float(w) / float(self._stride[2]))\n        # print out_t, out_h, out_w\n        pad_t = self.compute_pad(0, t)\n        pad_h = self.compute_pad(1, h)\n        pad_w = self.compute_pad(2, w)\n        # print pad_t, pad_h, pad_w\n\n        pad_t_f = pad_t // 2\n        pad_t_b = pad_t - pad_t_f\n        pad_h_f = pad_h // 2\n        pad_h_b = pad_h - pad_h_f\n        pad_w_f = pad_w // 2\n        pad_w_b = pad_w - pad_w_f\n\n        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)\n        # print x.size()\n        # print pad\n        x = F.pad(x, pad)\n        # print x.size()\n\n        x = self.conv3d(x)\n        if self._use_batch_norm:\n            x = self.bn(x)\n        if self._activation_fn is not None:\n            x = self._activation_fn(x)\n        return x\n\n\nclass InceptionModule(nn.Module):\n    def __init__(self, in_channels, out_channels, name):\n        super(InceptionModule, self).__init__()\n\n        self.b0 = Unit3D(in_channels=in_channels, output_channels=out_channels[0], kernel_shape=[1, 1, 1], padding=0,\n                         name=name + '/Branch_0/Conv3d_0a_1x1')\n        self.b1a = Unit3D(in_channels=in_channels, output_channels=out_channels[1], kernel_shape=[1, 1, 1], padding=0,\n                          name=name + '/Branch_1/Conv3d_0a_1x1')\n        self.b1b = Unit3D(in_channels=out_channels[1], output_channels=out_channels[2], kernel_shape=[3, 3, 3],\n                          name=name + '/Branch_1/Conv3d_0b_3x3')\n        self.b2a = Unit3D(in_channels=in_channels, output_channels=out_channels[3], kernel_shape=[1, 1, 1], padding=0,\n                          name=name + '/Branch_2/Conv3d_0a_1x1')\n        self.b2b = Unit3D(in_channels=out_channels[3], output_channels=out_channels[4], kernel_shape=[3, 3, 3],\n                          name=name + '/Branch_2/Conv3d_0b_3x3')\n        self.b3a = MaxPool3dSamePadding(kernel_size=[3, 3, 3],\n                                        stride=(1, 1, 1), padding=0)\n        self.b3b = Unit3D(in_channels=in_channels, output_channels=out_channels[5], kernel_shape=[1, 1, 1], padding=0,\n                          name=name + '/Branch_3/Conv3d_0b_1x1')\n        self.name = name\n\n    def forward(self, x):\n        b0 = self.b0(x)\n        b1 = self.b1b(self.b1a(x))\n        b2 = self.b2b(self.b2a(x))\n        b3 = self.b3b(self.b3a(x))\n        return torch.cat([b0, b1, b2, b3], dim=1)\n\n\nclass InceptionI3d(nn.Module):\n    \"\"\"Inception-v1 I3D architecture.\n    The model is introduced in:\n        Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset\n        Joao Carreira, Andrew Zisserman\n        https://arxiv.org/pdf/1705.07750v1.pdf.\n    See also the Inception architecture, introduced in:\n        Going deeper with convolutions\n        Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,\n        Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.\n        http://arxiv.org/pdf/1409.4842v1.pdf.\n    \"\"\"\n\n    # Endpoints of the model in order. During construction, all the endpoints up\n    # to a designated `final_endpoint` are returned in a dictionary as the\n    # second return value.\n    VALID_ENDPOINTS = (\n        'Conv3d_1a_7x7',\n        'MaxPool3d_2a_3x3',\n        'Conv3d_2b_1x1',\n        'Conv3d_2c_3x3',\n        'MaxPool3d_3a_3x3',\n        'Mixed_3b',\n        'Mixed_3c',\n        'MaxPool3d_4a_3x3',\n        'Mixed_4b',\n        'Mixed_4c',\n        'Mixed_4d',\n        'Mixed_4e',\n        'Mixed_4f',\n        'MaxPool3d_5a_2x2',\n        'Mixed_5b',\n        'Mixed_5c',\n        'Logits',\n        'Predictions',\n    )\n\n    def __init__(self, num_classes=400, spatial_squeeze=True, output_method='per_frame',\n                 final_endpoint='Logits', name='inception_i3d', in_channels=3, dropout_keep_prob=0.5):\n        \"\"\"Initializes I3D model instance.\n        Args:\n          num_classes: The number of outputs in the logit layer (default 400, which\n              matches the Kinetics dataset).\n          spatial_squeeze: Whether to squeeze the spatial dimensions for the logits\n              before returning (default True).\n          final_endpoint: The model contains many possible endpoints.\n              `final_endpoint` specifies the last endpoint for the model to be built\n              up to. In addition to the output at `final_endpoint`, all the outputs\n              at endpoints up to `final_endpoint` will also be returned, in a\n              dictionary. `final_endpoint` must be one of\n              InceptionI3d.VALID_ENDPOINTS (default 'Logits').\n          name: A string (optional). The name of this module.\n        Raises:\n          ValueError: if `final_endpoint` is not recognized.\n        \"\"\"\n\n        if final_endpoint not in self.VALID_ENDPOINTS:\n            raise ValueError('Unknown final endpoint %s' % final_endpoint)\n\n        super(InceptionI3d, self).__init__()\n        self.output_method = output_method\n        assert output_method in ('per_frame', 'avg_pool', 'max_pool', 'dual_pool')\n        self._num_classes = num_classes\n        self._spatial_squeeze = spatial_squeeze\n        self._final_endpoint = final_endpoint\n        self.logits = None\n\n        if self._final_endpoint not in self.VALID_ENDPOINTS:\n            raise ValueError('Unknown final endpoint %s' % self._final_endpoint)\n\n        self.end_points = {}\n        end_point = 'Conv3d_1a_7x7'\n        self.end_points[end_point] = Unit3D(in_channels=in_channels, output_channels=64, kernel_shape=[7, 7, 7],\n                                            stride=(2, 2, 2), padding=(3, 3, 3), name=name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'MaxPool3d_2a_3x3'\n        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),\n                                                          padding=0)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Conv3d_2b_1x1'\n        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=64, kernel_shape=[1, 1, 1], padding=0,\n                                            name=name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Conv3d_2c_3x3'\n        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=192, kernel_shape=[3, 3, 3], padding=1,\n                                            name=name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'MaxPool3d_3a_3x3'\n        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),\n                                                          padding=0)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_3b'\n        self.end_points[end_point] = InceptionModule(192, [64, 96, 128, 16, 32, 32], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_3c'\n        self.end_points[end_point] = InceptionModule(256, [128, 128, 192, 32, 96, 64], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'MaxPool3d_4a_3x3'\n        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[3, 3, 3], stride=(2, 2, 2),\n                                                          padding=0)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_4b'\n        self.end_points[end_point] = InceptionModule(128 + 192 + 96 + 64, [192, 96, 208, 16, 48, 64], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_4c'\n        self.end_points[end_point] = InceptionModule(192 + 208 + 48 + 64, [160, 112, 224, 24, 64, 64], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_4d'\n        self.end_points[end_point] = InceptionModule(160 + 224 + 64 + 64, [128, 128, 256, 24, 64, 64], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_4e'\n        self.end_points[end_point] = InceptionModule(128 + 256 + 64 + 64, [112, 144, 288, 32, 64, 64], name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_4f'\n        self.end_points[end_point] = InceptionModule(112 + 288 + 64 + 64, [256, 160, 320, 32, 128, 128],\n                                                     name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'MaxPool3d_5a_2x2'\n        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[2, 2, 2], stride=(2, 2, 2),\n                                                          padding=0)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_5b'\n        self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [256, 160, 320, 32, 128, 128],\n                                                     name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Mixed_5c'\n        self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [384, 192, 384, 48, 128, 128],\n                                                     name + end_point)\n        if self._final_endpoint == end_point: return\n\n        end_point = 'Logits'\n        self.avg_pool = nn.AvgPool3d(kernel_size=[2, 7, 7],\n                                     stride=(1, 1, 1))\n        self.dropout = nn.Dropout(dropout_keep_prob)\n        self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,\n                             kernel_shape=[1, 1, 1],\n                             padding=0,\n                             activation_fn=None,\n                             use_batch_norm=False,\n                             use_bias=True,\n                             name='logits')\n\n        self.build()\n\n    def replace_logits(self, num_classes):\n        self._num_classes = num_classes\n        self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,\n                             kernel_shape=[1, 1, 1],\n                             padding=0,\n                             activation_fn=None,\n                             use_batch_norm=False,\n                             use_bias=True,\n                             name='logits')\n\n    def build(self):\n        for k in self.end_points.keys():\n            self.add_module(k, self.end_points[k])\n\n    def forward(self, x):\n        for end_point in self.VALID_ENDPOINTS:\n            if end_point in self.end_points:\n                x = self._modules[end_point](x)  # use _modules to work with dataparallel\n\n        x = self.logits(self.dropout(self.avg_pool(x)))\n        if self._spatial_squeeze:\n            logits = x.squeeze(3).squeeze(3)\n        # logits is batch X time X classes, which is what we want to work with\n        if self.output_method == 'per_frame':\n            return F.interpolate(logits, 64, mode='linear')  # -> batch_size * 2 * 64\n        elif self.output_method == 'avg_pool':\n            avg_all_frames = F.adaptive_avg_pool1d(logits, output_size=1)\n            return avg_all_frames.squeeze(-1)  # -> batch_size * 2\n\n    def extract_features(self, x):\n        for end_point in self.VALID_ENDPOINTS:\n            if end_point in self.end_points:\n                x = self._modules[end_point](x)\n        return self.avg_pool(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv3x3x3(in_planes, out_planes, stride=1):\n    # 3x3x3 convolution with padding\n    return nn.Conv3d(\n        in_planes,\n        out_planes,\n        kernel_size=3,\n        stride=stride,\n        padding=1,\n        bias=False)\n\ndef downsample_basic_block(x, planes, stride):\n    out = F.avg_pool3d(x, kernel_size=1, stride=stride)\n    zero_pads = torch.Tensor(\n        out.size(0), planes - out.size(1), out.size(2), out.size(3),\n        out.size(4)).zero_()\n    if isinstance(out.data, torch.cuda.FloatTensor):\n        zero_pads = zero_pads.cuda()\n \n    out = Variable(torch.cat([out.data, zero_pads], dim=1))\n \n    return out\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n \n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super(BasicBlock, self).__init__()\n        self.conv1 = conv3x3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm3d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3x3(planes, planes)\n        self.bn2 = nn.BatchNorm3d(planes)\n        self.downsample = downsample\n        self.stride = stride\n \n    def forward(self, x):\n        residual = x\n \n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n \n        out = self.conv2(out)\n        out = self.bn2(out)\n \n        if self.downsample is not None:\n            residual = self.downsample(x)\n \n        out += residual\n        out = self.relu(out)\n \n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n \n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm3d(planes)\n        self.conv2 = nn.Conv3d(\n            planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm3d(planes)\n        self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm3d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n \n    def forward(self, x):\n        residual = x\n \n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n \n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n \n        out = self.conv3(out)\n        out = self.bn3(out)\n \n        if self.downsample is not None:\n            residual = self.downsample(x)\n \n        out += residual\n        out = self.relu(out)\n \n        return out\n\n\nclass ResNet(nn.Module):\n \n    def __init__(self,\n                 block,\n                 layers,\n                 sample_size,\n                 sample_duration,\n                 shortcut_type='B',\n                 num_classes=400):\n        self.inplanes = 64\n        super(ResNet, self).__init__()\n        self.conv1 = nn.Conv3d(\n            3,\n            64,\n            kernel_size=7,\n            stride=(1, 2, 2),\n            padding=(3, 3, 3),\n            bias=False)\n        self.bn1 = nn.BatchNorm3d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)\n        self.layer2 = self._make_layer(\n            block, 128, layers[1], shortcut_type, stride=2)\n        self.layer3 = self._make_layer(\n            block, 256, layers[2], shortcut_type, stride=2)\n        self.layer4 = self._make_layer(\n            block, 512, layers[3], shortcut_type, stride=2)\n        # last_duration = int(math.ceil(sample_duration / 16))\n        # last_size = int(math.ceil(sample_size / 32))\n        # self.avgpool = nn.AvgPool3d(\n        #     (last_duration, last_size, last_size), stride=1)\n        self.avgpool = nn.AdaptiveAvgPool3d(1)\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n \n        for m in self.modules():\n            if isinstance(m, nn.Conv3d):\n                m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out')\n            elif isinstance(m, nn.BatchNorm3d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n \n    def _make_layer(self, block, planes, blocks, shortcut_type, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            if shortcut_type == 'A':\n                downsample = partial(\n                    downsample_basic_block,\n                    planes=planes * block.expansion,\n                    stride=stride)\n            else:\n                downsample = nn.Sequential(\n                    nn.Conv3d(\n                        self.inplanes,\n                        planes * block.expansion,\n                        kernel_size=1,\n                        stride=stride,\n                        bias=False), nn.BatchNorm3d(planes * block.expansion))\n \n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n \n        return nn.Sequential(*layers)\n \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n \n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n \n        x = self.avgpool(x)\n \n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n \n        return x\n\ndef get_fine_tuning_parameters(model, ft_begin_index):\n    if ft_begin_index == 0:\n        return model.parameters()\n \n    ft_module_names = []\n    for i in range(ft_begin_index, 5):\n        ft_module_names.append('layer{}'.format(i))\n    ft_module_names.append('fc')\n \n    parameters = []\n    for k, v in model.named_parameters():\n        for ft_module in ft_module_names:\n            if ft_module in k:\n                parameters.append({'params': v})\n                break\n        else:\n            parameters.append({'params': v, 'lr': 0.0})\n \n    return parameters\n\n\ndef resnet18(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)\n    return model\n\ndef resnet34(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class HFlipWrapper(nn.Module):\n    def __init__(self, model, flip_dim=(-1,)):\n        super(HFlipWrapper, self).__init__()\n        self.model = model\n        self.flip_dim = flip_dim\n        \n    def forward(self, x):\n        with torch.no_grad():\n            xf = torch.flip(x, self.flip_dim)\n        pred = self.model(torch.stack([x, xf]))\n        return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models_3d = []\n\nfor modeldict in PRETRAINED_MODELS_3D:\n    if modeldict['type'] == 'i3d':\n        model = InceptionI3d(157, in_channels=3, output_method='avg_pool')\n        model.replace_logits(2)\n        model = model.cuda()\n        model.eval()\n        model.load_state_dict(torch.load(modeldict['path']))\n        models_3d.append({'norm':'i3d', 'model':model})\n        \n    elif modeldict['type'] == 'res18':\n        model = resnet18(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)  # , last_fc=True)\n        model.load_state_dict(torch.load(modeldict['path']))\n        model = model.cuda()\n        model.eval()\n        models_3d.append({'norm':'nil', 'model':model})\n        \n    elif modeldict['type'] == 'res34':\n        model = resnet34(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)  # , last_fc=True)\n        model.load_state_dict(torch.load(modeldict['path']))\n        model = model.cuda()\n        model.eval()\n        models_3d.append({'norm':'nil', 'model':model})\n        \n    elif modeldict['type'] == 'mc3_112':\n        model = mc3_18(num_classes=2)\n        model.load_state_dict(torch.load(modeldict['path']))\n        model = model.cuda()\n        model.eval()\n        models_3d.append({'norm':'112_imagenet', 'model':model})\n        \n    elif modeldict['type'] == 'mc3_224':\n        model = mc3_18(num_classes=2)\n        model.load_state_dict(torch.load(modeldict['path']))\n        model = model.cuda()\n        model.eval()\n        models_3d.append({'norm':'224_imagenet', 'model':model})\n        \n    elif modeldict['type'] == 'r2p1_112':\n        model = r2plus1d_18(num_classes=2)\n        model.load_state_dict(torch.load(modeldict['path']))\n        model = model.cuda()\n        model.eval()\n        models_3d.append({'norm':'112_imagenet', 'model':model})\n        \n        \n    else:\n        raise ValueError(f\"Unknown model type {modeldict['type']}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys ; sys.path.insert(0, '/kaggle/input/deepfake/deepfake/deepfake/skp/')\nimport yaml\n\nwith open('/kaggle/input/deepfake/deepfake/deepfake/skp/configs/experiments/experiment001.yaml') as f:\n    CFG = yaml.load(f, Loader=yaml.FullLoader)\n\nfrom factory.builder import build_model, build_dataloader\n\nCFG['model']['params']['pretrained'] = None\nmodel2d = build_model(CFG['model']['name'], CFG['model']['params'])\nmodel2d.load_state_dict(torch.load('/kaggle/input/deepfake/SRXT50_094_VM-0.2504.PTH'))\nmodel2d = model2d.eval().cuda()\nloader = build_dataloader(CFG, data_info={'vidfiles': [], 'labels': []}, mode='predict')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_and_square_face(video, output_size):\n    input_size = max(video.shape[1], video.shape[2])  # We will square it, so this is the effective input size\n    out_video = np.empty((len(video), output_size[0], output_size[1], 3), np.dtype('uint8'))\n    \n    for i_frame, frame in enumerate(video):\n        padded_image = np.zeros((input_size, input_size, 3))\n        padded_image[0:frame.shape[0], 0:frame.shape[1]] = frame\n        if (input_size, input_size) != output_size:\n            frame = cv2.resize(padded_image, (output_size[0], output_size[1])).astype(np.uint8)\n        else:\n            frame = padded_image.astype(np.uint8)\n        out_video[i_frame] = frame\n    return out_video","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def center_crop_video(video, crop_dimensions):\n    height, width = video.shape[1], video.shape[2]\n    crop_height, crop_width = crop_dimensions\n    \n    y1 = (height - crop_height) // 2\n    y2 = y1 + crop_height\n    x1 = (width - crop_width) // 2\n    x2 = x1 + crop_width\n        \n    video_out = np.zeros((len(video), crop_height, crop_width, 3))\n    for i_frame,frame in enumerate(video):\n        video_out[i_frame] = frame[y1:y2, x1:x2]\n        \n    return video_out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_last_frame_needed_across_faces(faces):\n    last_frame = 0\n    \n    for face in faces:\n        (frame_from, frame_to), (row_from, row_to), (col_from, col_to) = face\n        last_frame = max(frame_to, last_frame)\n        \n    return last_frame","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = defaultdict(list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transforms_114_imagenet = A.Compose([A.Resize(height=112, width=112),\n                                 A.Normalize()])\n\ntest_transforms_224_imagenet = A.Compose([A.Normalize()])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training the models**"},{"metadata":{"trusted":true},"cell_type":"code","source":"for videopath, faces in tqdm(faces_by_videopath.items(), total=len(faces_by_videopath)):\n    try:\n\n        if len(faces):\n            last_frame_needed = get_last_frame_needed_across_faces(faces)\n            video, rescale = load_video(videopath, every_n_frames=1, to_rgb=True, rescale=None, inc_pil=False, max_frames=last_frame_needed)\n\n        else:\n            print(f\"Skipping {videopath} as no faces found\")\n            continue\n            \n        for modeldict in models_3d:\n            preds_video = []\n            model = modeldict['model']\n            \n            model = HFlipWrapper(model=model)\n\n            for i_face, face in enumerate(faces):\n                (frame_from, frame_to), (row_from, row_to), (col_from, col_to) = face\n\n                x = video[frame_from:frame_to, row_from:row_to + 1, col_from:col_to + 1]\n                x = resize_and_square_face(x, output_size=OUTPUT_FACE_SIZE)\n\n                if PRE_INFERENCE_CROP and PRE_INFERENCE_CROP != OUTPUT_FACE_SIZE:\n                    x = center_crop_video(x, PRE_INFERENCE_CROP)\n\n                with torch.no_grad():\n                    \n                    if modeldict['norm'] == '112_imagenet':\n                        x = np.array([test_transforms_114_imagenet(image=frame)['image'] for frame in x])\n                    elif modeldict['norm'] == '224_imagenet':\n                        x = np.array([test_transforms_224_imagenet(image=frame)['image'] for frame in x])\n                        \n                    x = torch.from_numpy(x.transpose([3, 0, 1, 2])).float()\n                    \n                    if modeldict['norm'] == 'i3d':\n                        x = (x / 255.) * 2 - 1\n                    elif modeldict['norm'] == 'nil':\n                        pass\n                    elif modeldict['norm'] == '112_imagenet':\n                        pass\n                    elif modeldict['norm'] == '224_imagenet':\n                        pass\n                    else:\n                        raise ValueError(f\"Unknown normalisation mode {modeldict['norm']}\")\n\n                    y_pred = model(x.cuda())\n                    prob0, prob1 = torch.mean(torch.exp(F.log_softmax(y_pred, dim=1)),dim=0)\n                    if REVERSE_PROBS:\n                        prob1 = 1-prob1\n                    preds_video.append(float(prob1))\n                    \n            videoname = os.path.basename(videopath)\n            if preds_video:\n                predictions[videoname].extend([USE_FACE_FUNCTION(preds_video)] * RATIO_3D)\n            \n        try:\n            FRAMES2D = 32\n            # Ian's 2D model\n            coords = coords_by_videopath[videopath]\n            videoname = os.path.basename(videopath)\n            preds_video = []\n            for i_coord, coordinate in enumerate(coords):\n                (frame_from, frame_to), (row_from, row_to), (col_from, col_to) = faces[i_coord]\n                x = []\n                for coord_ind, frame_number in enumerate(range(frame_from, min(frame_from+FRAMES2D, frame_to-1))):\n                    if coord_ind >= len(coordinate):\n                        break\n                    x1, y1, x2, y2 = coordinate[coord_ind]\n                    x.append(video[frame_number, y1:y2, x1:x2])\n                x = np.asarray(x)\n                # Reverse back to BGR because it will get reversed to RGB when preprocessed\n                #x = x[...,::-1]\n                # Preprocess\n                x = loader.dataset.process_video(x)\n                #x = np.asarray([loader.dataset.process_image(_) for _ in x])\n                # Flip every other frame\n                x[:,::2] = x[:,::2,:,::-1]\n                # RGB reverse every 3rd frame\n                #x[:,::3] = x[::-1,::3]\n                with torch.no_grad():\n                    out = model2d(torch.from_numpy(np.ascontiguousarray(x)).unsqueeze(0).cuda())\n                #out = np.median(out.cpu().numpy())\n                preds_video.append(out.cpu().numpy())\n            if len(preds_video) > 0:\n                predictions[videoname].extend([USE_FACE_FUNCTION(preds_video)] * RATIO_2D)\n        except:\n            pass\n            \n    except Exception as e:\n        print(f\"ERROR: Video {videopath}: {e}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nx_show = x.transpose(1,2,3,0)\nx_show -= np.min(x_show)\nx_show /= np.max(x_show)\ndef view(x, nrows=4, ncols=4):\n    indices = np.linspace(0, len(x)-1, nrows*ncols) \n    for ind, i in enumerate(indices):\n        plt.subplot(nrows,ncols,ind+1)\n        plt.imshow(x[int(i),...,::-1])\n    plt.show()\n\nview(x_show, 5, 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Making predictions**\n* The first row is the mean of all the predictions from each of the models\n* The preceeding rows are predictions from each of the models."},{"metadata":{"trusted":true},"cell_type":"code","source":"for k,v in predictions.items():\n    string = '{} : {:.4f} //'.format(k, np.mean(v))\n    for proba in v:\n        string += ' {:.4f}'.format(proba)\n    print(string)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\nfor modeldict in models_3d:\n    del modeldict['model']\n    del modeldict\ndel models_3d\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Filling in missing predictions with 0.5**"},{"metadata":{"trusted":true},"cell_type":"code","source":"for videopath in videopaths:\n    videoname = os.path.basename(videopath)\n    if videoname not in predictions:\n        predictions[videoname] = [DEFAULT_MISSING_PRED]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Calculate ensembled prediction and clamp**"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_ensembled = {}\n\nfor videopath, pred in predictions.items():\n    #print(f\"{videopath} Got {len(pred)} predictions\")\n    preds_ensembled[videopath] = np.clip(np.mean(pred), PROB_MIN, PROB_MAX)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Save predictions**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"{len(preds_ensembled)} predictions\")\nwith open(\"submission.csv\", 'w') as f:\n    f.write(\"filename,label\\n\")\n    for filename, label in preds_ensembled.items():\n        f.write(f\"{filename},{label}\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head -10 submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}