{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Notebook Set-up"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import packages\n!pip install /kaggle/input/mtcnnpackage/mtcnn-0.1.0-py3-none-any.whl\n! tar xvf ../input/ffmpegpackage/ffmpeg-git-amd64-static.tar.xz\n#!pip install /kaggle/input/ffmpegpackage/ffmpeg_python-0.2.0-py3-none-any.whl \nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport os\nfrom tqdm import tqdm\nfrom mtcnn.mtcnn import MTCNN\n#import ffmpeg\nimport subprocess\nimport librosa\nfrom pathlib import Path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for dirname, _, filenames in os.walk('/kaggle/working/ffmpeg-git-20200305-amd64-static'):\n#    for filename in filenames:\n#        file.remove(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/deepfake-detection-challenge/'\n#os.listdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## load video metadata ##\nvideo_metadata = pd.read_json(path+'train_sample_videos/metadata.json')\nvideo_metadata = video_metadata.T","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Processing Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"## functions for getting facial features ##\n\ndef calculate_features(result, timestamp):\n  confidence = result[0]['confidence']\n  box_size = result[0]['box'][2]*result[0]['box'][3]\n  eye_width = result[0]['keypoints']['right_eye'][0] - result[0]['keypoints']['left_eye'][0] \n  eye_height = result[0]['keypoints']['right_eye'][1] - result[0]['keypoints']['left_eye'][1]\n  features = pd.DataFrame(data = {'confidence': confidence, 'box_size': box_size, 'eye_width': eye_width, 'eye_height': eye_height}, index = [timestamp])\n  return features\n\ndef get_face_detection_features(cap, detector):  \n  face_detection_features = pd.DataFrame()\n  frame_no=0\n  while (cap.isOpened()):\n    frame_no = frame_no+1\n    if frame_no <= 300:\n      if frame_no%30 == 0:\n        ret, frame = cap.read()\n        if ret == True:\n            timestamp = cap.get(cv2.CAP_PROP_POS_MSEC)\n            result = detector.detect_faces(frame)\n            if len(result) == 1:\n              features = calculate_features(result, timestamp)\n              face_detection_features = face_detection_features.append(features)\n        else:\n            break  \n    else:\n      break\n  return face_detection_features\n  \ndef get_face_features(cap, file_name, detector):\n  face_detection_features = get_face_detection_features(cap, detector)\n  if len(face_detection_features) > 0:\n    agg = face_detection_features.aggregate({\"confidence\":['max', 'min', 'mean', 'std'], \n                                             \"box_size\":['max', 'min', 'mean', 'std'], \n                                             \"eye_width\":['max', 'min', 'mean', 'std'],  \n                                             \"eye_height\":['max', 'min', 'mean', 'std']})  \n    face_features = pd.DataFrame(data = {'confidence_ave': agg['confidence'][2], \n                                         'confidence_std': agg['confidence'][3], \n                                         'box_size_std': agg['box_size'][3], \n                                         'eye_width_std': agg['eye_width'][3], \n                                         'eye_height_std': agg['eye_height'][3]}, \n                                 index = [file_name])\n    return face_features\n  else:\n    return []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## functions for getting audio features ##\n\noutput_format = 'wav'\noutput_dir = Path(f\"{output_format}s\")\nPath(output_dir).mkdir(exist_ok=True, parents=True)\ndef create_wav(file, output_dir, output_format):\n    command = f\"../working/ffmpeg-git-20200305-amd64-static/ffmpeg -y -i {file} -ab 192000 -ac 2 -ar 44100 -vn {output_dir/'audio_file'}.{output_format}\" # -y overwries file\n    subprocess.call(command, shell=True)\n\ndef get_audio_features(audio, sampling_rate, file_name):\n  #mfccs = np.mean(librosa.feature.mfcc(y=audio, sr=sampling_rate, n_mfcc=20).T,axis=0)\n  #chromagrams = np.mean(librosa.feature.chroma_stft(y=audio, sr=sampling_rate).T,axis=0)\n  rmse = np.mean(librosa.feature.rms(y=audio)) \n  chroma_stft = np.mean(librosa.feature.chroma_stft(y=audio, sr=sampling_rate)) \n  spec_cent = np.mean(librosa.feature.spectral_centroid(y=audio, sr=sampling_rate)) \n  spec_bw = np.mean(librosa.feature.spectral_bandwidth(y=audio, sr=sampling_rate)) \n  rolloff = np.mean(librosa.feature.spectral_rolloff(y=audio, sr=sampling_rate)) \n  zcr = np.mean(librosa.feature.zero_crossing_rate(audio)) \n  mfccs = np.mean(librosa.feature.mfcc(y=audio, sr=sampling_rate, n_mfcc = 20).T,axis=0) \n  audio_output = np.hstack((rmse, chroma_stft, spec_cent, spec_bw, rolloff, zcr, mfccs))\n  if len(audio_output)==26:\n    audio_features = pd.DataFrame([list(audio_output)], \n                                  columns = ['rmse', 'chroma_stft', 'spec_cent', 'spec_bw', 'rolloff', 'zcr',\n                                            'mfccs_1', 'mfccs_2', 'mfccs_3', 'mfccs_4', 'mfccs_5', 'mfccs_6', 'mfccs_7', 'mfccs_8', 'mfccs_9', 'mfccs_10',\n                                            'mfccs_11', 'mfccs_12', 'mfccs_13', 'mfccs_14', 'mfccs_15', 'mfccs_16', 'mfccs_17', 'mfccs_18', 'mfccs_19', 'mfccs_20'],\n                                  index=[file_name])\n  else:\n    audio_features = []    \n  return audio_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## functions for getting file features ##\n\ndef get_file_features(cap, file_name):\n  file_features = pd.DataFrame(data = {\n    'width': cap.get(cv2.CAP_PROP_FRAME_WIDTH),\n    'height': cap.get(cv2.CAP_PROP_FRAME_HEIGHT),\n    'fps': cap.get(cv2.CAP_PROP_FPS),\n    'no_frames': cap.get(cv2.CAP_PROP_FRAME_COUNT),\n    'brightness': cap.get(cv2.CAP_PROP_BRIGHTNESS)\n  }, index=[file_name])\n  return file_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## function pull together all features for capture ##\n\ndef get_all_features(path, file_name, detector):\n    create_wav(path+file_name, output_dir, output_format)\n    #audio, sampling_rate = librosa.load('/kaggle/working/wavs/'+file_name.replace('.mp4', '.wav'))\n    audio, sampling_rate = librosa.load('/kaggle/working/wavs/audio_file.wav')\n    capture = cv2.VideoCapture(path+file_name)\n    if capture.isOpened():\n      file_features = get_file_features(capture, file_name)\n      face_features = get_face_features(capture, file_name, detector)\n      audio_features = get_audio_features(audio, sampling_rate, file_name)\n      all_features = file_features.join(video_metadata)\n      if len(face_features) > 0:\n        all_features = all_features.join(face_features)\n      if len(audio_features) > 0:\n        all_features = all_features.join(audio_features)\n    else:\n      print('ERROR: '+file_name+' would not open')\n    return all_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate Test Data Frame"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_feature_names = ['rmse','chroma_stft','spec_cent','spec_bw','rolloff','zcr','mfccs_1','mfccs_2','mfccs_3','mfccs_4','mfccs_5','mfccs_6','mfccs_7','mfccs_8','mfccs_9','mfccs_1','mfccs_11','mfccs_12','mfccs_13','mfccs_14','mfccs_15','mfccs_16','mfccs_17','mfccs_18','mfccs_19','mfccs_20', 'box_size_std', 'confidence_ave','confidence_std','eye_height_std','eye_width_std']\nfeature_names = ['chroma_stft', 'mfccs_4', 'mfccs_5', 'mfccs_6', 'mfccs_7', 'mfccs_8', 'mfccs_9', 'mfccs_10', 'mfccs_11', 'mfccs_13', 'mfccs_14', 'mfccs_15', 'mfccs_16', 'mfccs_17', 'mfccs_18', 'mfccs_19', 'mfccs_20', 'confidence_ave', 'eye_height_std', 'eye_width_std']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## create audio files ##\n\n#output_format = 'wav'\n#output_dir = Path(f\"{output_format}s\")\n#Path(output_dir).mkdir(exist_ok=True, parents=True)\n#WAV_PATH = './wavs/'\n#def create_wav(file, output_dir, output_format):\n#    command = f\"../working/ffmpeg-git-20200305-amd64-static/ffmpeg -i {file} -ab 192000 -ac 2 -ar 44100 -vn {output_dir/file[-14:-4]}.{output_format}\"\n#    subprocess.call(command, shell=True)\n#    \n#test_videos = sorted([x for x in os.listdir(path+'test_videos/') if x[-4:] == \".mp4\"])\n#for video_file in test_videos: \n#    file_name = os.path.join(path,video_file)\n#    create_wav(file_name, output_dir, output_format)\n#    name = video_file[:-4] + \".wav\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## process into feature dataframe ##\n\ntest_df = pd.DataFrame()\nfile_names = os.listdir(path+'test_videos/')\n#file_names = file_names[0:3]\ndetector = MTCNN()\nfor file_name in tqdm(file_names):\n  all_features = get_all_features(path+'test_videos/', file_name, detector)\n  test_df = test_df.append(all_features)\ntest_df = test_df[all_feature_names].fillna(test_df.mean()) \ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict Using Pre-trained Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load Keras model traind on Databricks\nfrom numpy import loadtxt\nfrom keras.models import load_model\n \n# load model\nmodel = load_model('/kaggle/input/test-model-h5/deepfake_keras_model_01.h5')\n# summarize model.\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run prediction\npredictions = model.predict(test_df)[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate submission\nsubmission = pd.DataFrame(data={'filename': file_names, 'label': predictions}).fillna(0.5)\nsubmission = pd.DataFrame({'filename': submission['filename'], 'label': submission['label'].clip(0.4, 0.6)})\nsubmission.to_csv(\"/kaggle/working/submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}