{"cells":[{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%HTML\n\n<style type=\"text/css\">\n     \n \ndiv.h2 {\n    background-color: #159957;\n    background-image: linear-gradient(120deg, #155799, #159957);\n    text-align: left;\n    color: white;              \n    padding:9px;\n    padding-right: 100px; \n    font-size: 20px; \n    max-width: 1500px; \n    margin: auto; \n    margin-top: 40px; \n}\n                                     \n                                      \nbody {\n  font-size: 11px;\n}    \n     \n                                    \n                                      \ndiv.h3 {\n    color: #159957; \n    font-size: 18px; \n    margin-top: 20px; \n    margin-bottom:4px;\n}\n   \n                                      \ndiv.h4 {\n    color: #159957;\n    font-size: 15px; \n    margin-top: 20px; \n    margin-bottom: 8px;\n}\n   \n                                      \nspan.note {\n    font-size: 7; \n    color: gray; \n    font-style: italic;\n}\n  \n                                      \nhr {\n    display: block; \n    color: gray\n    height: 1px; \n    border: 0; \n    border-top: 1px solid;\n}\n  \n                                      \nhr.light {\n    display: block; \n    color: lightgray\n    height: 1px; \n    border: 0; \n    border-top: 1px solid;\n}   \n    \n                                      \ntable.dataframe th \n{\n    border: 1px darkgray solid;\n    color: black;\n    background-color: white;\n}\n    \n                                      \ntable.dataframe td \n{\n    border: 1px darkgray solid;\n    color: black;\n    background-color: white;\n    font-size: 11px;\n    text-align: center;\n} \n   \n            \n                                      \ntable.rules th \n{\n    border: 1px darkgray solid;\n    color: black;\n    background-color: white;\n    font-size: 11px;\n    align: left;\n}\n       \n                                      \ntable.rules td \n{\n    border: 1px darkgray solid;\n    color: black;\n    background-color: white;\n    font-size: 13px;\n    text-align: center;\n} \n                                       \n                                      \ntable.rules tr.best\n{\n    color: green;\n}    \n                             \n.output { \n    align-items: left; \n}\n        \n                                      \n.output_png {\n    display: table-cell;\n    text-align: left;\n    margin:auto;\n}                                          \n                                                                    \n                                      \n                                      \n</style> \n                                     \n                                      ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#  Reference: \n#      - I really liked the way JohnM's punt kaggle submission had the headers, extremely aesthetically pleasing\n#        and aids viewing - borrowing his div.h header concept (so much nicer looking than using conventional\n#        ## headers etc), and adding a 'cayman' color theme to it, as a nod to R ...  \n#        Isn't it nice looking ?  ->  https://jasonlong.github.io/cayman-theme/\n#      - I would strongly suggest we follow JohnM's push into professoinal looking css-based headers, we can't \n#        keep using old-fashioned markdown for headers, its so limited... just my personal opinion\n#\n# -%%HTML\n# <style type=\"text/css\">\n#\n# div.h2 {\n#     background-color: steelblue; \n#     color: white; \n#     padding: 8px; \n#     padding-right: 300px; \n#     font-size: 20px; \n#     max-width: 1500px; \n#     margin: auto; \n#     margin-top: 50px;\n# }\n# etc\n# etc\n# --- end reference ---\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n# UNCOMMENT ALL OF THIS OUT:\n# abc\n# def\n#\n#\n#\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nimport numpy as np\nimport pandas as pd\nimport matplotlib. pyplot as plt\n%matplotlib inline\nimport matplotlib.patches as patches\nimport seaborn as sns\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nimport warnings\nwarnings.filterwarnings('ignore')\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#import sparklines\nimport colorcet as cc\nplt.style.use('seaborn') \ncolor_pal = [x['color'] for x in plt.rcParams['axes.prop_cycle']]\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n##%config InlineBackend.figure_format = 'retina'   < - keep in case \n%config InlineBackend.figure_format = 'svg' \n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n# from sklearn import preprocessing\n# from sklearn.model_selection import KFold\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.ensemble import RandomForestRegressor\n# from sklearn.model_selection import KFold\n# from sklearn.feature_selection import SelectFromModel\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nfrom IPython.display import Video\nfrom IPython.display import HTML\nfrom IPython.display import Image\nfrom IPython.display import display\nfrom IPython.core.display import display\nfrom IPython.core.display import HTML\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nimport cv2 as cv  # or import cv2 as cv\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nimport json\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nfrom tqdm import tqdm_notebook\nfrom tqdm import tqdm\n#import gc, pickle, tqdm, os, datetime\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nfrom skimage.measure import compare_ssim\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Intro:</i></div>  \n* I pretty much know nothing about neural networks, face detection, OpenCV, etc, but let's give this a shot \n  * This is my second Kaggle competition"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Approach:</i></div>\n* Start at the absolute bottom, and learn how neural networks actually work\n* Research CV \n* Build on knowledge to get deeper and deeper into variations of CNN/RNN, etc\n* Learn Keras\n* Investigate autoencoders\n* Dive hard core into the mathematics\n* Determine precisely how Deepfakes are made/created/propagated\n* Determine conventional way of detecting Deepfakes\n* Create an unconventional approach to detecting Deepfakes\n* Determine path forward "},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Summary of our dataset:</i></div>\n<p style=\"margin-top: 50px\">It is always important to look at our entire dataset and examine the descriptive statistics:</p>\n\n&ensp; **Number of training videos (.mp4):** &ensp;  &nbsp;  400  \n&ensp; **Number of test videos (.mp4):** &ensp; &nbsp;  &ensp; &ensp; &nbsp;  401  "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\ntrain_sample_metadata.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Percentage of Fake vs Real within training video dataset:</i></div>\n* 323 Fakes\n* 77 Real "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"pd.DataFrame(train_sample_metadata['label'].value_counts(normalize=True))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#train_sample_metadata.groupby('label')['label'].count()\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\ntrain_dir = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/'\ntrain_video_files = [train_dir + x for x in os.listdir(train_dir) if x.endswith('.mp4')]\ntest_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\ntest_video_files = [test_dir + x for x in os.listdir(test_dir)]\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\ndf_train = pd.read_json('/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json').transpose()\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#df_train.head()\n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#df_train.shape \n#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Examining a single video:</i></div>\n* Let's take a look at a random video: &nbsp;  drcyabprvt.mp4\n* We will freeze it's first frame and output as an image\n* Dimension:  (1080, 1920, 3)"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"\n# FREEZE-FRAME:\nimport cv2 as cv\nimport matplotlib.pyplot as plt\ndp1 = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/drcyabprvt.mp4'\n#dp2 = 'dzieklokdr.mp4'    \nfig, ax = plt.subplots(1,1, \n                       figsize=(8,8))\n# fake:  cap = cv.VideoCapture('/kaggle/input/deepfake-detection-challenge/train_sample_videos/dkrvorliqc.mp4') \n# cap = cv.VideoCapture('/kaggle/input/deepfake-detection-challenge/train_sample_videos/dzieklokdr.mp4')\nmycap = cv.VideoCapture(dp1); mycap.set(1,2)\nret, image = mycap.read()\nimage = cv.cvtColor(image, cv.COLOR_BGR2RGB)\nraw_image = image\n#print(raw_image.shape)\nmycap.release() \ncv.destroyAllWindows()\nax.set_xticks([]); ax.set_yticks([]); ax.imshow(image);\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Let's take a look at another random video frame image\n* Here you can see that the image is 1080 pixels x 1920 pixels (tick marks)\n* This time we will show a deepfake, of the same person, while keeping the pixel tick marks/tags: "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"\nfig, ax = plt.subplots(1,1, figsize=(8,8))\ncap = cv.VideoCapture('/kaggle/input/deepfake-detection-challenge/train_sample_videos/dkrvorliqc.mp4') \ncap.set(1,2); ret, image = cap.read()\nimage = cv.cvtColor(image, cv.COLOR_BGR2RGB)\ncap.release()   \ncv.destroyAllWindows()\nfile_name = 'dkrvorliqc.mp4'\nax.title.set_text(file_name)\nax.imshow(image); \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<hr>"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>The Problem:</i></div> \n* Deepfake videos are becoming easier and easier to make\n* Deepfake video sophistication is increasing"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Potential Methods to Detect Deepfakes:</i></div> \n* An AI-produced video (generative adversarial networks based) could show a world leader doing or saying something inflammatory, and worst-case scenario could lead to the population formulating a different opinion of the leader, or even triggering violence and chaos. \n* Soft-biometric signatures such as blink rate ? \n* I don't think face landmarks will necessarily solve this issue \n* It would make sense to say that the longer the video under inspection is, the **greater** the probability of detecting that it is in fact a deepfake video (if it were a deepfake video), i.e. direct correlation (probably) between length of video and probability of detecting its status\n  * Ideally the deepfake under investigation was longer, as this would allow an algorithm to find 'signatures' that it was 'tainted'\n* **aayfryxljh.mp4:**  &nbsp; Possible correlation between turning head beyond 45 degrees and blink ?   Glasses reflection make it harder to modify video due to reflection background ? \n<img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/reflection.png\" width=\"200px\">\n<img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/reflection2.png\" width=\"400px\">\n"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>The problem is not the face, it's the concentric circle:</i></div> \n* Concentric Circles:  Two or more circles which have the same center point\n* Concentric Ovals:  Faces are closer to ovals than circles\n* Perhaps mathematically constructing the differences in the region between the inserted face and the original face, many times it appears this area has a signature that is easier to see\n<br>\n<img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/tom2.jpg\" width=\"800px\">\n<br>\n<img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/concentric.jpg\" width=\"200px\">\n\n"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Calendar 2020:</i></div> \n* Why do we care ? \n  * The United States of America has a presidential election November 3rd, 2020\n  * The period of time from July to November will be a period of time when U.S. citizens will be inundated (whether they like it or not) with media coverage\n  * Citizens may be particularly susceptible to misinformation/disinformation/deepfakes"},{"metadata":{},"cell_type":"markdown","source":"<img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/election3.png\" width=\"700px\">"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Categorizing Deepfakes:</i></div> \n* It would seem that categorizing/quantifying the potential damage a deepfake could inflict may be necessary at some point\n* Some deepfakes are relatively harmless, while others may be more damaging\n* J-level assignment, from 1 to 10, i.e. J-level of 9 would be a deepfake associated with inflicting chaos during the run up to an election, misrepresenting what a candidate stated as their position, etc. Deepfake with J-level of 1 would be effectively harmless.  \n"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Quantum Neural Network (QNN):</i></div>  \n* Conventional means of combating Deepfakes may not be possible\n* A new means of detecting Deepfakes may be more successful\n* Perhaps a new quantum-based approach ?  \n  * I don't think the term exists yet, but maybe what could be known as QNNs ? \n  * Why ?  Because using a conventional approach to detect Deepfakes could potentially be used in the same algorithm that is used to create Deepfakes, nullifying the gain"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>Reality Check:</i></div> \n* There seems to be the belief that it is in fact possible to detect Deepfakes, when over the course of time as technology/algorithms advance, it **may not actually be possible** to detect them with high probability\n* In that event, potential paths forward:\n* Modification of the 5G standard to allow direct IPSEC-like connections from user elements (UEs) to secure video servers, which are considered to be 'the source of truth' from various publications / news agencies / government agencies"},{"metadata":{},"cell_type":"markdown","source":"<div class=\"h3\"><i>References:</i></div>\n[1]  Deepfake, Wikipedia, https://en.wikipedia.org/wiki/Deepfake  "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# overall notes, do not destroy:\n#\n# ![title](https://www.desipio.com/wp-content/uploads/2019/06/walter-payton-leap-2-ah.jpg)\n# <br>&ensp; *Walter Payton (34) and the need for z-coordinate data ...*\n#\n#\n#\n#\n#\n#\n#\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n#\n#\n#\n#\n#\n# <img src=\"https://raw.githubusercontent.com/tombresee/Temp/master/ENTER/box2.png\" width=\"400px\">\n#\n#\n#\n#\n#\n#  https://matplotlib.org/3.1.0/gallery/subplots_axes_and_figures/gridspec_nested.html#sphx-glr-gallery-subplots-axes-and-figures-gridspec-nested-py\n#\n#\n#\n#\n# import numpy as np\n# import cv2\n\n# cap = cv2.VideoCapture('/kaggle/input/deepfake-detection-challenge/train_sample_videos/dkrvorliqc.mp4')\n\n# while(cap.isOpened()):\n#     ret, frame = cap.read()\n\n#     gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n\n#     cv2.imshow('frame',gray)\n#     if cv2.waitKey(1) & 0xFF == ord('q'):\n#         break\n\n# cap.release()\n# cv2.destroyAllWindows()\n\n\n# keep:\n# cap = cv2.VideoCapture(0)\n\n# while(True):\n#     # Capture frame-by-frame\n#     ret, frame = cap.read()\n\n#     # Our operations on the frame come here\n#     gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n\n#     # Display the resulting frame\n#     cv2.imshow('frame',gray)\n#     if cv2.waitKey(1) & 0xFF == ord('q'):\n#         break\n\n# # When everything done, release the capture\n# cap.release()\n# cv2.destroyAllWindows()\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.5"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":true,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":true,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}