{"cells":[{"metadata":{},"cell_type":"markdown","source":"This CPU-only kernel is a Deep Fakes video EDA. It relies on [static FFMPEG](http://https://www.kaggle.com/rakibilly/ffmpeg-static-build) to read/extract data from videos.\n\n- It extracts meta-data. They help us to know frame rate, dimensions and audio format (we can forget leak of \"display_ratio\" as it will be fixed).\n- It extracts frames of videos as PNG.\n- It extracts audio track as AAC (disabled).\n- It compares a few face detectors (OpenCV HaarCascade, MTCNN). More to come (Yolo, BlazeFace, DLib, Faced, ...).\n- It provides basic statistics on faces per video, face width/height and face detection confidence. It computes an average face width/height.\n\nWe notice that face detection (with MTCNN currently) is far from being perfect. An additional stage to clean-up detected faces is required before training a model! \nMaybe some kind of votes/ensemble with different detectors would help.\n\nIn this kernel you will see also some interesting edge cases of face detection:\n- Face detected on a t-shirt.\n- Face detected on a background board.\n- Face detected inside a face.\n\nFinally, most faces would fit inside a 256x256 to 320x320 box. It should help to define further models based on faces."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install some packages","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!tar xvf /kaggle/input/ffmpeg-static-build/ffmpeg-git-amd64-static.tar.xz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install mtcnn","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/working'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob, shutil\nimport timeit, os, gc\nimport subprocess as sp\nfrom tqdm import tqdm\nfrom collections import defaultdict\nfrom concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor\nimport json\nfrom IPython.display import HTML\nfrom base64 import b64encode\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_colwidth', -1)\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_rows', 4000)\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(font_scale=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HOME = \"./\"\nFFMPEG = \"/kaggle/working/ffmpeg-git-20191209-amd64-static\"\nFFMPEG_PATH = FFMPEG\nDATA_FOLDER = \"/kaggle/input/deepfake-detection-challenge\"\nTMP_FOLDER = HOME\nDATA_FOLDER_TRAIN = DATA_FOLDER\nVIDEOS_FOLDER_TRAIN = DATA_FOLDER_TRAIN + \"/train_sample_videos\"\nIMAGES_FOLDER_TRAIN = TMP_FOLDER + \"/images\"\nAUDIOS_FOLDER_TRAIN = TMP_FOLDER + \"/audios\"\nEXTRACT_META = True # False\nEXTRACT_CONTENT = True # False\nEXTRACT_FACES = True # False\nFRAME_RATE = 0.5 # Frame per second to extract (max is 30.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_command(*popenargs, **kwargs):\n    closeNULL = 0\n    try:\n        from subprocess import DEVNULL\n        closeNULL = 0\n    except ImportError:\n        import os\n        DEVNULL = open(os.devnull, 'wb')\n        closeNULL = 1\n\n    process = sp.Popen(stdout=sp.PIPE, stderr=DEVNULL, *popenargs, **kwargs)\n    output, unused_err = process.communicate()\n    retcode = process.poll()\n\n    if closeNULL:\n        DEVNULL.close()\n\n    if retcode:\n        cmd = kwargs.get(\"args\")\n        if cmd is None:\n            cmd = popenargs[0]\n        error = sp.CalledProcessError(retcode, cmd)\n        error.output = output\n        raise error\n        \n    return output\n\ndef ffprobe(filename, options = [\"-show_error\", \"-show_format\", \"-show_streams\", \"-show_programs\", \"-show_chapters\", \"-show_private_data\"]):\n    ret = {}\n    command = [FFMPEG_PATH + \"/ffprobe\", \"-v\", \"error\", *options, \"-print_format\", \"json\", filename]\n    ret = run_command(command)\n    if ret:\n        ret = json.loads(ret)\n    return ret\n\n# ffmpeg -i input.mov -r 0.25 output_%04d.png\ndef ffextract_frames(filename, output_folder, rate = 0.25):\n    command = [FFMPEG_PATH + \"/ffmpeg\", \"-i\", filename, \"-r\", str(rate), \"-y\", output_folder + \"/output_%04d.png\"]\n    ret = run_command(command)\n    return ret\n\n# ffmpeg -i input-video.mp4 output-audio.mp3\ndef ffextract_audio(filename, output_path):\n    command = [FFMPEG_PATH + \"/ffmpeg\", \"-i\", filename, \"-vn\", \"-ac\", \"1\", \"-acodec\", \"copy\", \"-y\", output_path]\n    ret = run_command(command)\n    return ret","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\njs = ffprobe(VIDEOS_FOLDER_TRAIN + \"/\"+ \"bqdjzqhcft.mp4\")\nprint(json.dumps(js, indent=4, sort_keys=True))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract some meta-data\nif EXTRACT_META == True:\n    results = []\n    subfolder = VIDEOS_FOLDER_TRAIN\n    filepaths = glob.glob(subfolder + \"/*.mp4\")\n    for filepath in tqdm(filepaths):\n        js = ffprobe(filepath)\n        if js:\n            results.append(\n                (js.get(\"format\", {}).get(\"filename\")[len(subfolder) + 1:],\n                js.get(\"format\", {}).get(\"format_long_name\"),\n                # Video \n                js.get(\"streams\", [{}, {}])[0].get(\"codec_name\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"height\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"width\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"nb_frames\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"bit_rate\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"duration\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"start_time\"),\n                js.get(\"streams\", [{}, {}])[0].get(\"avg_frame_rate\"),\n                # Audio\n                js.get(\"streams\", [{}, {}])[1].get(\"codec_name\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"channels\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"sample_rate\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"nb_frames\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"bit_rate\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"duration\"),\n                js.get(\"streams\", [{}, {}])[1].get(\"start_time\")),\n            )\n\n    meta_pd = pd.DataFrame(results, columns=[\"filename\", \"format\", \"video_codec_name\", \"video_height\", \"video_width\",\n                                            \"video_nb_frames\", \"video_bit_rate\", \"video_duration\", \"video_start_time\",\"video_fps\",\n                                            \"audio_codec_name\", \"audio_channels\", \"audio_sample_rate\", \"audio_nb_frames\",\n                                            \"audio_bit_rate\", \"audio_duration\", \"audio_start_time\"])\n    meta_pd[\"video_fps\"] = meta_pd[\"video_fps\"].apply(lambda x: float(x.split(\"/\")[0])/float(x.split(\"/\")[1]) if len(x.split(\"/\")) == 2 else None)\n    meta_pd[\"video_duration\"] = meta_pd[\"video_duration\"].astype(np.float32)\n    meta_pd[\"video_bit_rate\"] = meta_pd[\"video_bit_rate\"].astype(np.float32)\n    meta_pd[\"video_start_time\"] = meta_pd[\"video_start_time\"].astype(np.float32)\n    meta_pd[\"video_nb_frames\"] = meta_pd[\"video_nb_frames\"].astype(np.float32)\n    meta_pd[\"video_bit_rate\"] = meta_pd[\"video_bit_rate\"].astype(np.float32)\n    meta_pd[\"audio_sample_rate\"] = meta_pd[\"audio_sample_rate\"].astype(np.float32)\n    meta_pd[\"audio_nb_frames\"] = meta_pd[\"audio_nb_frames\"].astype(np.float32)\n    meta_pd[\"audio_bit_rate\"] = meta_pd[\"audio_bit_rate\"].astype(np.float32)\n    meta_pd[\"audio_duration\"] = meta_pd[\"audio_duration\"].astype(np.float32)\n    meta_pd[\"audio_start_time\"] = meta_pd[\"audio_start_time\"].astype(np.float32)\n    meta_pd.to_pickle(HOME + \"videos_meta.pkl\")\nelse:\n    meta_pd = pd.read_pickle(HOME + \"videos_meta.pkl\")\nmeta_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,6, figsize=(22, 3))\nd = sns.distplot(meta_pd[\"video_fps\"], ax=ax[0])\nd = sns.distplot(meta_pd[\"video_duration\"], ax=ax[1])\nd = sns.distplot(meta_pd[\"video_width\"], ax=ax[2])\nd = sns.distplot(meta_pd[\"video_height\"], ax=ax[3])\nd = sns.distplot(meta_pd[\"video_nb_frames\"], ax=ax[4])\nd = sns.distplot(meta_pd[\"video_bit_rate\"], ax=ax[5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,6, figsize=(22, 3))\nd = sns.distplot(meta_pd[\"audio_channels\"], ax=ax[0])\nd = sns.distplot(meta_pd[\"audio_sample_rate\"], ax=ax[1])\nd = sns.distplot(meta_pd[\"audio_nb_frames\"], ax=ax[2])\nd = sns.distplot(meta_pd[\"audio_bit_rate\"], ax=ax[3])\nd = sns.distplot(meta_pd[\"audio_duration\"], ax=ax[4])\nd = sns.distplot(meta_pd[\"audio_start_time\"], ax=ax[5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pd = pd.read_json(VIDEOS_FOLDER_TRAIN + \"/metadata.json\").T.reset_index().rename(columns={\"index\": \"filename\"})\ntrain_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pd = pd.merge(train_pd, meta_pd[[\"filename\", \"video_height\", \"video_width\", \"video_nb_frames\", \"video_bit_rate\", \"audio_nb_frames\"]], on=\"filename\", how=\"left\")\ntrain_pd[\"count\"] = train_pd.groupby([\"original\"])[\"original\"].transform('count')\n# train_pd.to_pickle(HOME + \"train_meta.pkl\")\ntrain_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Audio extract commented out to avoid disk full.\nAUDIO_FORMAT = \"aac\" # \"wav\"\nvideos_folder = VIDEOS_FOLDER_TRAIN\nimages_folder_path = IMAGES_FOLDER_TRAIN\naudios_folder_path = AUDIOS_FOLDER_TRAIN\nif EXTRACT_CONTENT == True:\n    # 1h20min for chunk#0 (11GB)\n    # Extract some images + audio track\n    for idx, row in tqdm(train_pd.iterrows(), total=meta_pd.shape[0]):\n        try:\n            video_path = videos_folder + \"/\" + row[\"filename\"]\n            images_path = images_folder_path + \"/\" + row[\"filename\"][:-4]\n            audio_path = audios_folder_path + \"/\" + row[\"filename\"][:-4]\n            # Extract images\n            if not os.path.exists(images_path): os.makedirs(images_path)\n            ret = ffextract_frames(video_path, images_path, rate = FRAME_RATE)\n            # Extract audio\n            if not os.path.exists(audio_path): os.makedirs(audio_path)\n            # ret = ffextract_audio(video_path, audio_path + \"/audio.\" + AUDIO_FORMAT)\n        except:\n            print(\"Cannot extract frames/audio for:\" + row[\"filename\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pd.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preview Fake/Real (this one is obvious)\nidx = 21 # 27 # 21 # 19 # 12 # 6\nfake = train_pd[\"filename\"][idx]\nreal = train_pd[\"original\"][idx]\nvid_width = train_pd[\"video_width\"][idx]\nvid_real = open(VIDEOS_FOLDER_TRAIN + \"/\" + real, 'rb').read()\ndata_url_real = \"data:video/mp4;base64,\" + b64encode(vid_real).decode()\nvid_fake = open(VIDEOS_FOLDER_TRAIN + \"/\" + fake, 'rb').read()\ndata_url_fake = \"data:video/mp4;base64,\" + b64encode(vid_fake).decode()\nHTML(\"\"\"\n<div style='width: 100%%; display: table;'>\n    <div style='display: table-row'>\n        <div style='width: %dpx; display: table-cell;'><b>Real</b>: %s<br/><video width=%d controls><source src=\"%s\" type=\"video/mp4\"></video></div>\n        <div style='display: table-cell;'><b>Fake</b>: %s<br/><video width=%d controls><source src=\"%s\" type=\"video/mp4\"></video></div>\n    </div>\n</div>\n\"\"\" % ( int(vid_width/3.2) + 10, \n       real, int(vid_width/3.2), data_url_real, \n       fake, int(vid_width/3.2), data_url_fake))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# OpenCV face detector\nface_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + \"haarcascade_frontalface_default.xml\")\n\ndef detect_face_cv2(img):\n    # Move to grayscale\n    gray_img = cv2.cvtColor(img.copy(), cv2.COLOR_RGB2GRAY)\n    face_locations = []\n    face_rects = face_cascade.detectMultiScale(gray_img, scaleFactor=1.3, minNeighbors=5)     \n    for (x,y,w,h) in face_rects: \n        face_location = (x,y,w,h)\n        face_locations.append((face_location, 1.0))\n    return face_locations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MTCNN face detector\nfrom mtcnn import MTCNN\ndetector = MTCNN()\n\ndef detect_face_mtcnn(img):\n    face_locations = []\n    items = detector.detect_faces(img)\n    for face in items:\n        face_location = tuple(face.get('box'))\n        face_confidence = float(face.get('confidence'))\n        face_locations.append((face_location, face_confidence))\n    return face_locations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# return ((x,y,w,h, confidence))\ndef extract_faces(files, source, detector=detect_face_cv2):\n    results = []\n    # for idx, file in tqdm(enumerate(files), total=len(files)):\n    for idx, file in enumerate(files):\n        try:\n            img = cv2.cvtColor(cv2.imread(file, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB)\n            face_locations = detector(img)\n            results.append((source, file[file.find(\"output_\"):], face_locations, len(face_locations)))\n        except:\n            print(\"Cannot extract faces for image: %s\" % file)\n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file = fake\ndump_folder = images_folder_path + \"/\" + file[:-4]\nfiles = glob.glob(dump_folder + \"/*\")\nDETECTORS = {\n    \"cv2\": detect_face_cv2,\n    \"mtcnn\": detect_face_mtcnn\n}\nfaces_pd = None\nfor key, value in DETECTORS.items():\n    tmp_pd = pd.DataFrame(extract_faces(files, file, detector=value), columns=[\"filename\", \"image\", \"boxes_\" + key , \"faces_\" + key])\n    if faces_pd is None:\n        faces_pd = tmp_pd\n    else:\n        faces_pd = pd.merge(faces_pd, tmp_pd, on=[\"filename\", \"image\"], how=\"left\")\nfaces_pd.head(12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot faces extracted images\ndef plot_faces_boxes(df, max_cols = 2, max_rows = 6, fsize=(24, 5), max_items=12):    \n    idx = 0    \n    for item_idx, item in df.iterrows():\n        img = cv2.cvtColor(cv2.imread(IMAGES_FOLDER_TRAIN + \"/\" + item[\"filename\"][:-4] +\"/\" + item[\"image\"], cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB)    \n        face_img = img #.copy()\n        # grid subplots\n        row = idx // max_cols\n        col = idx % max_cols\n        if col == 0: fig = plt.figure(figsize=fsize)\n        ax = fig.add_subplot(1, max_cols, col + 1)\n        ax.axis(\"off\")\n        # display image with boxes\n        cols = [c for c in df.columns if \"boxes\" in c]\n        for i, c in enumerate(cols, 0):\n            face_locations = item[c]\n            face_confidence = item[c]            \n            if len(face_locations) > 0:\n                for face_location in face_locations:        \n                    ((x,y,w,h), confidence) = face_location\n                    # face_img = face_img[y:y+h, x:x+w]\n                    cv2.rectangle(face_img, (x, y), (x+w, y+h), (255,i*255,0), 8)\n                    cv2.putText(face_img, '%.1f' % (confidence*100.0), (x+w, y+h), cv2.FONT_HERSHEY_SIMPLEX, 2.0, (255,i*255,0), 9, cv2.LINE_AA)\n                ax.imshow(face_img)\n            else:\n                ax.imshow(img)\n            ax.set_title(\"%s %s / %s - Faces: %d %s %s\" % (item[\"label\"] if \"label\" in df.columns else \"\", \n                                                           item[\"filename\"], item[\"image\"],\n                                                           item[\"faces_mtcnn\"] if \"faces_mtcnn\" in df.columns else len(face_locations),\n                                                           item[\"faces_mtcnn_median\"] if \"faces_mtcnn_median\" in df.columns else \"\",\n                                                           item[\"faces\"] if \"faces\" in df.columns else \"\"))\n        if (col == max_cols -1): plt.show()\n        idx = idx + 1\n        if (max_items > 0 and idx >=max_items): break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compare face boxes detected by OpenCV and MTCNN\nplot_faces_boxes(faces_pd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_detector_on_video(videos_filename, verbose=False):\n    if verbose == True: \n        print(\"Starting with batch of %d videos\" % len(videos_filename))\n    tmp_faces_pd = None\n    for file in videos_filename:\n        # Find out dump folder with images\n        dump_folder = images_folder_path + \"/\" + file[:-4]\n        # List files\n        files = glob.glob(dump_folder + \"/*\")\n        DETECTORS = {\n            \"mtcnn\": detect_face_mtcnn\n        }\n        for key, value in DETECTORS.items():\n            tmp_pd = pd.DataFrame(extract_faces(files, file, detector=value), columns=[\"filename\", \"image\", \"boxes_\" + key , \"faces_\" + key])\n            if tmp_faces_pd is None:\n                tmp_faces_pd = tmp_pd\n            else:\n                tmp_faces_pd = pd.concat([tmp_faces_pd, tmp_pd], axis=0)\n    return tmp_faces_pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport multiprocessing\ncpus = multiprocessing.cpu_count()\nif EXTRACT_FACES == True:\n    resultfutures = []\n    results = []\n    tasks = np.array_split(train_pd[\"filename\"].unique(), 20)\n    print(\"Tasks: %d\" % len(tasks))\n    with ThreadPoolExecutor(max_workers=cpus) as executor:\n        resultfutures = tqdm(executor.map(run_detector_on_video, tasks), total=len(tasks))\n    results = [x for x in resultfutures]\n    executor.shutdown()\n    # Gather results\n    all_faces_pd = None\n    for result in results:\n        if all_faces_pd is None:\n            all_faces_pd = result\n        else:\n            all_faces_pd = pd.concat([all_faces_pd, result], axis=0)\n    all_faces_pd = all_faces_pd.reset_index(drop=True)\n    all_faces_pd.to_pickle(HOME + \"faces.pkl\")\nelse:\n    all_faces_pd = pd.read_pickle(HOME + \"faces.pkl\")\nprint(all_faces_pd.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_faces_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# How many faces did we detect?\nall_faces_pd.groupby([\"faces_mtcnn\"]).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_faces_pd[\"faces_mtcnn_avg\"] = all_faces_pd.groupby(\"filename\")[\"faces_mtcnn\"].transform(np.nanmean)\nall_faces_pd[\"faces_mtcnn_median\"] = all_faces_pd.groupby(\"filename\")[\"faces_mtcnn\"].transform(np.nanmedian)\nall_faces_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(22, 3))\nd = sns.distplot(all_faces_pd[\"faces_mtcnn_avg\"], kde=True, ax=ax[0])\nd = sns.distplot(all_faces_pd[\"faces_mtcnn_median\"], kde=False, ax=ax[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_faces_pd.groupby([\"faces_mtcnn_median\"]).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MTCNN is not perfect. It detects face inside face and in trees.\nplot_faces_boxes(all_faces_pd[all_faces_pd[\"faces_mtcnn\"] == 3], max_items=24)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let see some frames with 2 faces detected by MTCNN as majority (median).\nplot_faces_boxes(all_faces_pd[(all_faces_pd[\"faces_mtcnn\"] == 2) & (all_faces_pd[\"faces_mtcnn_median\"] == 2)], max_items=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_faces_pd = pd.merge(all_faces_pd, train_pd, on=\"filename\", how=\"left\")\nclean_faces_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find out face width/height\ndef faces_max_item(boxes, idx1, idx2):\n    ret = 0\n    if len(boxes) > 0:\n        ret = max(boxes, key=lambda item: item[idx1][idx2])[idx1][idx2]\n    return ret\n\ndef faces_max_confidence(boxes):\n    ret = 0\n    if len(boxes) > 0:\n        ret = max(boxes, key=lambda item: item[1])[1]\n    return ret\n\ndef faces_min_confidence(boxes):\n    ret = 0\n    if len(boxes) > 0:\n        ret = min(boxes, key=lambda item: item[1])[1]\n    return ret\n\nclean_faces_pd[\"faces_max_width\"] = clean_faces_pd[\"boxes_mtcnn\"].apply(lambda x: faces_max_item(x, 0, 2)) \nclean_faces_pd[\"faces_max_height\"] = clean_faces_pd[\"boxes_mtcnn\"].apply(lambda x: faces_max_item(x, 0, 3))\nclean_faces_pd[\"faces_max_conf\"] = clean_faces_pd[\"boxes_mtcnn\"].apply(lambda x: faces_max_confidence(x))\nclean_faces_pd[\"faces_min_conf\"] = clean_faces_pd[\"boxes_mtcnn\"].apply(lambda x: faces_min_confidence(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# If we train a CNN, we will have to define a width/height. 256x256 or 320x320 looks good.\nprint(\"Faces stats:\")\nprint(clean_faces_pd[[\"faces_max_width\", \"faces_max_height\", \"faces_min_conf\", \"faces_max_conf\"]].describe(percentiles=[0.01,0.05, 0.1,0.25,0.5,0.75,0.9,0.95,0.99]))\nfig, ax = plt.subplots(1, 2, figsize=(22, 3))\nd = sns.distplot(clean_faces_pd[\"faces_max_width\"], kde=True, ax=ax[0])\nd = sns.distplot(clean_faces_pd[\"faces_max_height\"], kde=True, ax=ax[1])\nplt.show()\nfig, ax = plt.subplots(1, 2, figsize=(22, 3))\nd = sns.distplot(clean_faces_pd[\"faces_min_conf\"], kde=True, ax=ax[0])\nd = sns.distplot(clean_faces_pd[\"faces_max_conf\"], kde=True, ax=ax[1])\nfig, ax = plt.subplots(figsize=(22, 3))\nd = clean_faces_pd.plot(kind=\"scatter\", x=\"faces_max_width\", y=\"faces_max_conf\", c=\"red\", ax=ax, label=\"faces_max_width\", alpha=0.5)\nd = clean_faces_pd.plot(kind=\"scatter\", x=\"faces_max_height\", y=\"faces_max_conf\", c=\"blue\", ax=d,  label=\"faces_max_height\", alpha=0.5)\nd = plt.legend(loc=\"upper right\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_faces_pd.to_pickle(HOME + \"faces.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean temporary folders\nshutil.rmtree(FFMPEG)\nshutil.rmtree(IMAGES_FOLDER_TRAIN)\nshutil.rmtree(AUDIOS_FOLDER_TRAIN)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}