{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is a shameless copy from the amazing kernel . So please upvote the original\n\nhttps://www.kaggle.com/gpreda/deepfake-starter-kit"},{"metadata":{},"cell_type":"markdown","source":"# Data exploration"},{"metadata":{},"cell_type":"markdown","source":"## Load packages"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\n%matplotlib inline \nimport cv2 as cv\nfrom os.path import join\nimport argparse\nimport subprocess\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge/'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos/'\nTEST_FOLDER = 'test_videos/'\nDATA_PATH = os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)\nos.makedirs('/kaggle/working/output', exist_ok=True)\nos.makedirs('/kaggle/working/test_output', exist_ok=True)\nOUTPUT_PATH = '/kaggle/working/output'\nTEST_OUTPUT_PATH = '/kaggle/working/test_output/'\nprint(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\nprint(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")\nSPLIT='00'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check files type\n\nHere we check the train data files extensions. Most of the files looks to have `mp4` extension, let's check if there is other extension as well."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\next_dict = []\nfor file in train_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\")      ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's count how many files with each extensions there are."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for file_ext in ext_dict:\n    print(f\"Files with extension `{file_ext}`: {len([file for file in train_list if  file.endswith(file_ext)])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's repeat the same process for test videos folder."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test_list = list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))\next_dict = []\nfor file in test_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\")\nfor file_ext in ext_dict:\n    print(f\"Files with extension `{file_ext}`: {len([file for file in train_list if  file.endswith(file_ext)])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check the `json` file first."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"json_file = [file for file in train_list if  file.endswith('json')][0]\nprint(f\"JSON file: {json_file}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Aparently here is a metadata file. Let's explore this JSON file."},{"metadata":{"trusted":true},"cell_type":"code","source":"any('ccfoszqabv.mp4'  in item for item in os.listdir(DATA_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)\nmeta_train_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Meta data exploration\n\nLet's explore now the meta data in train sample. \n\n### Missing data\n\nWe start by checking for any missing values.  "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def missing_data(data):\n    total = data.isnull().sum()\n    percent = (data.isnull().sum()/data.isnull().count()*100)\n    tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    types = []\n    for col in data.columns:\n        dtype = str(data[col].dtype)\n        types.append(dtype)\n    tt['Types'] = types\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"missing_data(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are missing data 19.25% of the samples (or 77). We suspect that actually the real data has missing original (if we generalize from the data we glimpsed). Let's check this hypothesis."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"missing_data(meta_train_df.loc[meta_train_df.label=='REAL'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Indeed, all missing `original` data are the one associated with `REAL` label.  \n\n### Unique values\n\nLet's check into more details the unique values."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def unique_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Total']\n    uniques = []\n    for col in data.columns:\n        unique = data[col].nunique()\n        uniques.append(unique)\n    tt['Uniques'] = uniques\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"unique_values(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* We observe that `original` label has the same pattern for uniques values. We know that we have 77 missing data (that's why total is only 323) and we observe that we do have 209 unique examples.  \n\n### Most frequent originals\n\nLet's look now to the most frequent originals uniques in train sample data.  "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def most_frequent_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Total']\n    items = []\n    vals = []\n    for col in data.columns:\n        itm = data[col].value_counts().index[0]\n        val = data[col].value_counts().values[0]\n        items.append(itm)\n        vals.append(val)\n    tt['Most frequent item'] = items\n    tt['Frequence'] = vals\n    tt['Percent from total'] = np.round(vals / total * 100, 3)\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"most_frequent_values(meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see that most frequent **label** is `FAKE` (80.75%), `meawmsgiti.mp4` is the most frequent **original** (6 samples)."},{"metadata":{},"cell_type":"markdown","source":"Let's do now some data distribution visualizations."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def plot_count(feature, title, df, size=1):\n    f, ax = plt.subplots(1,1, figsize=(4*size,4))\n    total = float(len(df))\n    g = sns.countplot(df[feature], order = df[feature].value_counts().index[:20], palette='Set3')\n    g.set_title(\"Number and percentage of {}\".format(title))\n    if(size > 2):\n        plt.xticks(rotation=90, size=8)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(100*height/total),\n                ha=\"center\") \n    plt.show()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plot_count('split', 'split (train)', meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plot_count('label', 'label (train)', meta_train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see, the `REAL` are only 19.25% in train sample videos, with the `FAKE`s acounting for 80.75% of the samples. \n\n\n## Video data exploration\n\n\nIn the following we will explore some of the video data. \n\n\n### Missing video (or meta) data\n\nWe check first if the list of files in the meta info and the list from the folder are the same.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"meta = np.array(list(meta_train_df.index))\nstorage = np.array([file for file in train_list if  file.endswith('mp4')])\nprint(f\"Metadata: {meta.shape[0]}, Folder: {storage.shape[0]}\")\nprint(f\"Files in metadata and not in folder: {np.setdiff1d(meta,storage,assume_unique=False).shape[0]}\")\nprint(f\"Files in folder and not in metadata: {np.setdiff1d(storage,meta,assume_unique=False).shape[0]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize now the data.  \n\nWe select first a list of fake videos.\n\n### Few fake videos"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fake_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='FAKE'].sample(3).index)\nfake_train_sample_video","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From [4] ([Basic EDA Face Detection, split video, ROI](https://www.kaggle.com/marcovasquez/basic-eda-face-detection-split-video-roi)) we modified a function for displaying a selected image from a video."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def display_image_from_video(video_path):\n    '''\n    input: video_path - path for video\n    process:\n    1. perform a video capture from the video\n    2. read the image\n    3. display the image\n    '''\n    capture_image = cv.VideoCapture(video_path) \n    ret, frame = capture_image.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n    ax.imshow(frame)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"for video_file in fake_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's try now the same for few of the images that are real.  \n\n\n### Few real videos"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"real_train_sample_video = list(meta_train_df.loc[meta_train_df.label=='REAL'].sample(3).index)\nreal_train_sample_video","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Videos with same original\n\nLet's look now to set of samples with the same original."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"meta_train_df['original'].value_counts()[0:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We pick one of the originals with largest number of samples.   \n\nWe also modify our visualization function to work with multiple images."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def display_image_from_video_list(video_path_list, video_folder=TRAIN_SAMPLE_FOLDER):\n    '''\n    input: video_path_list - path for video\n    process:\n    0. for each video in the video path list\n        1. perform a video capture from the video\n        2. read the image\n        3. display the image\n    '''\n    plt.figure()\n    fig, ax = plt.subplots(2,3,figsize=(16,8))\n    # we only show images extracted from the first 6 videos\n    for i, video_file in enumerate(video_path_list[0:6]):\n        video_path = os.path.join(DATA_FOLDER, video_folder,video_file)\n        capture_image = cv.VideoCapture(video_path) \n        ret, frame = capture_image.read()\n        frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n        ax[i//3, i%3].imshow(frame)\n        ax[i//3, i%3].set_title(f\"Video: {video_file}\")\n        ax[i//3, i%3].axis('on')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='meawmsgiti.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look now to a different selection of videos with the same original. "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='atvmxvwyns.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='qeumxirsme.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"same_original_fake_train_sample_video = list(meta_train_df.loc[meta_train_df.original=='kgbkktcjxf.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test video files\n\nLet's also look to few of the test data files."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"test_videos = pd.DataFrame(list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER))), columns=['video'])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"test_videos.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize now one of the videos."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[0].video))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look to some more videos from test set."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_image_from_video_list(test_videos.sample(6).video, TEST_FOLDER)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate Output Files from multiple frames of video"},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2 as cv2\n#https://github.com/ondyari/FaceForensics\n    \ndef extract_frames(data_path, output_path, method='cv2'):\n    \"\"\"Method to extract frames, either with ffmpeg or opencv. FFmpeg won't\n    start from 0 so we would have to rename if we want to keep the filenames\n    coherent.\"\"\"\n    os.makedirs(output_path, exist_ok=True)\n\n\n    if method == 'ffmpeg':\n        subprocess.check_output(\n            'ffmpeg -i {} {}'.format(\n                data_path, os.path.join(output_path, '%04d.png')),\n            shell=True, stderr=subprocess.STDOUT)\n    elif method == 'cv2':\n        reader = cv2.VideoCapture(data_path)\n        fps = int(reader.get(cv2.CAP_PROP_FPS))\n        frame_num = 0\n        while reader.isOpened():\n            success, image = reader.read()\n            if not success:\n                break\n            if frame_num%(100*fps) == 0 :    ## Take every 10 seconds of frame and export as image\n                cv2.imwrite(join(output_path, '{:04d}.png'.format(frame_num)),\n                            image)\n            frame_num += 1\n        reader.release()\n    else:\n        raise Exception('Wrong extract frames method: {}'.format(method))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_method_videos(data_path, outpath, compression):\n    \"\"\"Extracts all videos of a specified method and compression in the\n    FaceForensics++ file structure\"\"\"\n    videos_path = data_path\n    images_path = outpath\n    for video in tqdm(os.listdir(videos_path)):\n        image_folder = video.split('.')[0]\n        extract_frames(join(videos_path, video),\n                       join(images_path, image_folder))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I commented this because I dont need full size image as output anymore \n#extract_method_videos(DATA_PATH,OUTPUT_PATH,'c0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize'] = [15, 10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets try to see DenseFlow method\n ### -if there is any difference between real and fake movements"},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_REAL_VIDEO_PATH = os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)+'ccfoszqabv.mp4'\nSAMPLE_FAKE_VIDEO_PATH = os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)+'acqfdwsrhi.mp4'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport numpy as np\ndef showdenseflow(SAMPLE_VIDEO_PATH):\n    cap = cv2.VideoCapture(SAMPLE_VIDEO_PATH)\n    ret, frame1 = cap.read()\n    prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)\n    hsv = np.zeros_like(frame1)\n    hsv[...,1] = 255\n    for i in range(50):\n        ret, frame2 = cap.read()\n        next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)\n        flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 5, 5, 1.2, 0)\n        mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])\n        hsv[...,0] = ang*180/np.pi/2\n        hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)\n        rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)\n        plt.axis(\"off\")\n        plt.imshow(rgb,interpolation='nearest', aspect='auto')\n    cap.release()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#showdenseflow(SAMPLE_REAL_VIDEO_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#showdenseflow(SAMPLE_FAKE_VIDEO_PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Face Recognition Problem "},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom mtcnn.mtcnn import MTCNN\ndetector = MTCNN()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(19, 1, figsize=(200, 200))\naxs = np.array(axs)\naxs = axs.reshape(-1)\ni = 0\nfor fn in meta_train_df.index[:23]:\n    label = meta_train_df.loc[fn]['label']\n    orig = meta_train_df.loc[fn]['label']\n    video_file = f'/kaggle/input/deepfake-detection-challenge/train_sample_videos/{fn}'\n    ax = axs[i]\n    cap = cv.VideoCapture(video_file)\n    success, image = cap.read()\n    image = cv.cvtColor(image, cv.COLOR_BGR2RGB)\n    face_locations =  detector.detect_faces(image)\n    if len(face_locations) > 0:\n        # Print first face\n        for person in face_locations:\n            bounding_box = person['box']\n            keypoints = person['keypoints']\n    \n            cv2.rectangle(image,\n                          (bounding_box[0], bounding_box[1]),\n                          (bounding_box[0]+bounding_box[2], bounding_box[1] + bounding_box[3]),\n                          (0,155,255),\n                          2)\n    \n            cv2.circle(image,(keypoints['left_eye']), 2, (0,155,255), 2)\n            cv2.circle(image,(keypoints['right_eye']), 2, (0,155,255), 2)\n            cv2.circle(image,(keypoints['nose']), 2, (0,155,255), 2)\n            cv2.circle(image,(keypoints['mouth_left']), 2, (0,155,255), 2)\n            cv2.circle(image,(keypoints['mouth_right']), 2, (0,155,255), 2)\n    #display resulting frame\n        ax.imshow(image)\n        ax.grid(False)\n        ax.title.set_text(f'{fn} - {label}')\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)\n        i += 1\n        if i>18 :\n            break\nplt.grid(False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ADDITIONAL UTILS : GENERATE DATA FROM DOWNLOADED VIDEOS "},{"metadata":{"trusted":true},"cell_type":"code","source":"##Utility for cropping the faces via bounding box , cropping and padding\ndef crop(box,image):\n    x0 = box[0]\n    y0 = box[1]\n    w= box[2]\n    h= box[3]   \n    return image[y0:y0+h , x0:x0+w, :]\n\ndef cropnpad(box,image,pad):\n    x0 = box[0]\n    if x0-pad < 0:\n        x0 =pad\n    y0 = box[1]\n    if y0-pad <0 :\n        y0 =pad\n    w= box[2]\n    h= box[3]   \n    return image[y0-pad:y0+h+pad , x0-pad:x0+w+pad, :]\n\n\n##00faces12frames dataset contains the data from 00 set . I downloaded locally cropped using MTCNN and uploaded here \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize'] = [10, 5]\n## It will generate 12 faces from each set of videos \nfor fn in meta_train_df.index[:23]:\n    label = meta_train_df.loc[fn]['label']\n    orig = meta_train_df.loc[fn]['label']\n    video_file = f'{DATA_FOLDER}{TRAIN_SAMPLE_FOLDER}{fn}'\n    print(f\"{fn.split('.')[0]} n {label}\")\n    count=0\n    cap = cv2.VideoCapture(video_file)\n    #cap.set(cv2.CAP_PROP_FRAME_COUNT, frame_seq-1)\n    while cap.isOpened():      \n        success, image = cap.read()\n        if success :  \n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            face_locations =  detector.detect_faces(image)\n            if len(face_locations) > 0:\n            # Print first face\n                for person in face_locations:\n                    i=0\n                    bounding_box = person['box']\n                    keypoints = person['keypoints']\n                    print(bounding_box)\n                    print(image.shape)\n                    plt.imshow(cropnpad(bounding_box,image,0))           \n                    #plt.imsave(f\"{OUTPUT_PATH}{fn.split('.')[0]}-{label}-{str(i)}-{str(count)}.png\",cropnpad(bounding_box,image,0),format='png')\n                    plt.show()\n                    i+=1\n            count += 200 # i.e. at 25 fps, each video creates 12 images . I want to limit the output to 500 due to kaggle\n            cap.set(1, count)                   \n        else:\n            cap.release()\n            break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Archive output for later use "},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive('output.zip', 'zip', '/kaggle/working/output/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Prepparation for Model "},{"metadata":{"trusted":true},"cell_type":"code","source":"label_list =[]\nimage_list =os.listdir('../input/train-images/01/')\nnp_image_list = np.array(os.listdir('../input/train-images/01/'))\n\nfor i in image_list:\n    if 'REAL' in i:\n        \n        label_list.append(0)\n    else:\n        label_list.append(1)\n        \nnp_label_list = np.array(label_list)        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.DataFrame()\ntrain_df[\"ImageId\"]=np_image_list\ntrain_df[\"Label\"]=np_label_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Test images for prediction\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/deepfake-detection-challenge/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir ./out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_OUTPUT_PATH = './out'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor fn in sub.filename[:100]:\n    video_file = f'{DATA_FOLDER}{TEST_FOLDER}{fn}'\n    count=0\n    cap = cv2.VideoCapture(video_file)\n    print(video_file)\n    #cap.set(cv2.CAP_PROP_FRAME_COUNT, frame_seq-1)\n    while cap.isOpened():      \n        success, image = cap.read()\n        if success :  \n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            face_locations =  detector.detect_faces(image)\n            if len(face_locations) > 0:\n            # Print first face\n                for person in face_locations:\n                    i=0\n                    bounding_box = person['box']\n                    keypoints = person['keypoints']\n                    print(bounding_box)\n                    print(image.shape)\n                    plt.imshow(cropnpad(bounding_box,image,20))           \n                    plt.imsave(f\"{TEST_OUTPUT_PATH}{fn.split('.')[0]}-{label}-{str(i)}-{str(count)}.png\",cropnpad(bounding_box,image,20),format='png')\n                    plt.show()\n                    i+=1\n            count += 400 # i.e. at 25 fps, each video creates 12 images . I want to limit the output to 500 due to kaggle\n            cap.set(1, count) \n            \n        else:\n            cap.release()\n            break\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive('test_images', 'zip', './out/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Popular Model MesoNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport argparse\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\nimport torchvision\n\nclass Meso4(nn.Module):\n\t\"\"\"\n\tPytorch Implemention of Meso4\n\tAutor: Honggu Liu\n\tDate: July 4, 2019\n\t\"\"\"\n\tdef __init__(self, num_classes=2):\n\t\tsuper(Meso4, self).__init__()\n\t\tself.num_classes = num_classes\n\t\tself.conv1 = nn.Conv2d(3, 8, 3, padding=1, bias=False)\n\t\tself.bn1 = nn.BatchNorm2d(8)\n\t\tself.relu = nn.ReLU(inplace=True)\n\t\tself.leakyrelu = nn.LeakyReLU(0.1)\n\n\t\tself.conv2 = nn.Conv2d(8, 8, 5, padding=2, bias=False)\n\t\tself.bn2 = nn.BatchNorm2d(16)\n\t\tself.conv3 = nn.Conv2d(8, 16, 5, padding=2, bias=False)\n\t\tself.conv4 = nn.Conv2d(16, 16, 5, padding=2, bias=False)\n\t\tself.maxpooling1 = nn.MaxPool2d(kernel_size=(2, 2))\n\t\tself.maxpooling2 = nn.MaxPool2d(kernel_size=(4, 4))\n\t\t#flatten: x = x.view(x.size(0), -1)\n\t\tself.dropout = nn.Dropout2d(0.5)\n\t\tself.fc1 = nn.Linear(16*8*8, 16)\n\t\tself.fc2 = nn.Linear(16, num_classes)\n\n\tdef forward(self, input):\n\t\tx = self.conv1(input) #(8, 256, 256)\n\t\tx = self.relu(x)\n\t\tx = self.bn1(x)\n\t\tx = self.maxpooling1(x) #(8, 128, 128)\n\n\t\tx = self.conv2(x) #(8, 128, 128)\n\t\tx = self.relu(x)\n\t\tx = self.bn1(x)\n\t\tx = self.maxpooling1(x) #(8, 64, 64)\n\n\t\tx = self.conv3(x) #(16, 64, 64)\n\t\tx = self.relu(x)\n\t\tx = self.bn2(x)\n\t\tx = self.maxpooling1(x) #(16, 32, 32)\n\n\t\tx = self.conv4(x) #(16, 32, 32)\n\t\tx = self.relu(x)\n\t\tx = self.bn2(x)\n\t\tx = self.maxpooling2(x) #(16, 8, 8)\n\n\t\tx = x.view(x.size(0), -1) #(Batch, 16*8*8)\n\t\tx = self.dropout(x)\n\t\tx = self.fc1(x) #(Batch, 16)\n\t\tx = self.leakyrelu(x)\n\t\tx = self.dropout(x)\n\t\tx = self.fc2(x)\n\n\t\treturn x\n\n\nclass MesoInception4(nn.Module):\n\t\"\"\"\n\tPytorch Implemention of MesoInception4\n\tAuthor: Honggu Liu\n\tDate: July 7, 2019\n\t\"\"\"\n\tdef __init__(self, num_classes=2):\n\t\tsuper(MesoInception4, self).__init__()\n\t\tself.num_classes = num_classes\n\t\t#InceptionLayer1\n\t\tself.Incption1_conv1 = nn.Conv2d(3, 1, 1, padding=0, bias=False)\n\t\tself.Incption1_conv2_1 = nn.Conv2d(3, 4, 1, padding=0, bias=False)\n\t\tself.Incption1_conv2_2 = nn.Conv2d(4, 4, 3, padding=1, bias=False)\n\t\tself.Incption1_conv3_1 = nn.Conv2d(3, 4, 1, padding=0, bias=False)\n\t\tself.Incption1_conv3_2 = nn.Conv2d(4, 4, 3, padding=2, dilation=2, bias=False)\n\t\tself.Incption1_conv4_1 = nn.Conv2d(3, 2, 1, padding=0, bias=False)\n\t\tself.Incption1_conv4_2 = nn.Conv2d(2, 2, 3, padding=3, dilation=3, bias=False)\n\t\tself.Incption1_bn = nn.BatchNorm2d(11)\n\n\n\t\t#InceptionLayer2\n\t\tself.Incption2_conv1 = nn.Conv2d(11, 2, 1, padding=0, bias=False)\n\t\tself.Incption2_conv2_1 = nn.Conv2d(11, 4, 1, padding=0, bias=False)\n\t\tself.Incption2_conv2_2 = nn.Conv2d(4, 4, 3, padding=1, bias=False)\n\t\tself.Incption2_conv3_1 = nn.Conv2d(11, 4, 1, padding=0, bias=False)\n\t\tself.Incption2_conv3_2 = nn.Conv2d(4, 4, 3, padding=2, dilation=2, bias=False)\n\t\tself.Incption2_conv4_1 = nn.Conv2d(11, 2, 1, padding=0, bias=False)\n\t\tself.Incption2_conv4_2 = nn.Conv2d(2, 2, 3, padding=3, dilation=3, bias=False)\n\t\tself.Incption2_bn = nn.BatchNorm2d(12)\n\n\t\t#Normal Layer\n\t\tself.conv1 = nn.Conv2d(12, 16, 5, padding=2, bias=False)\n\t\tself.relu = nn.ReLU(inplace=True)\n\t\tself.leakyrelu = nn.LeakyReLU(0.1)\n\t\tself.bn1 = nn.BatchNorm2d(16)\n\t\tself.maxpooling1 = nn.MaxPool2d(kernel_size=(2, 2))\n\n\t\tself.conv2 = nn.Conv2d(16, 16, 5, padding=2, bias=False)\n\t\tself.maxpooling2 = nn.MaxPool2d(kernel_size=(4, 4))\n\n\t\tself.dropout = nn.Dropout2d(0.5)\n\t\tself.fc1 = nn.Linear(16*8*8, 16)\n\t\tself.fc2 = nn.Linear(16, num_classes)\n\n\n\t#InceptionLayer\n\tdef InceptionLayer1(self, input):\n\t\tx1 = self.Incption1_conv1(input)\n\t\tx2 = self.Incption1_conv2_1(input)\n\t\tx2 = self.Incption1_conv2_2(x2)\n\t\tx3 = self.Incption1_conv3_1(input)\n\t\tx3 = self.Incption1_conv3_2(x3)\n\t\tx4 = self.Incption1_conv4_1(input)\n\t\tx4 = self.Incption1_conv4_2(x4)\n\t\ty = torch.cat((x1, x2, x3, x4), 1)\n\t\ty = self.Incption1_bn(y)\n\t\ty = self.maxpooling1(y)\n\n\t\treturn y\n\n\tdef InceptionLayer2(self, input):\n\t\tx1 = self.Incption2_conv1(input)\n\t\tx2 = self.Incption2_conv2_1(input)\n\t\tx2 = self.Incption2_conv2_2(x2)\n\t\tx3 = self.Incption2_conv3_1(input)\n\t\tx3 = self.Incption2_conv3_2(x3)\n\t\tx4 = self.Incption2_conv4_1(input)\n\t\tx4 = self.Incption2_conv4_2(x4)\n\t\ty = torch.cat((x1, x2, x3, x4), 1)\n\t\ty = self.Incption2_bn(y)\n\t\ty = self.maxpooling1(y)\n\n\t\treturn y\n\n\tdef forward(self, input):\n\t\tx = self.InceptionLayer1(input) #(Batch, 11, 128, 128)\n\t\tx = self.InceptionLayer2(x) #(Batch, 12, 64, 64)\n\n\t\tx = self.conv1(x) #(Batch, 16, 64 ,64)\n\t\tx = self.relu(x)\n\t\tx = self.bn1(x)\n\t\tx = self.maxpooling1(x) #(Batch, 16, 32, 32)\n\n\t\tx = self.conv2(x) #(Batch, 16, 32, 32)\n\t\tx = self.relu(x)\n\t\tx = self.bn1(x)\n\t\tx = self.maxpooling2(x) #(Batch, 16, 8, 8)\n\n\t\tx = x.view(x.size(0), -1) #(Batch, 16*8*8)\n\t\tx = self.dropout(x)\n\t\tx = self.fc1(x) #(Batch, 16)\n\t\tx = self.leakyrelu(x)\n\t\tx = self.dropout(x)\n\t\tx = self.fc2(x)\n\n\t\treturn x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision import transforms\n\nmesonet_data_transforms = {\n    'train': transforms.Compose([\n        transforms.Resize((256, 256)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ]),\n    'val': transforms.Compose([\n        transforms.Resize((256, 256)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize((256, 256)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5] * 3, [0.5] * 3)\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\nfrom torch.utils.data import DataLoader\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport argparse\nimport os\nimport cv2\nfrom torchvision import datasets, models, transforms\n\ndef main():\n\targs = parse.parse_args()\n\tname = args.name\n\ttrain_path = args.train_path\n\tval_path = args.val_path\n\tcontinue_train = args.continue_train\n\tepoches = args.epoches\n\tbatch_size = args.batch_size\n\tmodel_name = args.model_name\n\tmodel_path = args.model_path\n\toutput_path = os.path.join('./output', name)\n\tif not os.path.exists(output_path):\n\t\tos.mkdir(output_path)\n\ttorch.backends.cudnn.benchmark=True\n\n\t#creat train and val dataloader\n\ttrain_dataset = torchvision.datasets.ImageFolder(train_path, transform=mesonet_data_transforms['train'])\n\tval_dataset = torchvision.datasets.ImageFolder(val_path, transform=mesonet_data_transforms['val'])\n\ttrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=False, num_workers=8)\n\tval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True, drop_last=False, num_workers=8)\n\ttrain_dataset_size = len(train_dataset)\n\tval_dataset_size = len(val_dataset)\n\n\n\t#Creat the model\n\tmodel = Meso4()\n\tif continue_train:\n\t\tmodel.load_state_dict(torch.load(model_path))\n\tmodel = model.cuda()\n\tcriterion = nn.CrossEntropyLoss()\n\t#optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.001)\n\toptimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08)\n\tscheduler = lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n\n\t#Train the model using multiple GPUs\n\t#model = nn.DataParallel(model)\n\n\tbest_model_wts = model.state_dict()\n\tbest_acc = 0.0\n\titeration = 0\n\tfor epoch in range(epoches):\n\t\tprint('Epoch {}/{}'.format(epoch+1, epoches))\n\t\tprint('-'*10)\n\t\tmodel=model.train()\n\t\ttrain_loss = 0.0\n\t\ttrain_corrects = 0.0\n\t\tval_loss = 0.0\n\t\tval_corrects = 0.0\n\t\tfor (image, labels) in train_loader:\n\t\t\titer_loss = 0.0\n\t\t\titer_corrects = 0.0\n\t\t\timage = image.cuda()\n\t\t\tlabels = labels.cuda()\n\t\t\toptimizer.zero_grad()\n\t\t\toutputs = model(image)\n\t\t\t_, preds = torch.max(outputs.data, 1)\n\t\t\tloss = criterion(outputs, labels)\n\t\t\tloss.backward()\n\t\t\toptimizer.step()\n\t\t\titer_loss = loss.data.item()\n\t\t\ttrain_loss += iter_loss\n\t\t\titer_corrects = torch.sum(preds == labels.data).to(torch.float32)\n\t\t\ttrain_corrects += iter_corrects\n\t\t\titeration += 1\n\t\t\tif not (iteration % 20):\n\t\t\t\tprint('iteration {} train loss: {:.4f} Acc: {:.4f}'.format(iteration, iter_loss / batch_size, iter_corrects / batch_size))\n\t\tepoch_loss = train_loss / train_dataset_size\n\t\tepoch_acc = train_corrects / train_dataset_size\n\t\tprint('epoch train loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))\n\n\t\tmodel.eval()\n\t\twith torch.no_grad():\n\t\t\tfor (image, labels) in val_loader:\n\t\t\t\timage = image.cuda()\n\t\t\t\tlabels = labels.cuda()\n\t\t\t\toutputs = model(image)\n\t\t\t\t_, preds = torch.max(outputs.data, 1)\n\t\t\t\tloss = criterion(outputs, labels)\n\t\t\t\tval_loss += loss.data.item()\n\t\t\t\tval_corrects += torch.sum(preds == labels.data).to(torch.float32)\n\t\t\tepoch_loss = val_loss / val_dataset_size\n\t\t\tepoch_acc = val_corrects / val_dataset_size\n\t\t\tprint('epoch val loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))\n\t\t\tif epoch_acc > best_acc:\n\t\t\t\tbest_acc = epoch_acc\n\t\t\t\tbest_model_wts = model.state_dict()\n\t\tscheduler.step()\n\t\tif not (epoch % 10):\n\t\t#Save the model trained with multiple gpu\n\t\t#torch.save(model.module.state_dict(), os.path.join(output_path, str(epoch) + '_' + model_name))\n\t\t\ttorch.save(model.state_dict(), os.path.join(output_path, str(epoch) + '_' + model_name))\n\tprint('Best val Acc: {:.4f}'.format(best_acc))\n\tmodel.load_state_dict(best_model_wts)\n\t#torch.save(model.module.state_dict(), os.path.join(output_path, \"best.pkl\"))\n\ttorch.save(model.state_dict(), os.path.join(output_path, \"best.pkl\"))\n\n\n\nif __name__ == '__main__':\n\tparse = argparse.ArgumentParser(\n\t\tformatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\tparse.add_argument('--name', '-n', type=str, default='Mesonet')\n\tparse.add_argument('--train_path', '-tp' , type=str, default = '../input/train_images/')\n\tparse.add_argument('--val_path', '-vp' , type=str, default = '../input/train_images/')\n\tparse.add_argument('--batch_size', '-bz', type=int, default=64)\n\tparse.add_argument('--epoches', '-e', type=int, default='50')\n\tparse.add_argument('--model_name', '-mn', type=str, default='meso4.pkl')\n\tparse.add_argument('--continue_train', type=bool, default=False)\n\tparse.add_argument('--model_path', '-mp', type=str, default='./output/Mesonet/best.pkl')\n\tmain()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# First day submit\n\nThis submission will be totally irelevant from tomorrow. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/deepfake-detection-challenge/sample_submission.csv\")\nsubmission['label'] = 0.5\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}