{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n\n## This is the Execution Script of my model (see two cells ahead!) Trained  offline using quite a large % of varied sampled data from the whole 500 GB dataset provided.\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport keras\n#from keras.models import Sequential\n\nfrom keras.models import Sequential\nfrom keras.layers import Input,LSTM,Reshape,TimeDistributed,Concatenate\nfrom keras.layers.core import Flatten, Dense, Dropout\nfrom keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D\nfrom keras.optimizers import SGD,Adadelta,Adam\nimport scipy\nfrom keras.applications  import inception_v3\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Model\nfrom keras.preprocessing.image import ImageDataGenerator\n#from keras.sequences import LSTM\nimport sklearn\nfrom scipy.io import loadmat,savemat    \nimport sys,json\nfrom sklearn.preprocessing import MinMaxScaler\nsys.path.append('../Documents/Downloads/')\nfrom sklearn.metrics import average_precision_score,classification_report\n#from tensorflow import set_random_seed\n#set_random_seed(711)\nimport os,sys,re,time\nfrom collections import defaultdict\nimport imageio \nfrom PIL import Image\nimport cv2\nfrom sklearn import utils\nfrom sklearn.model_selection import train_test_split\nfrom collections import OrderedDict\n\nimport psutil\nprocess = psutil.Process(os.getpid())\n\n\nos.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\n\n!pip install '/kaggle/input/dlibpkg/dlib-19.19.0'\n!pip install '/kaggle/input/face-recognition/face_recognition_models-0.3.0/face_recognition_models-0.3.0'\n!pip install '/kaggle/input/facerecognition-123/face_recognition-1.2.3-py2.py3-none-any.whl'\n!pip install '/kaggle/input/imageio-ffmpeg/imageio_ffmpeg-0.3.0-py3-none-manylinux2010_x86_64.whl'\n\nimport scipy.misc\nimport PIL\nimport dlib\nimport face_recognition\nimport time\n\ndlib.DLIB_USE_CUDA=True\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\nclass Video:\n    def __init__(self, path):\n        self.path = path\n        self.container = imageio.get_reader(path, 'ffmpeg')\n        self.length = self.container.count_frames()\n        self.fps = self.container.get_meta_data()['fps']\n    \n    def init_head(self):\n        self.container.set_image_index(0)\n    \n    def next_frame(self):\n        self.container.get_next_data()\n    \n    def get(self, key):\n        return self.container.get_data(key)\n    \n    def __call__(self, key):\n        return self.get(key)\n    \n    def __len__(self):\n        return self.length","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## ML Model , Inception(LSTM) - i.e a spatio - temporal model. see -  https://engineering.purdue.edu/~dgueraco/content/deepfake.pdf\n\nseq_len=9\nimg_dimens=299\n\n\nimport h5py    \nimport numpy as np    \nf1 = h5py.File('/kaggle/input/inceptionv3/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5','r') \n\ndef predict_fakes(Xtest):\n\n    img_input = Input(shape=(seq_len,img_dimens,img_dimens,3))\n    # num_classes=1\n\n    batch_size=2\n    sequence_len = 9\n    num_lstm_units = num_features_per_Spatialframe_model = 2048\n    num_classes=2\n    img_width=299\n    img_height=299\n\n    ## I implemented an adapted architecture from th epaper - Delp & Guerra\n    # It combines the frasmewise spatial fetures using the famous inception_v3 model. Then it feeds this into an LSTM Recurrant NNet to learn the \n    ## temporal behaviour \"on top\" of the spatial. Or more accurately it combines each spatially learned model per frame across a sequence of frame to represent the video\n    \n    feature_out_1 = TimeDistributed(inception_v3.InceptionV3(include_top=False,pooling='max',weights='imagenet'))(img_input)\n    z = (LSTM(num_lstm_units,input_shape=(sequence_len,num_features_per_Spatialframe_model),return_sequences=False, recurrent_dropout = 0.5, dropout = 0.5))(feature_out_1)\n    x = (Dense(512,activation='relu'))(z)\n    x = (Dropout(0.5))(x)\n    out = (Dense(num_classes,activation='softmax'))(x)    \n\n    Xtest=Xtest.reshape(-1, seq_len,img_dimens,img_dimens,3)\n    #Xtest=Xtest.astype('float') / 255.0\n    Xtest = Xtest / 255.0\n    print('Xtest shape =' + str(Xtest.shape))\n\n    model = Model(img_input,out)\n hdf5\")\n    model.load_weights(\"/kaggle/input/inceptv3-lstmworkinglocallyontestset/ExtUtubDtaDelpGEnd2end_inceptV3_numfrmsPerSeq_9lstmUntsBalncd600SmplesNegnPos_With255NormlzePreProcRedoNum_clsses_1_2048_learn_rate_1e-05_dropot_usd_1_weight_usd_0_batch_size_2_epoch_num_12_.hdf5\")\n\n    \n    te_pr = model.predict(Xtest, batch_size=batch_size, verbose=1)\n    print('in model code and te_pr = ' + str(te_pr))\n    return te_pr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom keras.applications import inception_v3\n\npaths = []\npreds = []\nmargin = 0.2\n#!pip install '/kaggle/input/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'\nfilenames_init = os.listdir('/kaggle/input/deepfake-detection-challenge/test_videos/')\nfilenames = filenames_init\n#print('no. of test videos = ' + str(len(filenames)))\n#print('all video names = ' + str(filenames))\n\nreal_eg_count = 0\n\nbad_imageCunt = 0\nvid_counter = 0\nframe_cutoff = 9 # (== seq_len)\n\nnum_chunks = 50\n#np.set_printoptions(threshold  = float(\"inf\"))\nfor i in range(0,int(num_chunks/2)):\n    \n    step_size = int(len(filenames)/num_chunks)\n    \n    #print('step size = '+ str(step_size))\n\n    featurPerVideoDict = {}\n    allVidsFeatures    = {}\n    #print('doing chunk -- ' + str(i*step_size) + ' : ' + str((i+1)*step_size))\n    for filename in filenames[i*step_size:(i+1)*step_size]:\n        #if filename[-4:] != '.mp4':\n        #    continue\n        #print(os.path.join(dirname, filename))    \n        full_path = os.path.join('/kaggle/input/deepfake-detection-challenge/test_videos/', filename)\n        paths.append(filename)\n\n        #print('Processing video = ' + str(filename))\n\n        #try:\n\n        video_capture = Video(os.path.join(full_path)) # + '/', str(vid)))\n        #global countVid\n        #print('processing  Video : ' + str(filename)  + ' which is no. ' + str(vid_counter)) # has frame rate ' + str(video_capture.fps)  + ' and label = ' + str(json1_data[name]['label']))\n        vid_counter+=1\n\n        # Initialize variables\n        face_locations = []\n        y_img=[]\n        #os.mkdir(cropedTrainAllImgDir)\n        #except Exception as inst:\n        keys = filename[:-4]\n        #print(\"Error occured at doing marker stuff of image file  ...\")\n        #print(inst)    \n        size = (299,299)\n        frame_counter = 0\n        save_interval = 1\n        featureMat = []\n        last_key = \"pdufsewrec\"\n        for i in range(100,video_capture.__len__()-40,save_interval):\n\n            #print('back in for at iteration -- ' + str(i))\n\n            #print('just into try   ...  frame_counter = ' + str(frame_counter))\n            if frame_counter > 12 :\n                break\n\n            frame_counter += 1\n            #print(' processing  frame no ' + str(counter)  + '... :)')\n            # Grab a single frame of video\n            frame = video_capture.get(i)\n            # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)\n            rgb_frame = frame[:, :, ::-1]\n\n            # Find all the faces in the current frame of video\n            face_positions = face_recognition.face_locations(rgb_frame)\n            if face_positions is not None:\n\n                #print('got face_position , so in outer if here ..  face_positions = ' + str(face_positions))\n                #face_positions = list(face_positions)\n                # Display the results\n                for face_position in face_positions:\n                    # Draw a box around the face\n                    #print('in face_positions for loop  ...  margin = ' + str(margin))\n                    #print('in a face of face_position -- face_position[2] =  ' + str(face_position[2]))\n                    offset = round(margin * (face_position[2] - face_position[0]))\n                    y0 = max(face_position[0] - offset, 0)\n                    x1 = min(face_position[1] + offset, rgb_frame.shape[1])\n                    y1 = min(face_position[2] + offset, rgb_frame.shape[0])\n                    x0 = max(face_position[3] - offset, 0)\n                    face = rgb_frame[y0:y1,x0:x1]\n\n                    #inp = cv2.resize(face,(size,size))\n                    #IMAGES.append(np.expand_dims(inp,axis=0))   \n                    #if face :\n\n                    #print('in a face of face_position ...  face = ' + str(face))\n\n                    im = Image.fromarray(face) \n                    #print('read image from array of face = ')\n                    input_img = np.array(im.resize(size, Image.BICUBIC))\n                    #print('RESIZED image from array of face = ')\n                    ## Change pre-processing to subtract the 3 channel means instead, prior to inputting to Inception \n\n\n                    ## usually a preprocessing tchnique prior to running inception but not in Delp's paper\n                    #input_img = inception_v3.preprocess_input(input_img.astype(np.float32))\n                    input_img = input_img.astype(np.float32)\n                    #print('CONVERTEd RESIZED image to a float')\n                    featureMat.append(np.expand_dims(np.array(input_img),axis=0))\n                    #print('Done appended to featureMat...t')\n\n        #print('size of featureMat  = ' + str(len(featureMat)))\n        #print('about to do appendoing of all its feature data into dict @ key = ' + str(keys) + ' ...')\n        if featureMat:\n            featurPerVideoDict[keys] = np.array(featureMat)\n            allImgsPerVidMat = np.concatenate(featureMat,axis=0)\n            #last_key = keys\n\n        else:\n            #print('Encountered an empty sequnce of images for video => ' + str(keys))\n            featurPerVideoDict[keys] = np.random.random_sample((1,frame_cutoff,299,299,3))\n            allImgsPerVidMat = np.concatenate(featurPerVideoDict[keys],axis=0)\n            #preds.append(0.5)\n            bad_imageCunt += 1\n            #print('no. videos could not find a face = ' + str(bad_imageCunt))\n\n        #print('size of allImgsPerVidMat  = ' + str(allImgsPerVidMat.shape))        \n        allVidsFeatures[keys] = np.concatenate(np.expand_dims(allImgsPerVidMat,axis=0) ,axis=0) \n                            #imageio.imwrite(cropedTrainAllImgDir + '/' + name[:-4] + '_' + str(counter) + '.jpg', face)\n            \n\n    print('Total number of videos  = ' + str(len(allVidsFeatures)))\n\n    regurlySpacedVidsONlyDict = []\n    final_data=[]\n    properConcatdLSTMFetaurDataDict = allVidsFeatures\n    key_counter = 0\n    for indx in allVidsFeatures.keys():\n\n        #print(' in padding code video = ' + str(indx) + ' i.e video number => ' + str(key_counter))\n\n        if properConcatdLSTMFetaurDataDict[indx].shape[0] != 0 and properConcatdLSTMFetaurDataDict[indx].shape[0] < frame_cutoff :\n            firstBlock = properConcatdLSTMFetaurDataDict[indx][:properConcatdLSTMFetaurDataDict[indx].shape[0],:]  \n            arr_new = np.expand_dims(properConcatdLSTMFetaurDataDict[indx][properConcatdLSTMFetaurDataDict[indx].shape[0]-1],axis=0) \n            tiled_arr = np.repeat(arr_new,(frame_cutoff - properConcatdLSTMFetaurDataDict[indx].shape[0]),axis=0)\n            padded_arr = np.concatenate((firstBlock ,tiled_arr),axis=0)\n            regurlySpacedVidsONlyDict.append(padded_arr)\n\n        elif properConcatdLSTMFetaurDataDict[indx].shape[0] == 0 :\n            regurlySpacedVidsONlyDict.append(np.zeros((frame_cutoff,2048)))\n\n        else:    \n            regurlySpacedVidsONlyDict.append(properConcatdLSTMFetaurDataDict[indx][:frame_cutoff,:])\n\n        \n        #final_data.append(np.array(allVidsFeatures))\n        final_data.append(np.expand_dims(np.array(regurlySpacedVidsONlyDict[key_counter]),axis=0))  \n        #key_counter+=1 \n        #print('Processed video no.' + str(key_counter))\n\n    final_data=np.concatenate(final_data,axis=0)\n    \n    #final_data_np = np.concatenate(final_data,axis=0)             \n    probs = predict_fakes(np.array(final_data)) \n    \n    for prob in probs:\n        preds.append(prob[1])\n        \n        if prob[1] <= 0.5:\n            real_eg_count += 1\n            \n    print('memory usage = ' + str(process.memory_info().rss))        \n            \n    print('real_eg_count =  ' + str(real_eg_count))        \n\n    \nfor i in range(int(num_chunks/2)+1,num_chunks):\n    \n    step_size = int(len(filenames)/num_chunks)\n \n    featurPerVideoDict = {}\n    allVidsFeatures    = {}\n    #print('doing chunk -- ' + str(i*step_size) + ' : ' + str((i+1)*step_size))\n    for filename in filenames[i*step_size:(i+1)*step_size]:\n  \n        full_path = os.path.join('/kaggle/input/deepfake-detection-challenge/test_videos/', filename)\n        paths.append(filename)\n\n        video_capture = Video(os.path.join(full_path)) # + '/', str(vid)))\n        #global countVid\n        #print('processing  Video : ' + str(filename)  + ' which is no. ' + str(vid_counter)) # has frame rate ' + str(video_capture.fps)  + ' and label = ' + str(json1_data[name]['label']))\n        vid_counter+=1\n\n        # Initialize variables\n        face_locations = []\n        y_img=[]\n        #os.mkdir(cropedTrainAllImgDir)\n        #except Exception as inst:\n        keys = filename[:-4]\n        #print(\"Error occured at doing marker stuff of image file  ...\")\n        #print(inst)    \n        size = (299,299)\n        frame_counter = 0\n        save_interval = 1\n        featureMat = []\n        last_key = \"pdufsewrec\"\n        for i in range(100,video_capture.__len__()-40,save_interval):\n\n            #print('back in for at iteration -- ' + str(i))\n\n            #print('just into try   ...  frame_counter = ' + str(frame_counter))\n            if frame_counter > 12 :\n                break\n\n            frame_counter += 1\n            #print(' processing  frame no ' + str(counter)  + '... :)')\n            # Grab a single frame of video\n            frame = video_capture.get(i)\n            # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)\n            rgb_frame = frame[:, :, ::-1]\n\n            # Find all the faces in the current frame of video\n            face_positions = face_recognition.face_locations(rgb_frame)\n            if face_positions is not None:\n\n                #print('got face_position , so in outer if here ..  face_positions = ' + str(face_positions))\n                #face_positions = list(face_positions)\n                # Display the results\n                for face_position in face_positions:\n                    # Draw a box around the face\n                    #print('in face_positions for loop  ...  margin = ' + str(margin))\n                    #print('in a face of face_position -- face_position[2] =  ' + str(face_position[2]))\n                    offset = round(margin * (face_position[2] - face_position[0]))\n                    y0 = max(face_position[0] - offset, 0)\n                    x1 = min(face_position[1] + offset, rgb_frame.shape[1])\n                    y1 = min(face_position[2] + offset, rgb_frame.shape[0])\n                    x0 = max(face_position[3] - offset, 0)\n                    face = rgb_frame[y0:y1,x0:x1]\n\n                    #inp = cv2.resize(face,(size,size))\n                    #IMAGES.append(np.expand_dims(inp,axis=0))   \n                    #if face :\n\n                    #print('in a face of face_position ...  face = ' + str(face))\n\n                    im = Image.fromarray(face) \n                    #print('read image from array of face = ')\n                    input_img = np.array(im.resize(size, Image.BICUBIC))\n                    #print('RESIZED image from array of face = ')\n                    ## Change pre-processing to subtract the 3 channel means instead, prior to inputting to Inception \n\n                    ## usually a preprocessing tchnique prior to running inception but not in Delp's paper\n                    #input_img = inception_v3.preprocess_input(input_img.astype(np.float32))\n                    input_img = input_img.astype(np.float32)\n                    #print('CONVERTEd RESIZED image to a float')\n                    featureMat.append(np.expand_dims(np.array(input_img),axis=0))\n                    #print('Done appended to featureMat...t')\n\n        #print('size of featureMat  = ' + str(len(featureMat)))\n        #print('about to do appendoing of all its feature data into dict @ key = ' + str(keys) + ' ...')\n        if featureMat:\n            featurPerVideoDict[keys] = np.array(featureMat)\n            allImgsPerVidMat = np.concatenate(featureMat,axis=0)\n            #last_key = keys\n\n        else:\n            #print('Encountered an empty sequnce of images for video => ' + str(keys))\n            featurPerVideoDict[keys] = np.random.random_sample((1,frame_cutoff,299,299,3))\n            allImgsPerVidMat = np.concatenate(featurPerVideoDict[keys],axis=0)\n            #preds.append(0.5)\n            bad_imageCunt += 1\n            #print('no. videos could not find a face = ' + str(bad_imageCunt))\n\n        #print('size of allImgsPerVidMat  = ' + str(allImgsPerVidMat.shape))        \n        allVidsFeatures[keys] = np.concatenate(np.expand_dims(allImgsPerVidMat,axis=0) ,axis=0) \n                            #imageio.imwrite(cropedTrainAllImgDir + '/' + name[:-4] + '_' + str(counter) + '.jpg', face)\n            \n\n    print('Total number of videos  = ' + str(len(allVidsFeatures)))\n\n    regurlySpacedVidsONlyDict = []\n    final_data=[]\n    properConcatdLSTMFetaurDataDict = allVidsFeatures\n    key_counter = 0\n     #sequence_len\n    #unused parameter for now\n    #vid_frame_offset = 120\n    for indx in allVidsFeatures.keys():\n\n        #print(' in padding code video = ' + str(indx) + ' i.e video number => ' + str(key_counter))\n\n        if properConcatdLSTMFetaurDataDict[indx].shape[0] != 0 and properConcatdLSTMFetaurDataDict[indx].shape[0] < frame_cutoff :\n            firstBlock = properConcatdLSTMFetaurDataDict[indx][:properConcatdLSTMFetaurDataDict[indx].shape[0],:]  \n            arr_new = np.expand_dims(properConcatdLSTMFetaurDataDict[indx][properConcatdLSTMFetaurDataDict[indx].shape[0]-1],axis=0) \n            tiled_arr = np.repeat(arr_new,(frame_cutoff - properConcatdLSTMFetaurDataDict[indx].shape[0]),axis=0)\n            padded_arr = np.concatenate((firstBlock ,tiled_arr),axis=0)\n            regurlySpacedVidsONlyDict.append(padded_arr)\n\n        elif properConcatdLSTMFetaurDataDict[indx].shape[0] == 0 :\n            regurlySpacedVidsONlyDict.append(np.zeros((frame_cutoff,2048)))\n\n        else:    \n            regurlySpacedVidsONlyDict.append(properConcatdLSTMFetaurDataDict[indx][:frame_cutoff,:])\n\n        \n        #final_data.append(np.array(allVidsFeatures))\n        final_data.append(np.expand_dims(np.array(regurlySpacedVidsONlyDict[key_counter]),axis=0))  \n        #key_counter+=1 \n        #print('Processed video no.' + str(key_counter))\n\n    final_data=np.concatenate(final_data,axis=0)\n    \n    #final_data_np = np.concatenate(final_data,axis=0)             \n    probs = predict_fakes(np.array(final_data)) \n    \n    for prob in probs:\n        preds.append(prob[1])\n        \n        if prob[1] <= 0.5:\n            real_eg_count += 1\n            \n    print('In second batch of chunks loop  -- memory usage = ' + str(process.memory_info().rss))        \n            \n    print('real_eg_count =  ' + str(real_eg_count))        \n    \n    \n    \n    \n    \n    \n# #print('video: ' + str(indx) + ' probability: ' + str(prob))\nres = pd.DataFrame({\n'filename': paths,\n'label': preds,\n})    \n\nprint('paths =' + str(paths))\nprint('preds =' + str(preds))\n\n\n\n\nres.sort_values(by='filename', ascending=True, inplace=True)\n\nprint(res)\n#print(res.filename)\nres.to_csv('submission.csv', index=False)                \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}