{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PARAMS_VIDEO={\n    'shape':(256,256),\n    'is_face':True,\n    'is_first_face':False,\n    'is_first':False,\n    'on_each':30,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/efficientnet/efficientnet-1.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.keras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport sys\nimport pandas as pd\nfrom skimage.metrics import mean_squared_error as mse\nfrom skimage.metrics import structural_similarity as ssim\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR='/kaggle/input/deepfake-detection-challenge/'\nTRAIN_DIR=f'{DATA_DIR}train_sample_videos/'\nTEST_DIR=f'{DATA_DIR}test_videos/'\nSUB_DIR=f'{DATA_DIR}sample_submission.csv'\n\nFACE_DETECTION_FOLDER=f'/kaggle/input/haar-cascades-for-face-detection/'\nFACENET_DIR=f'/kaggle/input/facenet-keras/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ObjectDetector():\n    '''\n    Class for Object Detection\n    '''\n    def __init__(self,object_cascade_path):\n        '''\n        param: object_cascade_path - path for the *.xml defining the parameters for {face, eye, smile, profile}\n        detection algorithm\n        source of the haarcascade resource is: https://github.com/opencv/opencv/tree/master/data/haarcascades\n        '''\n\n        self.objectCascade=cv2.CascadeClassifier(object_cascade_path)\n\n\n    def detect(self, image, scale_factor=1.3,\n               min_neighbors=5,\n               min_size=(20,20)):\n        '''\n        Function return rectangle coordinates of object for given image\n        param: image - image to process\n        param: scale_factor - scale factor used for object detection\n        param: min_neighbors - minimum number of parameters considered during object detection\n        param: min_size - minimum size of bounding box for object detected\n        '''\n        rects=self.objectCascade.detectMultiScale(image,\n                                                scaleFactor=scale_factor,\n                                                minNeighbors=min_neighbors,\n                                                minSize=min_size)\n        return rects","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nf=open(TRAIN_DIR+'metadata.json')\ntrain_labels=json.loads(f.read())\n\ndef json2pd(jdata):\n    res=[]\n    for k in jdata.keys():\n        jdata[k]['name']=k\n        res.append(jdata[k])\n    return pd.DataFrame(res)\n\ntrain_labels=json2pd(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport sys\n\ndef get_obj_size(obj):\n    marked = {id(obj)}\n    obj_q = [obj]\n    sz = 0\n\n    while obj_q:\n        sz += sum(map(sys.getsizeof, obj_q))\n\n        # Lookup all the object referred to by the object in obj_q.\n        # See: https://docs.python.org/3.7/library/gc.html#gc.get_referents\n        all_refr = ((id(o), o) for o in gc.get_referents(*obj_q))\n\n        # Filter object that are already marked.\n        # Using dict notation will prevent repeated objects.\n        new_refr = {o_id: o for o_id, o in all_refr if o_id not in marked and not isinstance(o, type)}\n\n        # The new obj_q will be the ones that were not marked,\n        # and we will update marked with their ids so we will\n        # not traverse them again.\n        obj_q = new_refr.values()\n        marked.update(new_refr.keys())\n\n    return sz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mtcnn import MTCNN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"face_detect=MTCNN()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir(face_detect)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def detect_objects(image, scale_factor=1.3, min_neighbors=5, min_size=(50,50)):\n    \n    image_gray=cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n\n\n    eyes=eye_detector.detect(image_gray,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=(int(min_size[0]/2), int(min_size[1]/2)))\n\n    for x, y, w, h in eyes:\n        #detected eyes shown in color image\n        cv2.circle(image,(int(x+w/2),int(y+h/2)),(int((w + h)/4)),(0, 0,255),3)\n \n    # deactivated due to many false positive\n    #smiles=sd.detect(image_gray,\n    #               scale_factor=scale_factor,\n    #               min_neighbors=min_neighbors,\n    #               min_size=(int(min_size[0]/2), int(min_size[1]/2)))\n\n    #for x, y, w, h in smiles:\n    #    #detected smiles shown in color image\n    #    cv.rectangle(image,(x,y),(x+w, y+h),(0, 0,255),3)\n\n\n    profiles=profile_detector.detect(image_gray,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=min_size)\n\n    for x, y, w, h in profiles:\n        #detected profiles shown in color image\n        cv2.rectangle(image,(x,y),(x+w, y+h),(255, 0,0),3)\n\n    faces=front_detector.detect(image_gray,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=min_size)\n\n    for x, y, w, h in faces:\n        #detected faces shown in color image\n        cv2.rectangle(image,(x,y),(x+w, y+h),(0, 255,0),3)\n\n    # image\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    ax.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_face(image, scale_factor=2,preshape=(512,512), min_neighbors=5, min_size=(30,30),target_shape=(256,256)):\n    \n    image_gray=cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n\n    if(preshape):\n        image_gray=cv2.resize(image_gray,preshape)\n\n    profiles=profile_detector.detect(image_gray,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=min_size)\n\n    faces=front_detector.detect(image_gray,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=min_size)\n    res_faces=[]\n    if(len(profiles)!=0 or len(faces)!=0):\n        for p in profiles:\n            im=image[p[1]:p[1]+p[3],p[0]:p[0]+p[2]]\n            if(target_shape):\n                im=cv2.resize(im,target_shape, interpolation = cv2.INTER_AREA)\n            res_faces.append(im)\n        for p in faces:\n            im=image[p[1]:p[1]+p[3],p[0]:p[0]+p[2]]\n            if(target_shape):\n                im=cv2.resize(im,target_shape, interpolation = cv2.INTER_AREA)\n            res_faces.append(im)\n    filtered_faces=[]        \n    if(len(res_faces)!=0):\n        for f in res_faces:\n            eyes=eye_detector.detect(f,\n                   scale_factor=scale_factor,\n                   min_neighbors=min_neighbors,\n                   min_size=(int(min_size[0]/2), int(min_size[1]/2)))\n            if(len(eyes)!=0):\n                filtered_faces.append(f)\n    #features=model.predict(np.array(filtered_faces))\n    return filtered_faces","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_face(image, scale_factor=2,preshape=(256,256), target_shape=(256,256)):\n\n    original_shape=image.shape\n    if(preshape):\n        scale_y=original_shape[0]/preshape[0]\n        scale_x=original_shape[1]/preshape[1]\n        reshape_image=cv2.resize(image,preshape)\n    else:\n        scale_y=1\n        scale_x=1\n        reshape_image=image\n    \n    par_faces=face_detect.detect_faces(reshape_image)\n    \n    faces=[]\n    \n    \n    for p in par_faces:\n        width=int(p['box'][2]*scale_x)\n        height=int(p['box'][3]*scale_y)\n        new_width=int(width*scale_factor)\n        new_height=int(height*scale_factor)\n        x=int(p['box'][0]*scale_x)-(new_width-width)//2\n        y=int(p['box'][1]*scale_y)-(new_height-height)//2\n        if(x<0):\n            x=0\n        if(y<0):\n            y=0\n        \n        try:\n            if(x+width<image.shape[1] and y+height<image.shape[0] and new_width!=0 and new_height!=0):\n                face=image[y:y+new_width,x:x+new_width]\n                if(target_shape):\n                    face=cv2.resize(face,target_shape)\n                faces.append(face)\n            #else:\n            #    print('error',p)\n        except:\n            print('error',p)\n\n    return faces","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VideoReader():\n    def __init__(self,video_path,shape=None,is_gray=False,is_face=False,is_dif=False,is_first=False,is_first_face=False,on_each=1,offset=0):\n        self.video_path=video_path\n        self.codec=cv2.VideoCapture(self.video_path)\n        self.shape=shape\n        \n        self.stop_read=False\n        \n        self.cur_ind=0\n        \n        self.is_gray=is_gray\n        self.is_face=is_face\n        self.is_dif=is_dif\n        self.is_first=is_first\n        self.is_first_face=is_first_face\n        self.on_each=on_each\n        self.offset=offset\n        \n        self.is_error=False\n        \n        self.frames=[]\n        self.frames_vector=[]\n        \n        \n        self.dif_frames=[]\n        self.first_frame=None\n        self.last_frame=None\n        \n        self.faces=[]\n        self.miss_faces=[]\n        \n        \n        self.params={}\n        \n        self.dif_params={\n            'mse':[],\n            'ssim':[],\n        }\n    \n    def get_video(self):\n        while(self.codec.isOpened() and not self.stop_read):\n            self.on_frame()\n        #self.get_params()\n        \n    def on_frame(self):\n        ret, frame = self.codec.read()\n        if (ret==True and self.cur_ind>=self.offset):\n            if(type(frame)!=type(None) ):\n                if((self.cur_ind+1)%self.on_each==0):\n                    #frame=self.preprocess_frame(frame)\n                    if(self.is_face):\n                        self.get_face(frame)\n                    if(self.is_first):\n                        self.stop_read=True\n                    self.frames.append(frame)\n        else:\n            self.stop_read=True\n        self.cur_ind+=1\n            \n    def get_face(self,frame):\n        faces=get_face(frame,preshape=None,target_shape=self.shape)\n        if(len(faces)>0):\n            self.faces.append(faces)\n            if(self.is_first_face):\n                self.stop_read=True\n        else:\n            self.miss_faces.append(self.cur_ind)\n        \n            \n    \n    def get_params(self):\n        self.params['fname']=self.video_path\n        self.params['length']=len(self.frames)\n        self.params['size_obj']=get_obj_size(self)\n        \n    def preprocess_frame(self,frame):\n        if(self.shape):\n            frame=cv2.resize(frame,(self.shape[1],self.shape[0]))\n        if(self.is_gray):\n            frame=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)\n        return frame\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VideoGroupReader():\n    def __init__(self,original_file,list_fakes=[]):\n        self.original_file=original_file\n        self.list_fakes=list_fakes\n    def dif_videos(self):\n        for i in range(len(self.list_fakes)):\n            fake=self.list_fakes[i]\n            vr=VideoReader(fake,is_face=True,is_first_face=False,on_each=30)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"frontal_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_frontalface_default.xml')\neye_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_eye.xml')\nprofile_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_profileface.xml')\nsmile_cascade_path= os.path.join(FACE_DETECTION_FOLDER,'haarcascade_smile.xml')\n\nprint(eye_cascade_path)\n#Detector object created\n# frontal face\nfront_detector=ObjectDetector(frontal_cascade_path)\n# eye\neye_detector=ObjectDetector(eye_cascade_path)\n# profile face\nprofile_detector=ObjectDetector(profile_cascade_path)\n# smile\nsmile_detector=ObjectDetector(smile_cascade_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_files=os.listdir(TRAIN_DIR)\ntest_files=os.listdir(TEST_DIR)\nfor k in train_files:\n    if('.json' in k):\n        json_file=k\n        train_files.remove(k)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_files),len(test_files),json_file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nf=open(TRAIN_DIR+json_file)\ntrain_label=json.loads(f.read())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_label['dkzvdrzcnr.mp4']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import Sequence\nfrom skimage.measure import compare_ssim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"train_data={}\nbad_names=[]\nfor i in tqdm(range(len(train_files))):\n    video = VideoReader(TRAIN_DIR+train_files[i],shape=(256,256),is_face=True,is_first_face=True,is_first=False,on_each=30)\n    video.get_video()\n    if(len(video.faces)>0):\n        train_data[train_files[i]]=video.faces[0]\n    else:\n        bad_names.append(train_files[i])\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir face_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"#train_data_x=[]\ntrain_data_y=[]\nbad_names=[]\ntrain_decode_data={}\nfor i in tqdm(range(len(train_files))):\n    video = VideoReader(TRAIN_DIR+train_files[i],shape=PARAMS_VIDEO['shape'],is_face=PARAMS_VIDEO['is_face'],is_first_face=PARAMS_VIDEO['is_first_face'],is_first=PARAMS_VIDEO['is_first'],on_each=PARAMS_VIDEO['on_each'])\n    video.get_video()\n    label=int(train_label[train_files[i]]['label']=='FAKE')\n    start_ind=len(train_data_y)\n    for j in range(len(video.faces)):\n        if(len(video.faces[j])==1):\n            \n            #train_data_x+=video.faces[j]\n            for t in range(len(video.faces[j])):\n                #print(video.faces[j][t].shape)\n                cv2.imwrite(f'face_data/{train_files[i]}_{j}_{t}_{label}.png',video.faces[j][t])\n            train_data_y+=[label] * (len(video.faces[j]))\n            \n            train_decode_data[train_files[i]]=(start_ind,len(train_data_y))\n            \n        #print(i,len(train_data_x),len(train_data_y))\n    else:\n        bad_names.append(train_files[i])\"\"\"\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_data_x=[]\ntrain_data_y=[]\nbad_names=[]\ntrain_decode_data={}\nfor i in tqdm(range(len(train_files))):\n    video = VideoReader(TRAIN_DIR+train_files[i],shape=PARAMS_VIDEO['shape'],is_face=PARAMS_VIDEO['is_face'],is_first_face=PARAMS_VIDEO['is_first_face'],is_first=PARAMS_VIDEO['is_first'],on_each=PARAMS_VIDEO['on_each'])\n    video.get_video()\n    label=int(train_label[train_files[i]]['label']=='FAKE')\n    start_ind=len(train_data_y)\n    \n    for j in range(len(video.faces)):\n        if(len(video.faces[j])==1):\n            \n            #train_data_x+=video.faces[j]\n            for t in range(len(video.faces[j])):\n                #print(video.faces[j][t].shape)\n                fname=f'face_data/{train_files[i]}_{j}_{t}_{label}.png'\n                cv2.imwrite(fname,video.faces[j][t])\n                if(train_files[i] in train_decode_data.keys()):\n                    train_decode_data[train_files[i]].append(fname)\n                else:\n                    train_decode_data[train_files[i]]=[fname]\n            #train_data_y+=[label] * (len(video.faces[j]))\n    if(len(video.faces)==0):\n        bad_names.append(train_files[i])\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bad_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"test_data={}\nbad_names_test=[]\nfor i in tqdm(range(len(test_files))):\n    video = VideoReader(TEST_DIR+test_files[i])\n    face=video.get_first_face()\n    test_data[test_files[i]]=face\n    if(np.max(face)!=0):\n        bad_names_test.append(test_files[i])\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import skimage.filters\nclass DataGeneratorFull(Sequence):\n    'Generates data for Keras'\n    def __init__(self, files,data,jdata=None,len_frames=300,batch_size=4,shuffle=True,dim=(1024,1024),channels=3,mode='fit'):\n        self.dim = dim\n        self.files=files\n        self.data=data\n        self.jdata=jdata\n        self.len_frames=len_frames\n        self.batch_size=batch_size\n        self.shuffle=shuffle\n        self.dim=dim\n        self.channels=channels\n        self.mode=mode\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int((len(self.files) / self.batch_size))\n\n    def __getitem__(self, index):\n\n        batch_files = self.files[index*self.batch_size:(index+1)*self.batch_size]\n        \n        X = self.__generate_X(batch_files)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(batch_files)\n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n        else:\n            raise AttributeError('The parameter mode should be set to \"fit\" or \"predict\".')\n        \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.files))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n    \n    def __generate_X(self, batch_files):\n        x=np.zeros((self.batch_size,*self.dim,3),dtype=np.float32)\n        for i in range(len(batch_files)):\n            #if(self.mode=='fit'):\n            face = self.data[batch_files[i]][0]\n            #else:\n                #face = VideoReader(TEST_DIR+batch_files[i],shape=(256,256))\n            #print(face)\n            x[i,:,:]=face/255\n        return x\n    \n    def __generate_y(self, batch_files):\n        y=np.zeros((self.batch_size,1))\n        for i in range(len(batch_files)):\n            val=self.jdata[batch_files[i]]['label']=='FAKE'\n            y[i]=val#keras.utils.to_categorical(val,2)\n            #print(val)\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import skimage.filters\nclass DataGeneratorFull(Sequence):\n    'Generates data for Keras'\n    def __init__(self, data_files,data_dir='face_data',len_frames=300,batch_size=4,shuffle=True,dim=(1024,1024),channels=3,mode='fit'):\n        self.dim = dim\n        self.data_files=data_files\n        #print(len(self.data_files))\n        self.len_frames=len_frames\n        self.batch_size=batch_size\n        self.shuffle=shuffle\n        self.dim=dim\n        self.channels=channels\n        self.mode=mode\n        self.data_dir=data_dir\n        self.indexes = np.arange(len(self.data_files))\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return (len(self.data_files) // self.batch_size)\n\n    def __getitem__(self, index):\n\n        batch_files = self.data_files[index*self.batch_size:(index+1)*self.batch_size]\n        #print(index,len(batch_files),self.batch_size)\n        X = self.__generate_X(batch_files)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(batch_files)\n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n        else:\n            raise AttributeError('The parameter mode should be set to \"fit\" or \"predict\".')\n        \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.data_files))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n    \n    def __generate_X(self, batch_files):\n        x=np.zeros((self.batch_size,*self.dim,3),dtype=np.float32)\n        #print('len=',len(batch_files))\n        for i in range(len(batch_files)):\n            face = cv2.imread(f'{self.data_dir}/'+batch_files[i])\n            #print('max=',np.max(face))\n            x[i,]=face/255\n        return x\n    \n    def __generate_y(self, batch_files):\n        y=np.zeros((self.batch_size,2))\n        for i in range(len(batch_files)):\n            #val=self.jdata[batch_files[i]]['label']=='FAKE'\n            indxs=train_decode_data[batch_files[i].split('_')[0]]\n            label=train_data_y[indxs[0]]\n            y[i]=keras.utils.to_categorical(label,2)\n            #print(val)\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import skimage.filters\nclass DataGeneratorFull(Sequence):\n    'Generates data for Keras'\n    def __init__(self, data_files,data_dir='face_data',len_frames=300,batch_size=4,shuffle=True,dim=(1024,1024),channels=3,mode='fit'):\n        self.dim = dim\n        self.data_files=data_files\n        #print(len(self.data_files))\n        self.len_frames=len_frames\n        self.batch_size=batch_size\n        self.shuffle=shuffle\n        self.dim=dim\n        self.channels=channels\n        self.mode=mode\n        self.data_dir=data_dir\n        self.indexes = np.arange(len(self.data_files))\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return (len(self.data_files) // self.batch_size)\n\n    def __getitem__(self, index):\n\n        batch_files = self.data_files[index*self.batch_size:(index+1)*self.batch_size]\n        #print(index,len(batch_files),self.batch_size)\n        X = self.__generate_X(batch_files)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(batch_files)\n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n        else:\n            raise AttributeError('The parameter mode should be set to \"fit\" or \"predict\".')\n        \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.data_files))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n    \n    def __generate_X(self, batch_files):\n        #x=np.zeros((self.batch_size,*self.dim,3),dtype=np.float32)\n        #print('len=',len(batch_files))\n        x=[]\n        for i in range(len(batch_files)):\n            cur_files=train_decode_data[batch_files[i]]\n            for c in cur_files:\n                #print(c)\n                face = cv2.imread(c)\n                x.append(face/255)\n            #print('max=',np.max(face))\n            #x[i,]=face/255\n        return np.array([x])\n    \n    def __generate_y(self, batch_files):\n        y=np.zeros((self.batch_size,2))\n        for i in range(len(batch_files)):\n            cur_files=train_decode_data[batch_files[i]]\n            #val=self.jdata[batch_files[i]]['label']=='FAKE'\n            #indxs=train_decode_data[batch_files[i].split('_')[0]]\n            #try:\n            label=int(cur_files[0].split('_')[-1].split('.')[0])\n            #except:\n                #print(cur_files,batch_files)\n            #    label=1\n            y[i]=keras.utils.to_categorical(label,2)\n            #print(val)\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_decode_data['efdyrflcpg.mp4']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bad_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#new_train_files=os.listdir('face_data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_files=[]\nfor i in range(len(train_files)):\n    if(not (train_files[i] in bad_names)):\n        if(train_files[i] in train_decode_data.keys()):\n            if(len(train_decode_data[train_files[i]])!=0):\n                new_train_files.append(train_files[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(new_train_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"data_hist=[]\nfor i in range(len(train_files)):\n    data_hist.append(train_label[train_files[i]]['label']=='FAKE')\ndata_hist=np.array(data_hist)\ncoef_fake=data_hist.sum()/len(data_hist)\ncoef_real=1-coef_fake\nprint(coef_fake,coef_real)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x,val_x  = train_test_split(new_train_files, test_size=0.15, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_x),len(val_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from sklearn.model_selection import train_test_split\n#train_x,val_x,train_y,val_y  = train_test_split(train_data_x,train_data_y, test_size=0.15, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#len(train_x),len(val_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gen=DataGeneratorFull(train_x,dim=(256,256),batch_size=1)\nval=DataGeneratorFull(val_x,dim=(256,256),batch_size=1)\n#test_gen=DataGeneratorFull(test_files,test_data,train_label,dim=(256,256),batch_size=1,mode='predict')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#gen=DataGeneratorFull(train_x,train_y,dim=(256,256),batch_size=4)\n#val=DataGeneratorFull(val_x,val_y,dim=(256,256),batch_size=4)\n#test_gen=DataGeneratorFull(test_files,test_data,train_label,dim=(256,256),batch_size=1,mode='predict')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ng=gen.__getitem__(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.hist(g[0][0].flatten())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(g[0][7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#g[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lstm_model(input_shape=(300,256,256,1)):\n    inp=keras.layers.Input(input_shape)\n    #out1=keras.layers.ConvLSTM2D(16,3,return_sequences=True,activation='relu',padding='same')(inp)\n    #out2=keras.layers.ConvLSTM2D(16,3,return_sequences=True,activation='relu',padding='same')(inp)\n   # out=keras.layers.concatenate([out1,out2])\n\n    #print(out.shape)\n    #out=keras.layers.MaxPooling3D((1,2,2))(inp)\n    out1=keras.layers.Conv3D(16,3,activation='relu',padding='same')(inp)\n    out2=keras.layers.Conv3D(16,3,activation='relu',padding='same')(inp)\n    out=keras.layers.concatenate([out1,out2])\n\n    out=keras.layers.MaxPooling3D((1,2,2))(out)\n    out1=keras.layers.Conv3D(16,3,activation='relu',padding='same')(out)\n    out2=keras.layers.Conv3D(16,3,activation='relu',padding='same')(out)\n    out=keras.layers.concatenate([out1,out2])\n\n    out=keras.layers.GlobalAveragePooling3D()(out)\n    out=keras.layers.Dense(1,activation='sigmoid')(out)\n\n    model=keras.models.Model(input=inp,output=out)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#os.listdir('/kaggle/input/densenet-keras/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#os.listdir('/kaggle/input/efficientnet-keras-weights-b0b5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unet(input_size = (32,32,1),descr=1,classes=1,activation='sigmoid'):\n    #descr=2\n    inputs = keras.layers.Input(input_size)\n    conv1 = keras.layers.Conv2D(int(64/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(inputs)\n    conv1 = keras.layers.Conv2D(int(64/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv1)\n    pool1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv1)\n    conv2 = keras.layers.Conv2D(int(128/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool1)\n    conv2 = keras.layers.Conv2D(int(128/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv2)\n    pool2 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv2)\n    conv3 = keras.layers.Conv2D(int(256/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool2)\n    conv3 = keras.layers.Conv2D(int(256/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv3)\n    pool3 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv3)\n    conv4 = keras.layers.Conv2D(int(512/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool3)\n    conv4 = keras.layers.Conv2D(int(512/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv4)\n    drop4 = keras.layers.Dropout(0.5)(conv4)\n    pool4 = keras.layers.MaxPooling2D(pool_size=(2, 2))(drop4)\n\n    conv5 = keras.layers.Conv2D(int(1024/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool4)\n    conv5 = keras.layers.Conv2D(int(1024/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv5)\n    drop5 = keras.layers.Dropout(0.5)(conv5)\n\n    up6 = keras.layers.Conv2D(int(512/descr), 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(keras.layers.UpSampling2D(size = (2,2))(drop5))\n    merge6 = keras.layers.concatenate([drop4,up6], axis = 3)\n    conv6 = keras.layers.Conv2D(int(512/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge6)\n    conv6 = keras.layers.Conv2D(int(512/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv6)\n\n    up7 = keras.layers.Conv2D(int(256/descr), 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(keras.layers.UpSampling2D(size = (2,2))(conv6))\n    merge7 = keras.layers.concatenate([conv3,up7], axis = 3)\n    conv7 = keras.layers.Conv2D(int(256/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge7)\n    conv7 = keras.layers.Conv2D(int(256/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv7)\n\n    up8 = keras.layers.Conv2D(int(128/descr), 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(keras.layers.UpSampling2D(size = (2,2))(conv7))\n    merge8 = keras.layers.concatenate([conv2,up8], axis = 3)\n    conv8 = keras.layers.Conv2D(int(128/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge8)\n    conv8 = keras.layers.Conv2D(int(128/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv8)\n\n    up9 = keras.layers.Conv2D(int(64/descr), 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(keras.layers.UpSampling2D(size = (2,2))(conv8))\n    merge9 = keras.layers.concatenate([conv1,up9], axis = 3)\n    conv9 = keras.layers.Conv2D(int(64/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge9)\n    conv9 = keras.layers.Conv2D(int(64/descr), 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)\n    conv9 = keras.layers.Conv2D(classes*2, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)\n    conv10 = keras.layers.Conv2D(classes, 1, activation = activation)(conv9)\n    #conv10=tf.keras.layers.Attention()(conv10)\n    #glpool=GlobalAveragePooling2D()(conv9)\n    #x = Dense(512, activation=\"relu\")(glpool)\n    #x = Dropout(0.5)(x)\n    #x = Dense(256, activation=\"relu\")(x)\n    #predictions = Dense(1107, activation=\"softmax\")(x)\n    model = keras.models.Model(input = inputs, output = conv10)\n    \n    #model.summary()\n\n    #if(pretrained_weights):\n    \t#model.load_weights(pretrained_weights)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lstm_model(input_shape=(256,256,3)):\n    inp=keras.layers.Input(input_shape)\n    \n    unet_back=unet(input_size = input_shape,descr=8,classes=3,activation='relu')\n    back=keras.applications.mobilenet.MobileNet(input_shape=input_shape,pooling='avg',include_top=False,weights=None)\n    back.load_weights('/kaggle/input/mobilenet/mobilenet_1_0_224_tf_no_top.h5')\n    out=unet_back(inp)\n    out=back(out)\n    #out1=keras.layers.Conv2D(16,3,activation='relu',padding='same')(inp)\n    #out2=keras.layers.Conv2D(16,3,activation='relu',padding='same')(inp)\n    #out=keras.layers.concatenate([out1,out2])\n\n    #out=keras.layers.MaxPooling2D((2,2))(out)\n    #out1=keras.layers.Conv2D(16,3,activation='relu',padding='same')(out)\n    #out2=keras.layers.Conv2D(16,3,activation='relu',padding='same')(out)\n    #out=keras.layers.concatenate([out1,out2])\n\n    #out=keras.layers.GlobalAveragePooling2D()(out)\n    #out=keras.layers.Dense(1024,activation='relu')(out)\n    \n    out=keras.layers.Dense(512,activation='relu')(out)\n    out=keras.layers.Dropout(0.5)(out)\n    out=keras.layers.Dense(2,activation='softmax')(out)\n\n    model=keras.models.Model(input=inp,output=out)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lstm_model(input_shape=(256,256,3)):\n    inp=keras.layers.Input(input_shape)\n    back=efn.EfficientNetB0(input_shape=input_shape,include_top=False,weights=None,pooling='avg')\n    back.load_weights('/kaggle/input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5')\n    #back.trainable = False\n    out=back(inp)\n    #out1=keras.layers.Conv2D(16,3,activation='relu',padding='same')(inp)\n    #out2=keras.layers.Conv2D(16,3,activation='relu',padding='same')(inp)\n    #out=keras.layers.concatenate([out1,out2])\n\n    #out=keras.layers.MaxPooling2D((2,2))(out)\n    #out1=keras.layers.Conv2D(16,3,activation='relu',padding='same')(out)\n    #out2=keras.layers.Conv2D(16,3,activation='relu',padding='same')(out)\n    #out=keras.layers.concatenate([out1,out2])\n\n    #out=keras.layers.GlobalAveragePooling2D()(out)\n    #out=keras.layers.Dense(1024,activation='relu')(out)\n    #out=keras.layers.BatchNormalization()(out)\n    #out=keras.layers.Dropout(0.5)(out)\n    out=keras.layers.Dense(256,activation='relu')(out)\n    out=keras.layers.Dense(2,activation='softmax')(out)\n\n    model=keras.models.Model(input=inp,output=out)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lstm_model(input_shape=(None,256,256,3)):\n    inp=keras.layers.Input(input_shape)\n    back=efn.EfficientNetB0(input_shape=(256,256,3),include_top=False,weights=None,pooling=None)\n    back.load_weights('/kaggle/input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5')\n    #back.trainable = False\n    \n    #out=back(inp)\n    #out1=keras.layers.ConvLSTM2D(32,3,activation='relu',padding='same')(inp)\n    #out2=keras.layers.ConvLSTM2D(32,3,activation='relu',go_backwards=True,padding='same')(inp)\n    #out=keras.layers.concatenate([out1,out2])\n    \n    out=keras.layers.TimeDistributed(back)(inp)\n    \n    out1=keras.layers.ConvLSTM2D(128,3,activation='relu')(out)\n    out2=keras.layers.ConvLSTM2D(128,3,go_backwards=True,activation='relu')(out)\n    out=keras.layers.concatenate([out1,out2])\n\n    out=keras.layers.GlobalAveragePooling2D()(out)\n    #out=keras.layers.Dense(1024,activation='relu')(out)\n    #out=keras.layers.BatchNormalization()(out)\n    #out=keras.layers.Dropout(0.5)(out)\n    #out=keras.layers.Dense(256,activation='relu')(out)\n    out=keras.layers.Dense(2,activation='softmax')(out)\n\n    model=keras.models.Model(input=inp,output=out)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=build_lstm_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check=keras.callbacks.ModelCheckpoint('res_weights.h5', monitor='val_loss',save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history=model.fit_generator(gen,validation_data=val,verbose=1,epochs=50,callbacks=[check])\n#history=model.fit(np.array(train_data_x),keras.utils.to_categorical(train_data_y,2),validation_split=0.15,verbose=1,epochs=20,callbacks=[check],shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('res_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"test_data_x=[]\n\nbad_names_test=[]\ntest_decode_data={}\nfor i in tqdm(range(len(test_files))):\n    video = VideoReader(TEST_DIR+test_files[i],shape=PARAMS_VIDEO['shape'],is_face=PARAMS_VIDEO['is_face'],is_first_face=PARAMS_VIDEO['is_first_face'],is_first=PARAMS_VIDEO['is_first'],on_each=PARAMS_VIDEO['on_each'])\n    video.get_video()\n    start_ind=len(test_data_x)\n    for j in range(len(video.faces)):\n        if(len(video.faces[j])==1):\n            \n            for t in range(len(video.faces[j])):\n                #print(video.faces[j][t].shape)\n                cv2.imwrite(f'test_data/{test_files[i]}_{t}.png',video.faces[j][t])\n                test_data_x.append(test_files[i])\n                test_decode_data[test_files[i]]=(start_ind,len(test_data_x))    \n                \n        #print(i,len(train_data_x),len(train_data_y))\n        else:\n            bad_names_test.append(test_files[i])\n    \n    \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x=os.listdir('test_data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen=DataGeneratorFull(test_x,data_dir='test_data',dim=(256,256),batch_size=1,mode='predict')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res=model.predict_generator(test_gen,verbose=1)\n#res=model.predict(np.array(test_data_x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_res=np.argmax(res,axis=-1)\nplt.hist(pred_res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df=pd.read_csv(SUB_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_res=[]\nsub_test_files=list(sub_df['filename'].values)\nfor i in (range(len(sub_test_files))):\n    if(sub_test_files[i] in test_decode_data.keys()):\n        indxs=test_decode_data[sub_test_files[i]] \n        val=np.sum(res[indxs[0]:indxs[1]],axis=0)#,axis=-1)\n        if(val[0]==0 and val[1]==0):\n            val=np.array([0,1],dtype='float32')\n        val=np.argmax(val)\n        full_res.append(val)\n        #print(indxs,val)\n    else:\n        full_res.append(1)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(full_res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(full_res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_weights('res_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#res=model.predict_generator(test_gen,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['label']=full_res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('submission.csv').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}