{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is the simple implementation of convolution LSTM network and prediction. I have used https://www.kaggle.com/unkownhihi/dfdc-lrcn-inference method to extract face and feed to neaural network."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from keras import Input\n\nfrom pathlib import Path\nfrom keras.callbacks import Callback\nfrom keras.layers import Dense, Flatten, Dropout, ZeroPadding3D, ConvLSTM2D, Reshape, BatchNormalization, Activation\nfrom keras.layers.recurrent import LSTM\nfrom keras.models import Sequential, load_model\nfrom keras.optimizers import Adam, RMSprop\n\nfrom keras.layers.wrappers import TimeDistributed\nfrom keras.layers.convolutional import (Conv2D, MaxPooling3D, Conv3D,\n    MaxPooling2D)\nfrom collections import deque\nimport sys\nimport logging\nimport keras\nfrom keras.applications import Xception, ResNet50, InceptionV3\nfrom keras.layers import Dense, GlobalAveragePooling2D\nfrom keras.models import Model\n\nfrom keras.models import load_model\n\nfrom keras.callbacks import Callback, ModelCheckpoint\n\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom keras.utils import to_categorical\n\nfrom sklearn.utils import class_weight, shuffle\n\nimport tensorflow as tf\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\nimport pandas as pd\n\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, Callback\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import GroupKFold\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_graph = tf.Graph()\nwith detection_graph.as_default():\n    od_graph_def = tf.compat.v1.GraphDef()\n    with tf.io.gfile.GFile('../input/conv-lstm-deepfake-weights/frozen_inference_graph_face.pb', 'rb') as fid:\n        serialized_graph = fid.read()\n        od_graph_def.ParseFromString(serialized_graph)\n        tf.import_graph_def(od_graph_def, name='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsess=tf.compat.v1.Session(graph=detection_graph, config=config)\nimage_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\nboxes_tensor = detection_graph.get_tensor_by_name('detection_boxes:0')\nscores_tensor = detection_graph.get_tensor_by_name('detection_scores:0')\nnum_detections = detection_graph.get_tensor_by_name('num_detections:0')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport keras\nfrom scipy import io\n\n\nimport tensorflow as tf\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\n\nimport os\n\ndef get_img(images):\n    global boxes,scores,num_detections\n    im_heights,im_widths=[],[]\n    imgs=[]\n    for image in images:\n        (im_height,im_width)=image.shape[:-1]\n        imgs.append(image)\n        im_heights.append(im_height)\n        im_widths.append(im_widths)\n    imgs=np.array(imgs)\n    (boxes, scores_) = sess.run(\n        [boxes_tensor, scores_tensor],\n        feed_dict={image_tensor: imgs})\n    finals=[]\n    for x in range(boxes.shape[0]):\n        scores=scores_[x]\n        max_=np.where(scores==scores.max())[0][0]\n        box=boxes[x][max_]\n        ymin, xmin, ymax, xmax = box\n        (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n                                      ymin * im_height, ymax * im_height)\n        left, right, top, bottom = int(left), int(right), int(top), int(bottom)\n        image=imgs[x]\n        finals.append(cv2.cvtColor(cv2.resize(image[max([0,top-40]):bottom+80,max([0,left-40]):right+80],(128,128)),cv2.COLOR_BGR2RGB))\n    finals=np.array(finals)\n    return finals\ndef detect_video(video):\n    frame_count=10\n    capture = cv2.VideoCapture('../input/deepfake-detection-challenge/test_videos/'+video)\n    v_len = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_idxs = np.linspace(0,v_len,frame_count, endpoint=False, dtype=np.int)\n    imgs=[]\n    i=0\n    for frame_idx in range(int(v_len)):\n        ret = capture.grab()\n        if not ret: \n            print(\"Error grabbing frame %d from movie %s\" % (frame_idx, video))\n        if frame_idx >= frame_idxs[i]:\n            if frame_idx-frame_idxs[i]>20:\n                return None\n            ret, frame = capture.retrieve()\n            if not ret or frame is None:\n                print(\"Error retrieving frame %d from movie %s\" % (frame_idx, video))\n            else:\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                imgs.append(frame)\n            i += 1\n            if i >= len(frame_idxs):\n                break\n    imgs=get_img(imgs)\n    if len(imgs)<10:\n        return None\n    return np.stack(imgs)\n\nclass VideoSequenceGenerator(keras.utils.Sequence):\n\n    def __init__(self, data_path,batch_size=64, shuffle = False):\n        'Initialization'\n        self.train_set = data_path \n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.indices = np.arange(self.train_set.shape[0])\n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return self.train_set.shape[0]//self.batch_size\n\n    def on_epoch_end(self):\n        # update indexes after each epoch\n        if self.shuffle == True:\n            np.random.shuffle(self.indices)\n\n\t\t\n    def __getitem__(self, index):\n        #'Generate one batch of data'  \n        X=np.zeros((self.batch_size,10,128,128,3))\n\n\n        indexes = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n\n        for i in range (self.batch_size):\n            data= detect_video(self.train_set.iloc[indexes[i]])\n\n\n            # data=io.loadmat(self.train_set.path[indexes[i]])\n            batch_x = data\n\n        \n            \n        \n            X[i,:,:]= batch_x\n          \n   \n\n        return X\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nseq = Sequential()\nseq.add(Conv3D(filters=32, kernel_size=(2,2,2),input_shape=(10, 128, 128, 3),\n               activation='relu',\n               padding='same', data_format='channels_last'))\nseq.add(BatchNormalization())\nseq.add(ConvLSTM2D(filters=16, kernel_size=(2, 2),\n                   padding='same', return_sequences=True))\nseq.add(BatchNormalization())\nseq.add(Dropout(0.3))\n\nseq.add(MaxPooling3D(pool_size=(2,2,2)))\n\nseq.add(ConvLSTM2D(filters=16, kernel_size=(2, 2),\n                   padding='same', return_sequences=True))\nseq.add(BatchNormalization())\nseq.add(Dropout(0.3))\n\n\n\nseq.add(MaxPooling3D(pool_size=(2,2,2)))\nseq.add(Flatten())\n\nseq.add(BatchNormalization())\n\nseq.add(Dense(32,activation='elu'))\n\nseq.add(Dense(1,activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=seq\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.mkdir('./videos/')\n# for x in tqdm(glob.glob('../input/deepfake-detection-challenge/test_videos/*.mp4')):\n#     try:\n#         filename=x.replace('../input/deepfake-detection-challenge/test_videos/','').replace('.mp4','.jpg')\n#         a=detect_video(x)\n#         if a is None:\n#             continue\n#         cv2.imwrite('./videos/'+filename,a)\n#     except Exception as err:\n#         print(err)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel.load_weights('../input/conv-lstm-deepfake-weights/model-12-0.31.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/deepfake-detection-challenge/sample_submission.csv\")\n\nsubmission=pd.DataFrame()\nsubmission['filename']=os.listdir(('../input/deepfake-detection-challenge/test_videos/'))\nsubmission['label']=0.5\nfilenames=[]\nbatch=[]\nbatch1=[]\npreds=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_name=submission['filename']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre=VideoSequenceGenerator(x_name,8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model.predict_generator(pre, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames=submission['filename']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_preds=[]\nfor x in preds:\n    new_preds.append(x[0])\nprint(sum(new_preds)/len(new_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x,y in zip(new_preds,filenames):\n    submission.loc[submission['filename']==y,'label']=min([max([0.0,x]),1.0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(submission['label']).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}