{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q /kaggle/input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom numpy import savetxt\nimport pandas as pd\nimport os\nfrom time import time\nimport cv2 #use OpenCV package\nface_cascade = cv2.CascadeClassifier('/kaggle/input/haarcascades/haarcascade_frontalface_default.xml')\nfrom tqdm.notebook import tqdm\nfrom mtcnn import MTCNN\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport tensorflow as tf\nimport random\nimport copy\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, GRU, Dropout\nimport tensorflow.keras as tfk\nfrom keras import layers\ntfkl = tfk.layers\nfrom keras.optimizers import SGD","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/'\nmetadata = pd.read_json(os.path.join(INPUT_PATH, 'metadata.json')).T\n#EXTRACT_NOISE = True\nWINDOW = 224\nFACE_CONFIDENCE = .8\n#FRAMES_PER_VIDEO = 1\n#NOISE_DEPTH = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FILES = metadata.index\nLABELS = metadata.label\nprint(FILES[5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Experiment on function ***extract_face()***","metadata":{}},{"cell_type":"markdown","source":"Take an image for trial","metadata":{}},{"cell_type":"code","source":"#try on image\nfn = FILES[5]\nvideo_path = os.path.join(INPUT_PATH, fn)\nvidcap = cv2.VideoCapture(video_path)\nsuccess, image = vidcap.read()\nim = image.copy()\nim.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(im)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#input is unstandardized image, output is face with standard size\n#这里有个问题就是如果视频中有两张脸，这个函数只会提取一张脸；如果没有脸，会返回背景截图；如果提取不到脸，就会返回空值（试一下FILES[2]）\n#这个函数需要被改进\ndef extract_face(frame):\n    face_rects = face_cascade.detectMultiScale(frame,scaleFactor=1.3, minNeighbors=5)\n    if len(face_rects)==0:\n        roi = np.empty((100,100,3))\n        roi[:]=np.nan\n        return roi\n    for (x,y,w,h) in face_rects: \n        roi = frame[y:y+h,x:x+w] \n    std_roi=cv2.resize(roi, (100,100), interpolation=cv2.INTER_AREA)\n    return std_roi","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = extract_face(im)\nprint(a.shape)\nplt.imshow(a)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Experiment on function ***crop_faces()***: extract face of every frame (one video with at most 20 frames)","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n#input is 1 filename, output is faces crop of size cnt_limit*(100,100,3)\n#if there's not enough faces to capture, it will return np.empty((20,100,100,3))\ndef crop_faces(fn,cnt_limit=20, plot=False, itr_limit = 250):\n    faces=[]\n    video_path = os.path.join(INPUT_PATH, fn)\n    vidcap = cv2.VideoCapture(video_path)\n    #fps = round(vidcap.get(cv2.CAP_PROP_FPS)) #frames per second\n    #detector = MTCNN()\n    success, image = vidcap.read() #sucess=True, image=matrix of numbers of size (1080, 1920, 3)\n    count = 0\n    itr = 0\n    empty_faces = np.empty((20,100,100,3));empty_faces[:]=np.nan;\n    while success:\n        #if go beyond iterations limit, end loop\n        itr+=1\n        if itr>=itr_limit and count<cnt_limit*5:\n            ##print(\"return empty\")\n            return empty_faces\n        #every 5 frames, store captured face; if no face, continue capturing but don't add \"count\"\n        if count%5 ==0:\n            ##print(\"count:\",count)\n            face = extract_face(image) #crop the face in frame as size (100,100,3)\n            if np.isnan(face).any():\n                success,image = vidcap.read()\n                ##print(\"no face\",\" count:\",count)\n                continue #if there's no face captured, then go to capture next face and not running count+=1\n            else:\n                faces.append(face)\n                #plot captured faces\n                if plot:\n                    plt.figure(figsize=(80,80))\n                    plt.subplot(cnt_limit,1,count/5+1)\n                    plt.title(str(count))\n                    plt.axis('off')\n                    plt.imshow(face)\n                    ##print('Read a new frame: ', success)\n        success,image = vidcap.read()\n        count += 1\n        if count >=cnt_limit*5:\n            break\n    if len(faces)<cnt_limit:\n        return empty_faces\n    return np.asarray(faces)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = FILES[56]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = crop_faces(fn,plot=True)\nprint(\"frames size:\",frames.shape) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.isnan(frames).any()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Experiment on function ***face_noise()***: compute the noise df of face crop","metadata":{}},{"cell_type":"code","source":"# input is 1 image of face (size=(100,100,3)), output is noise df of the image (size=(100,100))\ndef face_noise(img, NOISE_DEPTH=1):\n    if np.isnan(img).any():\n        empty_sf = np.empty((100,100))\n        empty_sf[:]=np.nan\n        return empty_sf\n    else:\n        img1 = img - cv2.GaussianBlur(img, (3,3), 0)\n        imgs1 = np.sum(img1, axis=2)\n        if NOISE_DEPTH == 1:\n            sf = np.fft.fftshift(np.fft.fft2(imgs1))\n            eps = np.max(sf) * 1e-2\n            s1 = np.log(sf + eps) - np.log(eps) \n            sf = (s1 * 255 / np.max(s1))\n            sf = np.abs(sf)\n        else:\n            sf = np.stack([\n                 np.fft.fftshift(np.fft.fft2( imgs1 )),\n                 np.fft.fftshift(np.fft.fft2( img1[:,:,0] - img1[:,:,1] )),\n                 np.fft.fftshift(np.fft.fft2( img1[:,:,1] - img1[:,:,2] )),\n                 np.fft.fftshift(np.fft.fft2( img1[:,:,2] - img1[:,:,0] ))],\n                 axis=-1)\n            sf = np.abs(sf)\n            nchans = sf.shape[2]\n            for c in range(nchans):\n                eps = np.max(sf[:,:,c]) * 1e-2\n                s1 = np.log(sf[:,:,c] + eps) - np.log(eps) \n                sf[:, :, c] = (s1 * 255 / np.max(s1))\n        return sf.astype(np.float16)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = frames[15]\nprint(\"original shape:\",img.shape)\na = face_noise(img)\nprint(\"noise_df shape:\",a.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Gaussian Blur to compute the noise... \n\nThis is a plot of comparison between original image and Gaussian Blurred Vesion, and the noise is the difference between these two pictures.","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nw=3\nh=2\nfig=plt.figure(figsize=(10, 10))\ncolumns = 2\nrows = 3\nfor i in range(1, rows+1):\n    img = frames[i-1]\n    fig.add_subplot(rows, columns, columns*i-1)\n    plt.imshow(img)\n    plt.axis('off')\n    img1 = img - cv2.GaussianBlur(img, (3,3), 0)\n    fig.add_subplot(rows, columns, columns*i)\n    plt.imshow(img1)\n    plt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build the imput df with function ***build_features()***: every video becomes an input matrix of (20,100,100)","metadata":{}},{"cell_type":"markdown","source":" every single video corresponds to a matrix of size (20,100,100), where 20 is the number of frames in each video, (100,100) is the noise matrix which sums up the 3-d RGB colors.","metadata":{}},{"cell_type":"code","source":"#input is 1 filename, output is noise array for that video, and its size=(20,100,100)\ndef build_features(f):\n    frames = crop_faces(f,plot=False)\n    if np.isnan(frames).any():\n        empty_noise = np.empty((20,100,100))\n        empty_noise[:]=np.nan\n        return empty_noise\n    #print(\"filename:\",file,\"frames size:\",frames.shape)\n    noise = np.array(list(map(face_noise,frames))) #map function to every frame; noise is of (20,100,100) for each video\n    return noise","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = build_features(FILES[3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-trained features","metadata":{}},{"cell_type":"code","source":"\"\"\"\n%%time\n\ntqdm.pandas()\n\nfor n in range(1,5):\n\n    gap = int(len(FILES)/4)\n    \n    if n<4:\n    \n        FILES_store = FILES[range((n-1)*gap,n*gap+1)]\n        \n    else:\n    \n        FILES_store = FILES[(n-1)*gap:]\n        \n    all_video_fn = np.array(list(tqdm(map(build_features,FILES_store),\n                         total=len(FILES_store), desc='Build Feature Set over FILES')))\n                         \n    np.save(\"video\"+str(n)+\".npy\", all_video_fn)\n    \n    print(\"video\"+str(n)+\".npy\"+\" Complete!\")\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nn=4\ngap = int(len(FILES)/4)\nif n<4:\n    FILES_store = FILES[range((n-1)*gap,n*gap)]\nelse:\n    FILES_store = FILES[(n-1)*gap:]\nall_video_fn = np.array(list(tqdm(map(build_features,FILES_store),\n                     total=len(FILES_store), desc='Build Feature Set over FILES')))\nnp.save(\"video\"+str(n)+\".npy\", all_video_fn)\nprint(\"video\"+str(n)+\".npy\"+\" Complete!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load pre-trained features for each picture, and combine them with labels","metadata":{}},{"cell_type":"code","source":"ar_1 = np.load('/kaggle/input/pre-trained-features/video1.npy')\nar_2 = np.load('/kaggle/input/pre-trained-features/video2.npy')\nar_3 = np.load('/kaggle/input/pre-trained-features/video3.npy')\nar_4 = np.load('/kaggle/input/pre-trained-features/video4.npy')\nprint(ar_1.shape)\nprint(ar_2.shape)\nprint(ar_3.shape)\nprint(ar_4.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obtain Feature Set and LABELS","metadata":{}},{"cell_type":"code","source":"feature_set = np.concatenate((ar_1, ar_2, ar_3, ar_4))\n##print(feature_set[0,:,:,:])\n#convert strings ('REAL'/'FAKE') into int(1/0)\nlabels_set = np.array(LABELS[0:feature_set.shape[0]])\npos = np.where(labels_set=='REAL') #pos[0] is the indices of 'REAL' label\nlabels_set = np.zeros(shape=(feature_set.shape[0],1))\nlabels_set[pos[0],0]=1\nprint(labels_set[0:5,:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#eliminate positions with np.nan values\nall_pos=[]; real_pos=[]; fake_pos=[];\nfor i in range(0,feature_set.shape[0]):\n    a = np.isnan(feature_set[i,:,:,:]).any()\n    if not a:\n        all_pos.append(i)\n        if i in pos[0]:\n            real_pos.append(i)\n        else:\n            fake_pos.append(i)\nprint(feature_set.shape[0]-len(all_pos),\"videos are dropped out of\",feature_set.shape[0])\nprint(\"real videos:\",len(real_pos))\nprint(\"fake videos:\",len(fake_pos))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.shuffle(real_pos)\nrandom.shuffle(fake_pos)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Therefore, to balance the labels, we need to upsampling real_videos for 4 times","metadata":{}},{"cell_type":"markdown","source":"### split training set and test set","metadata":{}},{"cell_type":"code","source":"training_ix = real_pos[0:int(len(real_pos)*0.5+1)]*4+fake_pos[0:int(len(fake_pos)*0.7+1)]\nremain_real_pos = [j for j in real_pos if j not in training_ix]\nremain_fake_pos = [j for j in fake_pos if j not in training_ix]\ntest_ix = remain_real_pos+remain_fake_pos\n##\nrandom.shuffle(training_ix)\nrandom.shuffle(test_ix)\nX_train = feature_set[training_ix,:,:,:]\ny_train = labels_set[training_ix,:]\nX_test = feature_set[test_ix,:,:,:]\ny_test = labels_set[test_ix,:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"label <real> in train set:\",len(y_train[y_train==1]))\nprint(\"label <real> in test set:\",len(y_test[y_test==1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X_train:\",X_train.shape)\nprint(\"y_train:\",y_train.shape)\nprint(\"X_test:\",X_test.shape)\nprint(\"y_test:\",y_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cbs = [tfk.callbacks.EarlyStopping(min_delta=0.001,patience=3)]\nds_train = tf.data.Dataset.from_tensor_slices((X_train,y_train)).batch(16).repeat().prefetch(10) \nds_test = tf.data.Dataset.from_tensor_slices((X_test,y_test)).batch(16).prefetch(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build 3-d CNN","metadata":{}},{"cell_type":"code","source":"model = tfk.Sequential()\n\n#Block 1\nmodel.add(tfkl.Conv2D(filters = 20, kernel_size=(3,3), strides=(3,3), \n                      padding=\"valid\", activation='tanh', input_shape=(20, 100, 100))) #smooth stride; padding: handle edges;\nmodel.add(tfkl.MaxPool2D(pool_size=(1,2)))\n\n#Block 2\nmodel.add(tfkl.Conv2D(filters = 16, kernel_size=2, strides=1, \n                      padding=\"valid\", activation='relu')) #smooth stride; padding: handle edges;\n#model.add(tfkl.MaxPool2D(pool_size=2))\n#model.add(tfkl.Dropout(.2))\n\n#Block 3\nmodel.add(tfkl.Conv2D(filters = 8, kernel_size=2, strides=1, \n                      padding=\"valid\", activation='relu')) #smooth stride; padding: handle edges;\n#model.add(tfkl.MaxPool2D(pool_size=2))\n\n#Flatten\nmodel.add(tfkl.GlobalMaxPool2D())\nmodel.add(tfkl.Dense(1, activation='sigmoid'))\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=tfk.optimizers.RMSprop(), ##SGD(lr=0.01, decay=1e-7, momentum=0.9, nesterov=False)\n    loss=tfk.losses.BinaryCrossentropy(),\n    metrics=[\"acc\"]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = model.fit(ds_train, steps_per_epoch=10,epochs=20,\n                    callbacks=cbs,verbose=1,validation_data=ds_test,validation_steps=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(results.history[\"loss\"])\nplt.plot(results.history[\"val_loss\"])\nplt.legend(labels=[\"loss\", \"val_loss\"])\nplt.xlabel(\"epochs\")\nplt.ylabel(\"loss\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build GRU","metadata":{}},{"cell_type":"code","source":"print(feature_set[all_pos].shape)\nprint(labels_set[all_pos].shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape)\nprint(X_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xx_train = X_train.reshape(309,20,10000,1)\nxx_train = np.ndarray.mean(xx_train,axis=2)\nprint(xx_train.shape)\nxx_test = X_test.reshape(108,20,10000,1)\nxx_test = np.ndarray.mean(xx_test,axis=2)\nprint(xx_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gru_feature_set = feature_set[all_pos].reshape(321,20,10000,1)\ngru_feature_set = np.ndarray.mean(gru_feature_set,axis=2)\nprint(gru_feature_set.shape)\ngru_labels_set = labels_set[all_pos]\nprint(gru_labels_set.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cbs = [tfk.callbacks.EarlyStopping(min_delta=0.001,patience=3)]\ngru_ds_train = tf.data.Dataset.from_tensor_slices((xx_train,y_train)).batch(16).repeat().prefetch(10) \ngru_ds_test = tf.data.Dataset.from_tensor_slices((xx_test,y_test)).batch(16).prefetch(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The GRU architecture\nGRU_mod = Sequential()\n# First GRU layer with Dropout regularisation\nGRU_mod.add(GRU(units=20, return_sequences=True, input_shape=(xx_train.shape[1],1), activation='tanh'))\nGRU_mod.add(Dropout(0.2))\n# Second GRU layer\nGRU_mod.add(GRU(units=50, return_sequences=True, input_shape=(xx_train.shape[1],1), activation='tanh'))\nGRU_mod.add(Dropout(0.2))\n# Third GRU layer\nGRU_mod.add(GRU(units=30, return_sequences=True, input_shape=(xx_train.shape[1],1), activation='tanh'))\nGRU_mod.add(Dropout(0.2))\n# Fourth GRU layer\nGRU_mod.add(GRU(units=10, activation='tanh'))\nGRU_mod.add(Dropout(0.2))\n# The output layer\nGRU_mod.add(Dense(units=1,activation='sigmoid'))\nGRU_mod.summary()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GRU_mod.compile(optimizer=SGD(lr=0.01, decay=1e-7, momentum=0.9, nesterov=False),\n                     loss=tfk.losses.BinaryCrossentropy(),metrics=[\"acc\"])\noutput =  GRU_mod.fit(xx_train,y_train,epochs=50,batch_size=16)\n#output = GRU_mod.fit(gru_ds_train, steps_per_epoch=10,epochs=20,\n#                    verbose=1,validation_data=gru_ds_test,validation_steps=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(output.history[\"loss\"])\nplt.legend(labels=[\"loss\"])\nplt.xlabel(\"epochs\")\nplt.ylabel(\"loss\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GRU_mod.compile(optimizer=SGD(lr=0.01, decay=1e-7, momentum=0.9, nesterov=False),\n                     loss=tfk.losses.BinaryCrossentropy(),metrics=[\"acc\"])\noutput =  GRU_mod.fit(gru_feature_set,gru_labels_set,epochs=50,batch_size=16)\n#output = GRU_mod.fit(gru_ds_train, steps_per_epoch=10,epochs=20,\n#                    verbose=1,validation_data=gru_ds_test,validation_steps=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(output.history[\"loss\"])\nplt.legend(labels=[\"loss\"])\nplt.xlabel(\"epochs\")\nplt.ylabel(\"loss\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Keep Updating...","metadata":{}},{"cell_type":"markdown","source":"Thanks for the contribution of @","metadata":{}},{"cell_type":"markdown","source":"### Reference\n\n\nhttps://www.kaggle.com/timesler/facial-recognition-model-in-pytorch\n\nhttps://www.kaggle.com/thebrownviking20/intro-to-recurrent-neural-networks-lstm-gru\n\nhttps://www.kaggle.com/pathofdata/dpn50","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}