{"cells":[{"metadata":{},"cell_type":"markdown","source":"Feel free to use my \"DeepFake\" dataset. It includes 16 chunks of training data(only include cropped faces) and I will add new ones once in a while. **BEFORE YOU USE THE DATASET ATACHED TO THIS KERNEL, YOU HAVE TO AGREE THE DEEPFAKE COMPETITION RULE.**"},{"metadata":{},"cell_type":"markdown","source":"Edit: \n1. Change CNN More Similar To MesoNet\n2. Add more data\n3. Did some hyperparameter tuning\n4. Selected the 150th frame instead of the first frame\n5. Bigger image size."},{"metadata":{},"cell_type":"markdown","source":"Code for generating dataset:"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''from mtcnn import MTCNN\nimport tqdm\nimport datetime\nimport smtplib\nimport os\nimport cv2\nimport numpy as np\nimport sys\nimport shutil\nd_num=sys.argv[1]\nif len(d_num)==1:\n    a_num = d_num\n    d_num='0'+d_num\nelse:\n    a_num=d_num\ndetector = MTCNN()\ndef detect_face(img):\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    final = []\n    detected_faces_raw = detector.detect_faces(img)\n    if detected_faces_raw==[]:\n        #print('no faces found')\n        return []\n    confidences=[]\n    for n in detected_faces_raw:\n        x,y,w,h=n['box']\n        final.append([x,y,w,h])\n        confidences.append(n['confidence'])\n    if max(confidences)<0.7:\n        return []\n    max_conf_coord=final[confidences.index(max(confidences))]\n    #return final\n    return max_conf_coord\ndef crop(img,x,y,w,h):\n    x-=40\n    y-=40\n    w+=80\n    h+=80\n    if x<0:\n        x=0\n    if y<=0:\n        y=0\n    return cv2.cvtColor(cv2.resize(img[y:y+h,x:x+w],(256,256)),cv2.COLOR_BGR2RGB)\ndef detect_video(video):\n    v_cap = cv2.VideoCapture(video)\n    v_cap.set(1, NUM_FRAME)\n    success, vframe = v_cap.read()\n    vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n    bounding_box=detect_face(vframe)\n    if bounding_box==[]:\n        count=0\n        current=NUM_FRAME\n        while bounding_box==[] and count<MAX_SKIP:\n            current+=1\n            v_cap.set(1,current)\n            success, vframe = v_cap.read()\n            vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n            bounding_box=detect_face(vframe)\n            count+=1\n        if bounding_box==[]:\n            print('hi')\n            return None\n    x,y,w,h=bounding_box\n    v_cap.release()\n    return crop(vframe,x,y,w,h)\ntest_dir = './dfdc_train_part_' + a_num + '/'\ntest_video_files = [test_dir + x for x in os.listdir(test_dir)]\nos.makedirs('./DeepFake' + d_num,exist_ok=True)\nMAX_SKIP=10\nNUM_FRAME=150\ncount=0\nfor video in tqdm.tqdm(test_video_files):\n    try:\n        if video=='./dfdc_train_part_'+a_num+'/metadata.json':\n            shutil.copyfile(video,'./metadata'+str(a_num)+'.json')\n        img_file=detect_video(video)\n        os.remove(video)\n        if img_file is None:\n            count+=1\n            continue\n        cv2.imwrite('./DeepFake'+d_num+'/'+video.replace('.mp4','').replace(test_dir,'')+'.jpg',img_file)\n    except Exception as err:\n      print(err)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Install MTCNN**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Import Libraries**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport keras\nimport os\nimport numpy as np\nfrom sklearn.metrics import log_loss\nfrom keras import Sequential\nfrom keras.layers import *\nfrom keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom mtcnn import MTCNN\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Train Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train0 = pd.read_json('../input/deepfake/metadata0.json')\ndf_train1 = pd.read_json('../input/deepfake/metadata1.json')\ndf_train2 = pd.read_json('../input/deepfake/metadata2.json')\ndf_train3 = pd.read_json('../input/deepfake/metadata3.json')\ndf_train4 = pd.read_json('../input/deepfake/metadata4.json')\ndf_train5 = pd.read_json('../input/deepfake/metadata5.json')\ndf_train6 = pd.read_json('../input/deepfake/metadata6.json')\ndf_train7 = pd.read_json('../input/deepfake/metadata7.json')\ndf_train8 = pd.read_json('../input/deepfake/metadata8.json')\ndf_train9 = pd.read_json('../input/deepfake/metadata9.json')\ndf_train10 = pd.read_json('../input/deepfake/metadata10.json')\ndf_train11 = pd.read_json('../input/deepfake/metadata11.json')\ndf_train12 = pd.read_json('../input/deepfake/metadata12.json')\ndf_train13 = pd.read_json('../input/deepfake/metadata13.json')\ndf_train14 = pd.read_json('../input/deepfake/metadata14.json')\ndf_train15 = pd.read_json('../input/deepfake/metadata15.json')\ndf_train16 = pd.read_json('../input/deepfake/metadata16.json')\nLABELS = ['REAL','FAKE']\ndf_trains = [df_train0 ,df_train1, df_train2, df_train3, df_train4,\n             df_train5, df_train6, df_train7, df_train8, df_train9,\n            df_train11, df_train12, df_train13, df_train14, df_train15,\n            df_train16]\nnums = list(range(len(df_trains)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook\ndef read_image(num,name):\n    num=str(num)\n    if len(num)==2:\n        path='../input/deepfake/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)\n    else:\n        path='../input/deepfake/DeepFake0'+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)\n        \nX = []\ny = []\nfor df_train,num in tqdm_notebook(zip(df_trains,nums),total=len(df_trains)):\n    images = list(df_train.columns.values)\n    for x in images:\n        try:\n            X.append(read_image(num,x))\n            y.append(LABELS.index(df_train[x]['label']))\n        except Exception as err:\n            pass\n            #print(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.shape(X))\nprint(np.shape(y))\nprint(type(X))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Apply Underbalancing Techinique"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake samples')\nprint('There are '+str(y.count(0))+' real samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The data is not balanced. We are going to use the undersampling technique."},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nreal=[]\nfake=[]\nfor m,n in zip(X,y):\n    if n==0:\n        real.append(m)\n    else:\n        fake.append(m)\nfake=random.sample(fake,len(real))\nX,y=[],[]\nfor x in real:\n    X.append(x)\n    y.append(0)\nfor x in fake:\n    X.append(x)\n    y.append(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake samples')\nprint('There are '+str(y.count(0))+' real samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, the data is balanced."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X,val_X,train_y,val_y = train_test_split(X, y, test_size=0.15,shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def define_model():\n    model = Sequential(\n        [\n            Conv2D(8, (3, 3), padding=\"same\", activation = 'elu', input_shape=(256, 256,3)),\n            BatchNormalization(),\n            MaxPooling2D(2, 2),\n            Conv2D(8, (5, 5), padding=\"same\", activation = 'elu'),\n            BatchNormalization(),\n            MaxPooling2D(2, 2),\n            Conv2D(16, (5, 5), padding=\"same\", activation = 'elu'),\n            BatchNormalization(),\n            MaxPooling2D(2, 2),\n            Conv2D(16, (5, 5), padding=\"same\", activation = 'elu'),\n            BatchNormalization(),\n            MaxPooling2D(4, 4),\n            Flatten(),\n            Dropout(0.5),\n            Dense(16,activation='relu'),\n            Dropout(0.5),\n            Dense(1, activation=\"sigmoid\"),\n        ]\n    )\n    # Define the optimizer\n    optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n    model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This model is the same as MesoNet"},{"metadata":{},"cell_type":"markdown","source":"# Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\nfrom keras.optimizers import RMSprop\n# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='accuracy', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\n# Training hyperparameters\nepochs = 4\nbatch_size = 20\n\nmodel=define_model()\nhistory = model.fit([train_X], [train_y], batch_size = batch_size, epochs = 4, verbose = 1,\n                    callbacks=[learning_rate_reduction])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Check Validation Log Loss**"},{"metadata":{"trusted":true},"cell_type":"code","source":"answer=[LABELS[n] for n in val_y]\npred=np.random.random(len(val_X))\nprint('random loss: ' + str(log_loss(answer,pred.clip(0.0001,0.99999))))\npred=np.array([1 for _ in range(len(val_X))])\nprint('1 loss: ' + str(log_loss(answer,pred)))\npred=np.array([0 for _ in range(len(val_X))])\nprint('0 loss: ' + str(log_loss(answer,pred)))\npred=np.array([0.5 for _ in range(len(val_X))])\nprint('0.5 loss: ' + str(log_loss(answer,pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=model.predict([val_X])\nprint('model loss: '+str(log_loss(answer,pred.clip(0.1,0.9))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(pred.mean())\nprint(pred[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SKIP=10\nNUM_FRAME=150\ntest_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nfilenames = os.listdir(test_dir)\nprediction_filenames = filenames\ntest_video_files = [test_dir + x for x in filenames]\ndetector = MTCNN()\ndef detect_face(img):\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    final = []\n    detected_faces_raw = detector.detect_faces(img)\n    if detected_faces_raw==[]:\n        #print('no faces found')\n        return []\n    confidences=[]\n    for n in detected_faces_raw:\n        x,y,w,h=n['box']\n        final.append([x,y,w,h])\n        confidences.append(n['confidence'])\n    if max(confidences)<0.7:\n        return []\n    max_conf_coord=final[confidences.index(max(confidences))]\n    #return final\n    return max_conf_coord\ndef crop(img,x,y,w,h):\n    x-=40\n    y-=40\n    w+=80\n    h+=80\n    if x<0:\n        x=0\n    if y<=0:\n        y=0\n    return cv2.cvtColor(cv2.resize(img[y:y+h,x:x+w],(256,256)),cv2.COLOR_BGR2RGB)\ndef detect_video(video):\n    v_cap = cv2.VideoCapture(video)\n    v_cap.set(1, NUM_FRAME)\n    success, vframe = v_cap.read()\n    vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n    bounding_box=detect_face(vframe)\n    if bounding_box==[]:\n        count=0\n        current=NUM_FRAME\n        while bounding_box==[] and count<MAX_SKIP:\n            current+=1\n            v_cap.set(1,current)\n            success, vframe = v_cap.read()\n            vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n            bounding_box=detect_face(vframe)\n            count+=1\n        if bounding_box==[]:\n            print('no faces found')\n            prediction_filenames.remove(video.replace('/kaggle/input/deepfake-detection-challenge/test_videos/',''))\n            return None\n    x,y,w,h=bounding_box\n    v_cap.release()\n    return crop(vframe,x,y,w,h)\ntest_X = []\nfor video in tqdm(test_video_files):\n    x=detect_video(video)\n    if x is None:\n        continue\n    test_X.append(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test=pd.read_csv('/kaggle/input/deepfake-detection-challenge/sample_submission.csv')\ndf_test['label']=0.5\npreds=model.predict([test_X]).clip(0.1,0.9)\nfor pred,name in zip(preds,prediction_filenames):\n    name=name.replace('/kaggle/input/deepfake-detection-challenge/test_videos/','')\n    df_test.iloc[list(df_test['filename']).index(name),1]=pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Further Work\n1. Do some more hyperparamater tuning\n2. Train on the whole video(and maybe also sound)\n3. Try LSTM-CNN\n4. K Folds(I will try it later when I upload more data)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}