{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":841326,"sourceType":"datasetVersion","datasetId":444017},{"sourceId":893807,"sourceType":"datasetVersion","datasetId":451078},{"sourceId":903208,"sourceType":"datasetVersion","datasetId":448076},{"sourceId":129400,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":109020,"modelId":133344}],"dockerImageVersionId":29845,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deepfake Detection Competition - Keras CNN Starter Epoch & Learning Rate EDA","metadata":{}},{"cell_type":"markdown","source":"- This kernel was built from/upon: https://www.kaggle.com/unkownhihi/starter-kernel-with-cnn-model-ll-lb-0-69235\n- Changes:\n    - Varied learning rates and epochs","metadata":{}},{"cell_type":"markdown","source":"**If you found this helpful, please *upvote* this kernel and the associated datasets.** In particular, do credit the original kernel by https://www.kaggle.com/unkownhihi/starter-kernel-with-cnn-model-ll-lb-0-69235.","metadata":{}},{"cell_type":"markdown","source":"Code for generating dataset:","metadata":{}},{"cell_type":"code","source":"'''from mtcnn import MTCNN\nimport tqdm\nimport datetime\nimport smtplib\nimport os\nimport cv2\nimport numpy as np\nimport sys\nimport shutil\nd_num=sys.argv[1]\nif len(d_num)==1:\n    a_num = d_num\n    d_num='0'+d_num\nelse:\n    a_num=d_num\ndetector = MTCNN()\ndef detect_face(img):\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    final = []\n    detected_faces_raw = detector.detect_faces(img)\n    if detected_faces_raw==[]:\n        #print('no faces found')\n        return []\n    confidences=[]\n    for n in detected_faces_raw:\n        x,y,w,h=n['box']\n        final.append([x,y,w,h])\n        confidences.append(n['confidence'])\n    if max(confidences)<0.7:\n        return []\n    max_conf_coord=final[confidences.index(max(confidences))]\n    #return final\n    return max_conf_coord\ndef crop(img,x,y,w,h):\n    x-=120\n    y-=120\n    w+=135\n    h+=136\n    if x<0:\n        x=0\n    if y<=0:\n        y=0\n    return cv2.cvtColor(cv2.resize(img[y:y+h,x:x+w],(256,256)),cv2.COLOR_BGR2RGB)\ndef detect_video(video):\n    v_cap = cv2.VideoCapture(video)\n    v_cap.set(1, NUM_FRAME)\n    success, vframe = v_cap.read()\n    vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n    bounding_box=detect_face(vframe)\n    if bounding_box==[]:\n        count=0\n        current=NUM_FRAME\n        while bounding_box==[] and count<MAX_SKIP:\n            current+=1\n            v_cap.set(1,current)\n            success, vframe = v_cap.read()\n            vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n            bounding_box=detect_face(vframe)\n            count+=1\n        if bounding_box==[]:\n            print('hi')\n            return None\n    x,y,w,h=bounding_box\n    v_cap.release()\n    return crop(vframe,x,y,w,h)\ntest_dir = './dfdc_train_part_' + a_num + '/'\ntest_video_files = [test_dir + x for x in os.listdir(test_dir)]\nos.makedirs('./DeepFake' + d_num,exist_ok=True)\nMAX_SKIP=10\nNUM_FRAME=256\ncount=0\nfor video in tqdm.tqdm(test_video_files):\n    try:\n        if video=='./dfdc_train_part_'+a_num+'/metadata.json':\n            shutil.copyfile(video,'./metadata'+str(a_num)+'.json')\n        img_file=detect_video(video)\n        os.remove(video)\n        if img_file is None:\n            count+=1\n            continue\n        cv2.imwrite('./DeepFake'+d_num+'/'+video.replace('.mp4','').replace(test_dir,'')+'.jpg',img_file)\n    except Exception as err:\n      print(err)'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:37:09.327708Z","iopub.execute_input":"2024-10-13T06:37:09.32807Z","iopub.status.idle":"2024-10-13T06:37:09.339697Z","shell.execute_reply.started":"2024-10-13T06:37:09.328016Z","shell.execute_reply":"2024-10-13T06:37:09.339044Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Install MTCNN","metadata":{}},{"cell_type":"code","source":"!pip install ../input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:37:09.344671Z","iopub.execute_input":"2024-10-13T06:37:09.344997Z","iopub.status.idle":"2024-10-13T06:37:39.104529Z","shell.execute_reply.started":"2024-10-13T06:37:09.344947Z","shell.execute_reply":"2024-10-13T06:37:39.103699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install keras","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:37:39.107033Z","iopub.execute_input":"2024-10-13T06:37:39.10737Z","iopub.status.idle":"2024-10-13T06:38:05.226446Z","shell.execute_reply.started":"2024-10-13T06:37:39.107309Z","shell.execute_reply":"2024-10-13T06:38:05.225471Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport keras\nimport os\nimport numpy as np\nfrom sklearn.metrics import log_loss\nfrom keras import Model,Sequential\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom tqdm.notebook import tqdm\nimport glob\nfrom mtcnn import MTCNN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:38:05.228202Z","iopub.execute_input":"2024-10-13T06:38:05.228478Z","iopub.status.idle":"2024-10-13T06:38:15.872099Z","shell.execute_reply.started":"2024-10-13T06:38:05.228426Z","shell.execute_reply":"2024-10-13T06:38:15.871007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sorted(glob.glob('../input/deepfake/meta*'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:38:15.873347Z","iopub.execute_input":"2024-10-13T06:38:15.873597Z","iopub.status.idle":"2024-10-13T06:38:15.919887Z","shell.execute_reply.started":"2024-10-13T06:38:15.873557Z","shell.execute_reply":"2024-10-13T06:38:15.919209Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Train Data","metadata":{}},{"cell_type":"code","source":"df_train0 = pd.read_json('../input/deepfake/metadata0.json')\ndf_train1 = pd.read_json('../input/deepfake/metadata1.json')\ndf_train2 = pd.read_json('../input/deepfake/metadata2.json')\ndf_train3 = pd.read_json('../input/deepfake/metadata3.json')\ndf_train4 = pd.read_json('../input/deepfake/metadata4.json')\ndf_train5 = pd.read_json('../input/deepfake/metadata5.json')\ndf_train6 = pd.read_json('../input/deepfake/metadata6.json')\ndf_train7 = pd.read_json('../input/deepfake/metadata7.json')\ndf_train8 = pd.read_json('../input/deepfake/metadata8.json')\ndf_train9 = pd.read_json('../input/deepfake/metadata9.json')\ndf_train10 = pd.read_json('../input/deepfake/metadata10.json')\ndf_train11 = pd.read_json('../input/deepfake/metadata11.json')\ndf_train12 = pd.read_json('../input/deepfake/metadata12.json')\ndf_train13 = pd.read_json('../input/deepfake/metadata13.json')\ndf_train14 = pd.read_json('../input/deepfake/metadata14.json')\ndf_train15 = pd.read_json('../input/deepfake/metadata15.json')\ndf_train16 = pd.read_json('../input/deepfake/metadata16.json')\ndf_train17 = pd.read_json('../input/deepfake/metadata17.json')\ndf_train18 = pd.read_json('../input/deepfake/metadata18.json')\ndf_train19 = pd.read_json('../input/deepfake/metadata19.json')\ndf_train20 = pd.read_json('../input/deepfake/metadata20.json')\ndf_train21 = pd.read_json('../input/deepfake/metadata21.json')\ndf_train22 = pd.read_json('../input/deepfake/metadata22.json')\ndf_train23 = pd.read_json('../input/deepfake/metadata23.json')\ndf_train24 = pd.read_json('../input/deepfake/metadata24.json')\ndf_train25 = pd.read_json('../input/deepfake/metadata25.json')\ndf_train26 = pd.read_json('../input/deepfake/metadata26.json')\ndf_train27 = pd.read_json('../input/deepfake/metadata27.json')\ndf_train28 = pd.read_json('../input/deepfake/metadata28.json')\ndf_train29 = pd.read_json('../input/deepfake/metadata29.json')\ndf_train30 = pd.read_json('../input/deepfake/metadata30.json')\ndf_train31 = pd.read_json('../input/deepfake/metadata31.json')\ndf_train32 = pd.read_json('../input/deepfake/metadata32.json')\ndf_train33 = pd.read_json('../input/deepfake/metadata33.json')\ndf_train34 = pd.read_json('../input/deepfake/metadata34.json')\ndf_train35 = pd.read_json('../input/deepfake/metadata35.json')\ndf_train36 = pd.read_json('../input/deepfake/metadata36.json')\ndf_train37 = pd.read_json('../input/deepfake/metadata37.json')\ndf_train38 = pd.read_json('../input/deepfake/metadata38.json')\ndf_train39 = pd.read_json('../input/deepfake/metadata39.json')\ndf_train40 = pd.read_json('../input/deepfake/metadata40.json')\ndf_train41 = pd.read_json('../input/deepfake/metadata41.json')\ndf_train42 = pd.read_json('../input/deepfake/metadata42.json')\ndf_train43 = pd.read_json('../input/deepfake/metadata43.json')\ndf_train44 = pd.read_json('../input/deepfake/metadata44.json')\ndf_train45 = pd.read_json('../input/deepfake/metadata45.json')\ndf_train46 = pd.read_json('../input/deepfake/metadata46.json')\ndf_val1 = pd.read_json('../input/deepfake/metadata47.json')\ndf_val2 = pd.read_json('../input/deepfake/metadata48.json')\ndf_val3 = pd.read_json('../input/deepfake/metadata49.json')\ndf_trains = [df_train0 ,df_train1, df_train2, df_train3, df_train4,\n             df_train5, df_train6, df_train7, df_train8, df_train9,df_train10,\n            df_train11, df_train12, df_train13, df_train14, df_train15,df_train16, \n            df_train17, df_train18, df_train19, df_train20, df_train21, df_train22, \n            df_train23, df_train24, df_train25, df_train26, df_train27, df_train28, \n            df_train29, df_train30, df_train31, df_train32, df_train33, df_train34,\n            df_train34, df_train35, df_train36, df_train37, df_train38, df_train39,\n            df_train40, df_train41, df_train42, df_train43, df_train44, df_train45,\n            df_train46]\ndf_vals=[df_val1, df_val2, df_val3]\nnums = list(range(len(df_trains)+1))\nLABELS = ['REAL','FAKE']\nval_nums=[47, 48, 49]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:38:15.923261Z","iopub.execute_input":"2024-10-13T06:38:15.923489Z","iopub.status.idle":"2024-10-13T06:39:03.803208Z","shell.execute_reply.started":"2024-10-13T06:38:15.923451Z","shell.execute_reply":"2024-10-13T06:39:03.802532Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Get Paths","metadata":{}},{"cell_type":"markdown","source":"Explanation:\nThe basic idea behind this is to get the path and y first, apply underbalancing, then read X. The reason why I did this in such way is to avoid large RAM usage.","metadata":{}},{"cell_type":"code","source":"#from pathlib import Path\n#for x in range(49):\n#      Path('/kaggle/working/DeepFake/DeepFake' + str(x) + \"/DeepFake\"+ str(x)).mkdir(parents=True, exist_ok=True)\n        \n\"\"\"       \n        \ndef get_path(num,x):\n    num=str(num)\n    if len(num)==2:\n        first_path = \"../input/deepfake/DeepFake/\"\n        second_path = \"/kaggle/working/DeepFake/\"\n        path=num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        #path='/kaggle/working/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        cv2.imwrite(\"/kaggle/working/DeepFake/DeepFake\"+path, cv2.resize(cv2.cvtColor(cv2.imread(\"../input/deepfake/DeepFake\"+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'),cv2.COLOR_BGR2RGB),(256,256)))\n\n    else:\n        path=num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        #print(\"/kaggle/working/DeepFake0\"+path)\n        #path='/kaggle/working/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n        cv2.imwrite(\"/kaggle/working/DeepFake/DeepFake0\"+path, cv2.resize(cv2.cvtColor(cv2.imread(\"../input/deepfake/DeepFake0\"+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'),cv2.COLOR_BGR2RGB),(256,256)))\n\n    if not os.path.exists(path):\n       raise Exception\n    return path\n\n\n\n\n\n\npaths=[]\ny=[]\nfor df_train,num in tqdm(zip(df_trains,nums),total=len(df_trains)):\n    images = list(df_train.columns.values)\n    for x in images:\n        try:\n            paths.append(get_path(num,x))\n            y.append(LABELS.index(df_train[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass\n\nval_paths=[]\nval_y=[]\nfor df_val,num in tqdm(zip(df_vals,val_nums),total=len(df_vals)):\n    images = list(df_val.columns.values)\n    for x in images:\n        try:\n            val_paths.append(get_path(num,x))\n            val_y.append(LABELS.index(df_val[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass\n            \n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:39:03.806077Z","iopub.execute_input":"2024-10-13T06:39:03.806344Z","iopub.status.idle":"2024-10-13T06:39:03.822623Z","shell.execute_reply.started":"2024-10-13T06:39:03.806292Z","shell.execute_reply":"2024-10-13T06:39:03.82189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_path(num,x):\n    num=str(num)\n    if len(num)==2:\n        path='../input/deepfake/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    else:\n        path='../input/deepfake/DeepFake0'+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    if not os.path.exists(path):\n       raise Exception\n    return path\npaths=[]\ny=[]\nfor df_train,num in tqdm(zip(df_trains,nums),total=len(df_trains)):\n    images = list(df_train.columns.values)\n    for x in images:\n        try:\n            paths.append(get_path(num,x))\n            y.append(LABELS.index(df_train[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass\n\nval_paths=[]\nval_y=[]\nfor df_val,num in tqdm(zip(df_vals,val_nums),total=len(df_vals)):\n    images = list(df_val.columns.values)\n    for x in images:\n        try:\n            val_paths.append(get_path(num,x))\n            val_y.append(LABELS.index(df_val[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:39:03.82429Z","iopub.execute_input":"2024-10-13T06:39:03.824599Z","iopub.status.idle":"2024-10-13T06:45:19.711012Z","shell.execute_reply.started":"2024-10-13T06:39:03.824542Z","shell.execute_reply":"2024-10-13T06:45:19.71028Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Apply Underbalancing Techinique","metadata":{}},{"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:45:19.712286Z","iopub.execute_input":"2024-10-13T06:45:19.712518Z","iopub.status.idle":"2024-10-13T06:45:19.720049Z","shell.execute_reply.started":"2024-10-13T06:45:19.712478Z","shell.execute_reply":"2024-10-13T06:45:19.719328Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The data is not balanced. We are going to use the undersampling technique.","metadata":{}},{"cell_type":"code","source":"import random\nreal=[]\nfake=[]\nfor m,n in zip(paths,y):\n    if n==0:\n        real.append(m)\n    else:\n        fake.append(m)\nfake=random.sample(fake,len(real))\npaths,y=[],[]\nfor x in real:\n    paths.append(x)\n    y.append(0)\nfor x in fake:\n    paths.append(x)\n    y.append(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:45:19.72153Z","iopub.execute_input":"2024-10-13T06:45:19.72178Z","iopub.status.idle":"2024-10-13T06:45:19.778672Z","shell.execute_reply.started":"2024-10-13T06:45:19.721712Z","shell.execute_reply":"2024-10-13T06:45:19.778059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"real=[]\nfake=[]\nfor m,n in zip(val_paths,val_y):\n    if n==0:\n        real.append(m)\n    else:\n        fake.append(m)\nfake=random.sample(fake,len(real))\nval_paths,val_y=[],[]\nfor x in real:\n    val_paths.append(x)\n    val_y.append(0)\nfor x in fake:\n    val_paths.append(x)\n    val_y.append(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:45:19.780072Z","iopub.execute_input":"2024-10-13T06:45:19.780446Z","iopub.status.idle":"2024-10-13T06:45:19.792461Z","shell.execute_reply.started":"2024-10-13T06:45:19.780319Z","shell.execute_reply":"2024-10-13T06:45:19.791829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:45:19.7937Z","iopub.execute_input":"2024-10-13T06:45:19.793955Z","iopub.status.idle":"2024-10-13T06:45:19.804077Z","shell.execute_reply.started":"2024-10-13T06:45:19.79391Z","shell.execute_reply":"2024-10-13T06:45:19.803431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, the data is balanced.","metadata":{}},{"cell_type":"markdown","source":"# Read Images","metadata":{}},{"cell_type":"code","source":"def read_img(path):\n    return cv2.cvtColor(cv2.imread(path),cv2.COLOR_BGR2RGB)\nX=[]\nfor img in tqdm(paths):\n    X.append(read_img(img))\nval_X=[]\nfor img in tqdm(val_paths):\n    val_X.append(read_img(img))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:45:19.805515Z","iopub.execute_input":"2024-10-13T06:45:19.805931Z","iopub.status.idle":"2024-10-13T06:49:49.141674Z","shell.execute_reply.started":"2024-10-13T06:45:19.805775Z","shell.execute_reply":"2024-10-13T06:49:49.14089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef shuffle(X,y):\n    new_train=[]\n    for m,n in zip(X,y):\n        new_train.append([m,n])\n    random.shuffle(new_train)\n    X,y=[],[]\n    for x in new_train:\n        X.append(x[0])\n        y.append(x[1])\n    return X,y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:49.143507Z","iopub.execute_input":"2024-10-13T06:49:49.143874Z","iopub.status.idle":"2024-10-13T06:49:49.152054Z","shell.execute_reply.started":"2024-10-13T06:49:49.143812Z","shell.execute_reply":"2024-10-13T06:49:49.151254Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Shuffle**","metadata":{}},{"cell_type":"code","source":"X,y=shuffle(X,y)\nval_X,val_y=shuffle(val_X,val_y)\n\n#print(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:49.153801Z","iopub.execute_input":"2024-10-13T06:49:49.154126Z","iopub.status.idle":"2024-10-13T06:49:49.227083Z","shell.execute_reply.started":"2024-10-13T06:49:49.154071Z","shell.execute_reply":"2024-10-13T06:49:49.226476Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"def InceptionLayer(a, b, c, d):\n    def func(x):\n        x1 = Conv2D(a, (1, 1), padding='same', activation='elu')(x)\n        \n        x2 = Conv2D(b, (1, 1), padding='same', activation='elu')(x)\n        x2 = Conv2D(b, (3, 3), padding='same', activation='elu')(x2)\n            \n        x3 = Conv2D(c, (1, 1), padding='same', activation='elu')(x)\n        x3 = Conv2D(c, (3, 3), dilation_rate = 2, strides = 1, padding='same', activation='elu')(x3)\n        \n        x4 = Conv2D(d, (1, 1), padding='same', activation='elu')(x)\n        x4 = Conv2D(d, (3, 3), dilation_rate = 3, strides = 1, padding='same', activation='elu')(x4)\n        y = Concatenate(axis = -1)([x1, x2, x3, x4])\n            \n        return y\n    return func\n    \ndef define_model(shape=(256,256,3)):\n    x = Input(shape = shape)\n    \n    x1 = InceptionLayer(1, 4, 4, 2)(x)\n    x1 = BatchNormalization()(x1)\n    x1 = MaxPooling2D(pool_size=(2, 2), padding='same')(x1)\n    \n    x2 = InceptionLayer(2, 4, 4, 2)(x1)\n    x2 = BatchNormalization()(x2)        \n    x2 = MaxPooling2D(pool_size=(2, 2), padding='same')(x2)        \n        \n    x3 = Conv2D(16, (3, 3), padding='same', activation = 'elu')(x2)\n    x3 = BatchNormalization()(x3)\n    x3 = MaxPooling2D(pool_size=(2, 2), padding='same')(x3)\n        \n    x4 = Conv2D(16, (3, 3), padding='same', activation = 'elu')(x3)\n    x4 = BatchNormalization()(x4)\n    x4 = MaxPooling2D(pool_size=(2, 2), padding='same')(x4)\n\n    x5 = Conv2D(32, (5, 5), padding='same', activation = 'elu')(x4)\n    x5 = BatchNormalization()(x5)\n    x5 = MaxPooling2D(pool_size=(2, 2), padding='same')(x5)\n    \n    y = Flatten()(x4)\n    y = Dropout(0.5)(y)\n    y = Dense(16)(y)\n    y = LeakyReLU(alpha=0.1)(y)\n    y = Dropout(0.5)(y)\n    y = Dense(1, activation = 'sigmoid')(y)\n    model=Model(inputs = x, outputs = y)\n    model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-4))\n    #model.summary()\n    return model\ndf_model=define_model()\n#df_model.load_weights('../input/meso-pretrain/MesoInception_DF')\nf2f_model=define_model()\n#f2f_model.load_weights('../input/meso-pretrain/MesoInception_F2F')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:49.228572Z","iopub.execute_input":"2024-10-13T06:49:49.228837Z","iopub.status.idle":"2024-10-13T06:49:56.266403Z","shell.execute_reply.started":"2024-10-13T06:49:49.228791Z","shell.execute_reply":"2024-10-13T06:49:56.265438Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This model is the almost same as MesoNet.","metadata":{}},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import LearningRateScheduler\n\n# we list out > 25 LRs\nlrs=[1e-2,9e-3,8e-3,7e-3,6e-3,5e-3,4e-3,3e-3,2e-3,\n     1e-3,9e-4,8e-4,7e-4,6e-4,5e-4,4e-4,3e-4,2e-4,\n     1e-4,9e-5,8e-5,7e-5,6e-5,5e-5,4e-5,3e-5,2e-5,\n     1e-5,9e-6,8e-6,7e-6,6e-6,5e-6,4e-6,3e-6,2e-6,\n     1e-6,9e-7,8e-7,7e-7,6e-7,5e-7,4e-7,3e-7,2e-7,\n     1e-7,9e-8,8e-8,7e-8,6e-8,5e-8,4e-8,3e-8,2e-8]\ndef schedule(epoch):\n    return lrs[epoch]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:56.26774Z","iopub.execute_input":"2024-10-13T06:49:56.268038Z","iopub.status.idle":"2024-10-13T06:49:56.275486Z","shell.execute_reply.started":"2024-10-13T06:49:56.267985Z","shell.execute_reply":"2024-10-13T06:49:56.274623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LOAD_PRETRAIN = False # originally True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:56.276835Z","iopub.execute_input":"2024-10-13T06:49:56.277141Z","iopub.status.idle":"2024-10-13T06:49:56.28539Z","shell.execute_reply.started":"2024-10-13T06:49:56.277087Z","shell.execute_reply":"2024-10-13T06:49:56.284845Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train for 50 epochs","metadata":{}},{"cell_type":"code","source":"import gc\nkfolds=5\nlosses=[]\nif LOAD_PRETRAIN:\n    # import keras.backend as K\n    df_models=[]\n    f2f_models=[]\n    i=0\n    while len(df_models)<kfolds:\n        model=define_model((150,150,3))\n        if i==0:\n            model.summary()\n        #model.load_weights('../input/meso-pretrain/MesoInception_DF')\n        for new_layer, layer in zip(model.layers[1:-8], df_model.layers[1:-8]):\n            new_layer.set_weights(layer.get_weights())\n        model.fit([X],[y],epochs=25,callbacks=[LearningRateScheduler(schedule)]) #original 2 epochs\n        pred=model.predict([val_X])\n        loss=log_loss(val_y,pred)\n        losses.append(loss)\n        print('fold '+str(i)+' model loss: '+str(loss))\n        df_models.append(model)\n        K.clear_session()\n        del model\n        gc.collect()\n        i+=1\n    i=0\n    while len(f2f_models)<kfolds:\n        model=define_model((150,150,3))\n        #model.load_weights('../input/meso-pretrain/MesoInception_DF')\n        for new_layer, layer in zip(model.layers[1:-8], f2f_model.layers[1:-8]):\n            new_layer.set_weights(layer.get_weights())\n        model.fit([X],[y],epochs=25,callbacks=[LearningRateScheduler(schedule)]) #original 2 epochs\n        pred=model.predict([val_X])\n        loss=log_loss(val_y,pred)\n        losses.append(loss)\n        print('fold '+str(i)+' model loss: '+str(loss))\n        f2f_models.append(model)\n        K.clear_session()\n        del model\n        gc.collect()\n        i+=1\n        models=f2f_models+df_models\nelse:\n    models=[]\n    i=0\n    while len(models)<kfolds:\n        model=define_model((150,150,3))\n        if i==0:\n            model.summary()\n        model.fit([X],[y],epochs=50,callbacks=[LearningRateScheduler(schedule)]) #original 2 epochs\n        pred=model.predict([val_X])\n        loss=log_loss(val_y,pred)\n        losses.append(loss)\n        print('fold '+str(i)+' model loss: '+str(loss))\n        if loss<0.68:\n            models.append(model)\n        else:\n            print('loss too bad, retrain!')\n        K.clear_session()\n        del model\n        gc.collect()\n        i+=1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T06:49:56.286974Z","iopub.execute_input":"2024-10-13T06:49:56.287278Z","iopub.status.idle":"2024-10-13T09:27:05.937342Z","shell.execute_reply.started":"2024-10-13T06:49:56.287224Z","shell.execute_reply":"2024-10-13T09:27:05.936505Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Explanation:\nBecause of the smaller input size, this code:\n```\n    for new_layer, layer in zip(model.layers[1:-8], f2f_model.layers[1:-8]):\n        new_layer.set_weights(layer.get_weights())\n```\nfetches only the conv layers weight and apply it onto our model.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction_pipline(X,two_times=False):\n    preds=[]\n    for model in tqdm(models):\n        pred=model.predict([X])\n        preds.append(pred)\n    preds=sum(preds)/len(preds)\n    if two_times:\n        return larger_range(preds,2)\n    else:\n        return preds\ndef larger_range(model_pred,time):\n    return (((model_pred-0.5)*time)+0.5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:17.920913Z","iopub.execute_input":"2024-10-13T09:31:17.921214Z","iopub.status.idle":"2024-10-13T09:31:17.928359Z","shell.execute_reply.started":"2024-10-13T09:31:17.921173Z","shell.execute_reply":"2024-10-13T09:31:17.927434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model_pred=models[losses.index(min(losses))].predict([val_X])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:23.660882Z","iopub.execute_input":"2024-10-13T09:31:23.661317Z","iopub.status.idle":"2024-10-13T09:31:24.971326Z","shell.execute_reply.started":"2024-10-13T09:31:23.661127Z","shell.execute_reply":"2024-10-13T09:31:24.970634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_pred=prediction_pipline(val_X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:27.476238Z","iopub.execute_input":"2024-10-13T09:31:27.476553Z","iopub.status.idle":"2024-10-13T09:31:33.634865Z","shell.execute_reply.started":"2024-10-13T09:31:27.476497Z","shell.execute_reply":"2024-10-13T09:31:33.634079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.models import Model as KerasModel\nfrom keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, BatchNormalization, Dropout, Reshape, Concatenate, LeakyReLU\nfrom keras.optimizers import Adam\n\n#model_pred = Meso4()\n#classifier.load('/kaggle/input/meso-pretrain/Meso4_DF')\n#classifier.load('/kaggle/input/v2/keras/default/1/oleh_w31.h5')\n\n#model_pred = MesoInception4()\n#model_pred.load('/kaggle/input/v/keras/default/1/oleh_w32.h5')\n\nmodels = MesoInception4()\n#models.load('/kaggle/input/v/keras/default/1/oleh_w32.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:35.024427Z","iopub.execute_input":"2024-10-13T09:31:35.024709Z","iopub.status.idle":"2024-10-13T09:31:35.052965Z","shell.execute_reply.started":"2024-10-13T09:31:35.024664Z","shell.execute_reply":"2024-10-13T09:31:35.051685Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Some baselines:","metadata":{}},{"cell_type":"code","source":"random_pred=np.random.random(len(val_X))\nprint('random loss: ' + str(log_loss(val_y,random_pred.clip(0.35,0.65))))\nallone_pred=np.array([1 for _ in range(len(val_X))])\nprint('1 loss: ' + str(log_loss(val_y,allone_pred)))\nallzero_pred=np.array([0 for _ in range(len(val_X))])\nprint('0 loss: ' + str(log_loss(val_y,allzero_pred)))\nallpoint5_pred=np.array([0.5 for _ in range(len(val_X))])\nprint('0.5 loss: ' + str(log_loss(val_y,allpoint5_pred)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:49.918885Z","iopub.execute_input":"2024-10-13T09:31:49.919215Z","iopub.status.idle":"2024-10-13T09:31:49.948905Z","shell.execute_reply.started":"2024-10-13T09:31:49.919169Z","shell.execute_reply":"2024-10-13T09:31:49.948016Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Model loss","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Simple Averaging Loss: '+str(log_loss(val_y,model_pred.clip(0.35,0.65))))\nprint('Two Times Larger Range(Averaging) Loss: '+str(log_loss(val_y,larger_range(model_pred,2).clip(0.35,0.65))))\nprint('Best Single Model Loss: '+str(log_loss(val_y,best_model_pred.clip(0.35,0.65))))\nprint('Two Times Larger Range(Single Model) Loss: '+str(log_loss(val_y,larger_range(best_model_pred,2).clip(0.35,0.65))))\nif log_loss(val_y,model_pred.clip(0.35,0.65))<log_loss(val_y,larger_range(model_pred,2).clip(0.35,0.65)):\n    two_times=False\n    print('simple averaging is better')\nelse:\n    two_times=True\n    print('two times larger range is better')\ntwo_times=False #This is not a bug. I did this intentionally because the model can't get most of the private validation set right(based on LB)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:53.387282Z","iopub.execute_input":"2024-10-13T09:31:53.387569Z","iopub.status.idle":"2024-10-13T09:31:53.424873Z","shell.execute_reply.started":"2024-10-13T09:31:53.387521Z","shell.execute_reply":"2024-10-13T09:31:53.424081Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to validation loss, averaging is better than best single model.","metadata":{}},{"cell_type":"markdown","source":"Take a look at predictions","metadata":{}},{"cell_type":"code","source":"import scipy\nprint(model_pred.clip(0.35,0.65).mean())\nprint(scipy.stats.median_absolute_deviation(model_pred.clip(0.35,0.65))[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:31:58.662319Z","iopub.execute_input":"2024-10-13T09:31:58.662612Z","iopub.status.idle":"2024-10-13T09:31:58.669432Z","shell.execute_reply.started":"2024-10-13T09:31:58.662567Z","shell.execute_reply":"2024-10-13T09:31:58.668477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_answers(pred,real,num):\n    for i,(x,y) in enumerate(zip(pred,real)):\n        correct_incorrect='correct ✅ ' if round(float(x),0)==round(float(y),0) else 'incorrect❌'\n        print(correct_incorrect+' prediction: '+str(x[0])+', answer: '+str(y))\n        if i>num:\n            return\ndef correct_precentile(pred,real):\n    correct=0\n    incorrect=0\n    for x,y in zip(pred,real):\n        if round(float(x),0)==round(float(y),0):\n            correct+=1\n        else:\n            incorrect+=1\n    print('number correct: '+str(correct)+', number incorrect: '+str(incorrect))\n    print(str(round(correct/len(real)*100,1))+'% correct'+', '+str(round(incorrect/len(real)*100,1))+'% incorrect')   \ncheck_answers(model_pred,val_y,15)\ncorrect_precentile(model_pred,val_y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:32:01.921868Z","iopub.execute_input":"2024-10-13T09:32:01.922166Z","iopub.status.idle":"2024-10-13T09:32:01.939513Z","shell.execute_reply.started":"2024-10-13T09:32:01.922122Z","shell.execute_reply":"2024-10-13T09:32:01.938802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del X,y,val_X,val_y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:32:12.194958Z","iopub.execute_input":"2024-10-13T09:32:12.195297Z","iopub.status.idle":"2024-10-13T09:32:12.263116Z","shell.execute_reply.started":"2024-10-13T09:32:12.195236Z","shell.execute_reply":"2024-10-13T09:32:12.262195Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Make submission","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"MAX_SKIP=10\nNUM_FRAME=256\ntest_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nfilenames = os.listdir(test_dir)\nprediction_filenames = filenames\ntest_video_files = [test_dir + x for x in filenames]\ndetector = MTCNN()\ndef detect_face(img):\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    final = []\n    detected_faces_raw = detector.detect_faces(img)\n    if detected_faces_raw==[]:\n        #print('no faces found')\n        return []\n    confidences=[]\n    for n in detected_faces_raw:\n        x,y,w,h=n['box']\n        final.append([x,y,w,h])\n        confidences.append(n['confidence'])\n    if max(confidences)<0.9:\n        return []\n    max_conf_coord=final[confidences.index(max(confidences))]\n    #return final\n    return max_conf_coord\ndef crop(img,x,y,w,h):\n    x-=40\n    y-=40\n    w+=80\n    h+=80\n    if x<0:\n        x=0\n    if y<=0:\n        y=0\n    return cv2.cvtColor(cv2.resize(img[y:y+h,x:x+w],(256,256)),cv2.COLOR_BGR2RGB)\ndef detect_video(video):\n    v_cap = cv2.VideoCapture(video)\n    v_cap.set(1, NUM_FRAME)\n    success, vframe = v_cap.read()\n    vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n    bounding_box=detect_face(vframe)\n    if bounding_box==[]:\n        count=0\n        current=NUM_FRAME\n        while bounding_box==[] and count<MAX_SKIP:\n            current+=1\n            v_cap.set(1,current)\n            success, vframe = v_cap.read()\n            vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n            bounding_box=detect_face(vframe)\n            count+=1\n        if bounding_box==[]:\n            print('no faces found')\n            prediction_filenames.remove(video.replace('/kaggle/input/deepfake-detection-challenge/test_videos/',''))\n            return None\n    x,y,w,h=bounding_box\n    v_cap.release()\n    return crop(vframe,x,y,w,h)\ntest_X = []\nfor video in tqdm(test_video_files):\n    x=detect_video(video)\n    if x is None:\n        continue\n    test_X.append(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:32:16.194923Z","iopub.execute_input":"2024-10-13T09:32:16.195236Z","iopub.status.idle":"2024-10-13T09:50:22.701552Z","shell.execute_reply.started":"2024-10-13T09:32:16.195182Z","shell.execute_reply":"2024-10-13T09:50:22.700778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test=pd.read_csv('/kaggle/input/deepfake-detection-challenge/sample_submission.csv')\ndf_test['label']=0.5\npreds=prediction_pipline(test_X,False).clip(0.35,0.65)\nfor pred,name in zip(preds,prediction_filenames):\n    name=name.replace('/kaggle/input/deepfake-detection-challenge/test_videos/','')\n    df_test.iloc[list(df_test['filename']).index(name),1]=pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:27:13.617675Z","iopub.status.idle":"2024-10-13T09:27:13.618325Z","shell.execute_reply":"2024-10-13T09:27:13.618042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(preds.clip(0.35,0.65).mean())\nprint(scipy.stats.median_absolute_deviation(preds.clip(0.35,0.65))[0])\nprint(preds[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:27:13.619532Z","iopub.status.idle":"2024-10-13T09:27:13.620039Z","shell.execute_reply":"2024-10-13T09:27:13.61978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:27:13.621131Z","iopub.status.idle":"2024-10-13T09:27:13.621625Z","shell.execute_reply":"2024-10-13T09:27:13.621353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.to_csv('submission1.csv',index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-13T09:27:13.622974Z","iopub.status.idle":"2024-10-13T09:27:13.623516Z","shell.execute_reply":"2024-10-13T09:27:13.623259Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Further Work\n1. Do some hyperparamater tuning\n2. Train on the whole video(and maybe also sound)\n3. Try LSTM-CNN","metadata":{}},{"cell_type":"markdown","source":"If you have any questions about the code or found a possible bug, please comment below. Cheers.","metadata":{}}]}