{"cells":[{"metadata":{},"cell_type":"markdown","source":"this originally was just a fork from a kernel https://www.kaggle.com/unkownhihi/starter-kernel-with-cnn-ll-lb-0-69203-no-leak\n"},{"metadata":{},"cell_type":"markdown","source":"Code for generating dataset:"},{"metadata":{},"cell_type":"markdown","source":"# Install MTCNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/mtcnn-package/mtcnn-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport keras\nimport os\nimport numpy as np\nfrom sklearn.metrics import log_loss\nfrom keras import Model,Sequential\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom tqdm.notebook import tqdm\nimport glob\n#from mtcnn import MTCNN","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Train Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train0 = pd.read_json('../input/deepfake/metadata0.json')\ndf_train1 = pd.read_json('../input/deepfake/metadata1.json')\ndf_train2 = pd.read_json('../input/deepfake/metadata2.json')\ndf_train3 = pd.read_json('../input/deepfake/metadata3.json')\ndf_train4 = pd.read_json('../input/deepfake/metadata4.json')\ndf_train5 = pd.read_json('../input/deepfake/metadata5.json')\ndf_train6 = pd.read_json('../input/deepfake/metadata6.json')\ndf_train7 = pd.read_json('../input/deepfake/metadata7.json')\ndf_train8 = pd.read_json('../input/deepfake/metadata8.json')\ndf_train9 = pd.read_json('../input/deepfake/metadata9.json')\ndf_train10 = pd.read_json('../input/deepfake/metadata10.json')\ndf_train11 = pd.read_json('../input/deepfake/metadata11.json')\ndf_train12 = pd.read_json('../input/deepfake/metadata12.json')\ndf_train13 = pd.read_json('../input/deepfake/metadata13.json')\ndf_train14 = pd.read_json('../input/deepfake/metadata14.json')\ndf_train15 = pd.read_json('../input/deepfake/metadata15.json')\ndf_train16 = pd.read_json('../input/deepfake/metadata16.json')\ndf_train17 = pd.read_json('../input/deepfake/metadata17.json')\ndf_train18 = pd.read_json('../input/deepfake/metadata18.json')\ndf_train19 = pd.read_json('../input/deepfake/metadata19.json')\ndf_train20 = pd.read_json('../input/deepfake/metadata20.json')\ndf_train21 = pd.read_json('../input/deepfake/metadata21.json')\ndf_train22 = pd.read_json('../input/deepfake/metadata22.json')\ndf_train23 = pd.read_json('../input/deepfake/metadata23.json')\ndf_train24 = pd.read_json('../input/deepfake/metadata24.json')\ndf_train25 = pd.read_json('../input/deepfake/metadata25.json')\ndf_train26 = pd.read_json('../input/deepfake/metadata26.json')\ndf_train27 = pd.read_json('../input/deepfake/metadata27.json')\ndf_train28 = pd.read_json('../input/deepfake/metadata28.json')\ndf_train29 = pd.read_json('../input/deepfake/metadata29.json')\ndf_train30 = pd.read_json('../input/deepfake/metadata30.json')\ndf_train31 = pd.read_json('../input/deepfake/metadata31.json')\ndf_train32 = pd.read_json('../input/deepfake/metadata32.json')\ndf_train33 = pd.read_json('../input/deepfake/metadata33.json')\ndf_train34 = pd.read_json('../input/deepfake/metadata34.json')\ndf_train35 = pd.read_json('../input/deepfake/metadata35.json')\ndf_train36 = pd.read_json('../input/deepfake/metadata36.json')\ndf_train37 = pd.read_json('../input/deepfake/metadata37.json')\ndf_train38 = pd.read_json('../input/deepfake/metadata38.json')\ndf_train39 = pd.read_json('../input/deepfake/metadata39.json')\ndf_train40 = pd.read_json('../input/deepfake/metadata40.json')\ndf_train41 = pd.read_json('../input/deepfake/metadata41.json')\ndf_train42 = pd.read_json('../input/deepfake/metadata42.json')\ndf_train43 = pd.read_json('../input/deepfake/metadata43.json')\ndf_train44 = pd.read_json('../input/deepfake/metadata44.json')\ndf_train45 = pd.read_json('../input/deepfake/metadata45.json')\ndf_train46 = pd.read_json('../input/deepfake/metadata46.json')\ndf_val1 = pd.read_json('../input/deepfake/metadata47.json')\ndf_val2 = pd.read_json('../input/deepfake/metadata48.json')\ndf_val3 = pd.read_json('../input/deepfake/metadata49.json')\ndf_trains = [df_train0 ,df_train1, df_train2, df_train3, df_train4,\n df_train5, df_train6, df_train7, df_train8, df_train9,df_train10,\n df_train11, df_train12, df_train13, df_train14, df_train15,df_train16, \n df_train17, df_train18, df_train19, df_train20, df_train21, df_train22, \n df_train23, df_train24, df_train25, df_train26, df_train27, df_train28, \n df_train29, df_train30, df_train31, df_train32, df_train33, df_train34,\n df_train34, df_train35, df_train36, df_train37, df_train38, df_train39,\n df_train40, df_train41, df_train42, df_train43, df_train44, df_train45,\n df_train46]\ndf_vals=[df_val1, df_val2, df_val3]\nnums = list(range(len(df_trains)+1))\nLABELS = ['REAL','FAKE']\nval_nums=[47, 48, 49]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get Paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_path(num,n):\n num=str(num)\n if len(num)==2:\n path='../input/deepfake/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n else:\n path='../input/deepfake/DeepFake0'+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n if not os.path.exists(path):\n raise Exception\n return path\npaths=[]\ny=[]\nfor df_train,num in tqdm(zip(df_trains,nums),total=len(df_trains)):\n images = list(df_train.columns.values)\n for x in images:\n try:\n paths.append(get_path(num,x))\n y.append(LABELS.index(df_train[x]['label']))\n except Exception as err:\n #print(err)\n pass\n\nval_paths=[]\nval_y=[]\nfor df_val,num in tqdm(zip(df_vals,val_nums),total=len(df_vals)):\n images = list(df_val.columns.values)\n for x in images:\n try:\n val_paths.append(get_path(num,x))\n val_y.append(LABELS.index(df_val[x]['label']))\n except Exception as err:\n #print(err)\n pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Apply Underbalancing Techinique"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The data is not balanced. We are going to use the undersampling technique."},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nreal=[]\nfake=[]\nfor m,n in zip(paths,y):\n if n==0:\n real.append(m)\n else:\n fake.append(m)\nfake=random.sample(fake,len(real))\npaths,y=[],[]\nfor x in real:\n paths.append(x)\n y.append(0)\nfor x in fake:\n paths.append(x)\n y.append(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real=[]\nfake=[]\nfor m,n in zip(val_paths,val_y):\n if n==0:\n real.append(m)\n else:\n fake.append(m)\nfake=random.sample(fake,len(real))\nval_paths,val_y=[],[]\nfor x in real:\n val_paths.append(x)\n val_y.append(0)\nfor x in fake:\n val_paths.append(x)\n val_y.append(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, the data is balanced."},{"metadata":{},"cell_type":"markdown","source":"# Read Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(path):\n return cv2.cvtColor(cv2.imread(path),cv2.COLOR_BGR2RGB)\nX=[]\nfor img in tqdm(paths):\n X.append(read_img(img))\nval_X=[]\nfor img in tqdm(val_paths):\n val_X.append(read_img(img))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndef shuffle(X,y):\n new_train=[]\n for m,n in zip(X,y):\n new_train.append([m,n])\n random.shuffle(new_train)\n X,y=[],[]\n for x in new_train:\n X.append(x[0])\n y.append(x[1])\n return X,y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def InceptionLayer(a, b, c, d):\n def func(x):\n x1 = Conv2D(a, (1, 1), padding='same', activation='relu')(x)\n \n x2 = Conv2D(b, (1, 1), padding='same', activation='relu')(x)\n x2 = Conv2D(b, (3, 3), padding='same', activation='relu')(x2)\n \n x3 = Conv2D(c, (1, 1), padding='same', activation='relu')(x)\n x3 = Conv2D(c, (3, 3), dilation_rate = 2, strides = 1, padding='same', activation='relu')(x3)\n \n x4 = Conv2D(d, (1, 1), padding='same', activation='relu')(x)\n x4 = Conv2D(d, (3, 3), dilation_rate = 3, strides = 1, padding='same', activation='relu')(x4)\n y = Concatenate(axis = -1)([x1, x2, x3, x4])\n \n return y\n return func\n \ndef define_model():\n x = Input(shape = (150, 150, 3))\n \n x1 = InceptionLayer(1, 4, 4, 2)(x)\n x1 = BatchNormalization()(x1)\n x1 = MaxPooling2D(pool_size=(2, 2), padding='same')(x1)\n \n x2 = InceptionLayer(2, 4, 4, 2)(x1)\n x2 = BatchNormalization()(x2) \n x2 = MaxPooling2D(pool_size=(2, 2), padding='same')(x2)\n \n x2 = InceptionLayer(2, 4, 4, 2)(x2)\n x2 = BatchNormalization()(x2) \n x2 = MaxPooling2D(pool_size=(2, 2), padding='same')(x2) \n \n x3 = Conv2D(16, (5, 5), padding='same', activation = 'relu')(x2)\n x3 = BatchNormalization()(x3)\n x3 = MaxPooling2D(pool_size=(2, 2), padding='same')(x3)\n \n #x4 = Dense(128)(x4)\n #x4 = Dense(64)(x4)\n \n x4 = Conv2D(16, (5, 5), padding='same', activation = 'relu')(x3)\n x4 = BatchNormalization()(x4)\n x4 = MaxPooling2D(pool_size = 4, padding='same')(x4)\n \n x4 = Reshape((16,16))(x4)\n x4 = LSTM(2048, return_sequences=False)(x4)\n #x4 = LSTM(1024)(x4)\n \n #y = Flatten()(x4)\n y = Dropout(0.5)(x4)\n y = Dense(32)(y)\n y = Dense(16)(y)\n y = LeakyReLU(alpha=0.1)(y)\n y = Dropout(0.5)(y)\n #y = Reshape((3,3,16))(y)\n y = Dense(1, activation = 'sigmoid')(y)\n model=Model(inputs = x, outputs = y)\n model.compile(loss='mean_squared_error',optimizer=Adam(lr=2e-4))\n model.summary()\n return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This model is the same as MesoNet."},{"metadata":{},"cell_type":"markdown","source":"# Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model=define_model()\nmodel.summary()\n#model.load_weights('../input/meso-pretrain/MesoInception_F2F')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit([X],[y],epochs=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check Validation Log Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"answer=[LABELS[n] for n in val_y]\npred=np.random.random(len(val_X))\nprint('random loss: ' + str(log_loss(answer,pred.clip(0.45,0.65))))\npred=np.array([1 for _ in range(len(val_X))])\nprint('1 loss: ' + str(log_loss(answer,pred)))\npred=np.array([0 for _ in range(len(val_X))])\nprint('0 loss: ' + str(log_loss(answer,pred)))\npred=np.array([0.5 for _ in range(len(val_X))])\nprint('0.5 loss: ' + str(log_loss(answer,pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=model.predict([val_X])\nprint('model loss: '+str(log_loss(answer,pred)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take a look at predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(pred.mean())\nprint(pred.std())\nprint(pred[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SKIP=10\nNUM_FRAME=150\ntest_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nfilenames = os.listdir(test_dir)\nprediction_filenames = filenames\ntest_video_files = [test_dir + x for x in filenames]\ndetector = MTCNN()\ndef detect_face(img):\n img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n final = []\n detected_faces_raw = detector.detect_faces(img)\n if detected_faces_raw==[]:\n #print('no faces found')\n return []\n confidences=[]\n for n in detected_faces_raw:\n x,y,w,h=n['box']\n final.append([x,y,w,h])\n confidences.append(n['confidence'])\n if max(confidences)<0.7:\n return []\n max_conf_coord=final[confidences.index(max(confidences))]\n #return final\n return max_conf_coord\ndef crop(img,x,y,w,h):\n x-=40\n y-=40\n w+=80\n h+=80\n if x<0:\n x=0\n if y<=0:\n y=0\n return cv2.cvtColor(cv2.resize(img[y:y+h,x:x+w],(256,256)),cv2.COLOR_BGR2RGB)\ndef detect_video(video):\n v_cap = cv2.VideoCapture(video)\n v_cap.set(1, NUM_FRAME)\n success, vframe = v_cap.read()\n vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n bounding_box=detect_face(vframe)\n if bounding_box==[]:\n count=0\n current=NUM_FRAME\n while bounding_box==[] and count