{"cells":[{"metadata":{},"cell_type":"markdown","source":"We started in last five lays and landed onto bronze. Thanks to [Harshit](https://www.kaggle.com/harshitsheoran) and [Shangqiu Li](https://www.kaggle.com/unkownhihi) for their kernels and opensourcing their approaches.<br><br>\nThis kernel shows even if you start in the end, there is a scope to get something."},{"metadata":{},"cell_type":"markdown","source":"<h3> Approach</h3>\n+ [Sampling and balancing](https://www.kaggle.com/c/deepfake-detection-challenge/discussion/132700#759482) + Flipping the frames\n+ [Training](https://www.kaggle.com/unkownhihi/dfdc-lrcn-training)\n+ [Inference](https://www.kaggle.com/unkownhihi/dfdc-lrcn-inference)\n\nWe trained multiple models (same data and same procedure) and took their average prediction as output.<br><br>\nThe hope is optimizing some hyperparameters may get you much better scores."},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nsuper_start = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/efficientnet/efficientnet-1.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nimport cv2\nimport glob\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport os\nimport efficientnet.keras as efn\nfrom keras.layers import *\nfrom keras import Model\nimport matplotlib.pyplot as plt\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialize Face Extractor"},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_graph = tf.Graph()\nwith detection_graph.as_default():\n    od_graph_def = tf.compat.v1.GraphDef()\n    with tf.io.gfile.GFile('../input/mobilenet-face/frozen_inference_graph_face.pb', 'rb') as fid:\n        serialized_graph = fid.read()\n        od_graph_def.ParseFromString(serialized_graph)\n        tf.import_graph_def(od_graph_def, name='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = detection_graph.as_default()\ncm.__enter__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsess=tf.compat.v1.Session(graph=detection_graph, config=config)\nimage_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\nboxes_tensor = detection_graph.get_tensor_by_name('detection_boxes:0')\nscores_tensor = detection_graph.get_tensor_by_name('detection_scores:0')\nnum_detections = detection_graph.get_tensor_by_name('num_detections:0')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffb09dc9-4297-4768-97e3-a9a9a8f37fe0","_cell_guid":"ef1c1fd7-b763-4a40-a5c0-524bfbc503e0","trusted":true},"cell_type":"code","source":"def get_img(images):\n    global boxes,scores,num_detections\n    im_heights,im_widths=[],[]\n    imgs=[]\n    for image in images:\n        (im_height,im_width)=image.shape[:-1]\n        imgs.append(image)\n        im_heights.append(im_height)\n        im_widths.append(im_widths)\n    imgs=np.array(imgs)\n    (boxes, scores_) = sess.run(\n        [boxes_tensor, scores_tensor],\n        feed_dict={image_tensor: imgs})\n    finals=[]\n    for x in range(boxes.shape[0]):\n        scores=scores_[x]\n        max_=np.where(scores==scores.max())[0][0]\n        box=boxes[x][max_]\n        ymin, xmin, ymax, xmax = box\n        (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n                                      ymin * im_height, ymax * im_height)\n        left, right, top, bottom = int(left), int(right), int(top), int(bottom)\n        image=imgs[x]\n        finals.append(cv2.cvtColor(cv2.resize(image[max([0,top-40]):bottom+80,max([0,left-40]):right+80],(240,240)),cv2.COLOR_BGR2RGB))\n    return finals\ndef detect_video(video, start_frame):\n    frame_count=10\n    capture = cv2.VideoCapture(video)\n    v_len = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_idxs = np.linspace(start_frame,v_len,frame_count, endpoint=False, dtype=np.int)\n    imgs=[]\n    i=0\n    for frame_idx in range(int(v_len)):\n        ret = capture.grab()\n        if not ret: \n            print(\"Error grabbing frame %d from movie %s\" % (frame_idx, video))\n        if frame_idx >= frame_idxs[i]:\n            if frame_idx-frame_idxs[i]>20:\n                return None\n            ret, frame = capture.retrieve()\n            if not ret or frame is None:\n                print(\"Error retrieving frame %d from movie %s\" % (frame_idx, video))\n            else:\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                imgs.append(frame)\n            i += 1\n            if i >= len(frame_idxs):\n                break\n    imgs=get_img(imgs)\n    if len(imgs)<10:\n        return None\n    return np.hstack(imgs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('./videos/')\nfor x in tqdm(glob.glob('../input/deepfake-detection-challenge/test_videos/*.mp4')):\n    try:\n        filename=x.replace('../input/deepfake-detection-challenge/test_videos/','').replace('.mp4','.jpg')\n        a=detect_video(x,0)\n        if a is None:\n            continue\n        cv2.imwrite('./videos/'+filename,a)\n    except Exception as err:\n        print(err)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('./videos_2/')\nfor x in tqdm(glob.glob('../input/deepfake-detection-challenge/test_videos/*.mp4')):\n    try:\n        filename=x.replace('../input/deepfake-detection-challenge/test_videos/','').replace('.mp4','.jpg')\n        a=detect_video(x,95)\n        if a is None:\n            continue\n        cv2.imwrite('./videos_2/'+filename,a)\n    except Exception as err:\n        print(err)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm.__exit__(None,Exception,'exit')\nsess.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialize Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"bottleneck = efn.EfficientNetB1(weights=None,include_top=False,pooling='avg')\ninp=Input((10,240,240,3))\nx=TimeDistributed(bottleneck)(inp)\nx = LSTM(128)(x)\nx = Dense(64, activation='elu')(x)\nx = Dense(1,activation='sigmoid')(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=Model(inp,x)\n\nweights = ['../input/deepfake-20/saved-model-01-0.06.hdf5', '../input/deepfake-20/saved-model-02-0.05.hdf5', '../input/model-epoch-3/saved-model-03-0.06.hdf5','../input/model-02/saved-model-01-0.06.hdf5']*2\n\nsub_file = ['submission_'+str(i)+'.csv' for i in range(1,9)]\n\nvideo = ['./videos/']*4+['./videos_2/']*4\n\nfor xxxxx in range(8):\n    start = time.time()\n    model.load_weights(weights[xxxxx])\n\n    def get_birghtness(img):\n        return img/img.max()\n    # %% [code]\n    def process_img(img,flip=[False]*10):\n        imgs=[]\n        for x in range(10):\n            if flip[x]:\n                imgs.append(get_birghtness(cv2.flip(img[:,x*240:(x+1)*240,:],1)))\n            else:\n                imgs.append(get_birghtness(img[:,x*240:(x+1)*240,:]))\n        return np.array(imgs)\n\n    sample_submission = pd.read_csv(\"../input/deepfake-detection-challenge/sample_submission.csv\")\n    test_files=glob.glob(video[xxxxx]+'*.jpg')\n    submission=pd.DataFrame()\n    submission['filename']=os.listdir(('../input/deepfake-detection-challenge/test_videos/'))\n    submission['label']=0.5\n    filenames=[]\n\n    batch=[]\n    batch1=[]\n    batch2=[]\n    batch3=[]\n\n    preds=[]\n\n    for x in test_files:\n        img=process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB))\n        if img is None:\n            continue\n        batch.append(img)\n        batch1.append(process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB),[True]*10))\n        batch2.append(process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB),[True,False]*5))\n        batch3.append(process_img(cv2.cvtColor(cv2.imread(x),cv2.COLOR_BGR2RGB),[False,True]*5))\n\n        filenames.append(x.replace(video[xxxxx],'').replace('.jpg','.mp4'))\n        if len(batch)==16:\n            preds+=(((0.25*model.predict(np.array(batch))))+((0.25*model.predict(np.array(batch1))))+((0.25*model.predict(np.array(batch2))))+((0.25*model.predict(np.array(batch3))))).tolist()\n            batch=[]\n            batch1=[]\n            batch2=[]\n            batch3=[]\n    if len(batch)!=0:\n        preds+=(((0.25*model.predict(np.array(batch))))+((0.25*model.predict(np.array(batch1))))+((0.25*model.predict(np.array(batch2))))+((0.25*model.predict(np.array(batch3))))).tolist()\n\n    print(time.time()-start)\n\n    new_preds=[]\n    for x in preds:\n        new_preds.append(x[0])\n    print(sum(new_preds)/len(new_preds))\n\n    for x,y in zip(new_preds,filenames):\n        submission.loc[submission['filename']==y,'label']=min([max([0.05,x]),0.95])\n\n    submission.to_csv(sub_file[xxxxx], index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r videos\n!rm -r videos_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = pd.read_csv('submission_1.csv').set_index('filename').transpose().to_dict()\ndf2 = pd.read_csv('submission_2.csv').set_index('filename').transpose().to_dict()\ndf3 = pd.read_csv('submission_3.csv').set_index('filename').transpose().to_dict()\ndf4 = pd.read_csv('submission_4.csv').set_index('filename').transpose().to_dict()\ndf5 = pd.read_csv('submission_5.csv').set_index('filename').transpose().to_dict()\ndf6 = pd.read_csv('submission_6.csv').set_index('filename').transpose().to_dict()\ndf7 = pd.read_csv('submission_7.csv').set_index('filename').transpose().to_dict()\ndf8 = pd.read_csv('submission_8.csv').set_index('filename').transpose().to_dict()\nfilename = []\nlabel = []\nfor i in df1.keys():\n    filename.append(i)\n    a = []\n    if df1[i]['label']!=0.5:\n        a.append(df1[i]['label'])\n    if df2[i]['label']!=0.5:\n        a.append(df2[i]['label'])\n    if df3[i]['label']!=0.5:\n        a.append(df3[i]['label'])\n    if df4[i]['label']!=0.5:\n        a.append(df4[i]['label'])\n    if df5[i]['label']!=0.5:\n        a.append(df5[i]['label'])\n    if df6[i]['label']!=0.5:\n        a.append(df6[i]['label'])\n    if df7[i]['label']!=0.5:\n        a.append(df7[i]['label'])\n    if df8[i]['label']!=0.5:\n        a.append(df8[i]['label'])\n    if len(a)==0:\n        label.append(0.5)\n    else:\n        label.append(min([max([0.05,sum(a)/len(a)]),0.95]))\ndf = pd.DataFrame()\ndf['filename'] = filename\ndf['label'] = label\nprint(np.array(df['label']).mean())\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm submission_1.csv\n!rm submission_2.csv\n!rm submission_3.csv\n!rm submission_4.csv\n!rm submission_5.csv\n!rm submission_6.csv\n!rm submission_7.csv\n!rm submission_8.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(df['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(time.time()-super_start)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":4}