{"cells":[{"metadata":{},"cell_type":"markdown","source":"Install missing packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install '/kaggle/input/dlibpkg/dlib-19.19.0'\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Import necessary Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport dlib\nimport time\nimport pandas as pd\nfrom keras.layers import *\nfrom keras.optimizers import *\nimport cv2\nfrom tqdm import tqdm\nfrom keras.applications.inception_v3 import InceptionV3, preprocess_input\nfrom keras.models import Model, Sequential\nfrom keras.layers import Input, Dense, Dropout\nfrom keras.models import model_from_json\nimport scipy.stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_face_detector(image, net):\n    # load the input image and construct an input blob for the image\n    # by resizing to a fixed 300x300 pixels and then normalizing it\n    (h, w) = image.shape[:2]\n    blob = cv2.dnn.blobFromImage(cv2.resize(image, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))\n    net.setInput(blob)\n    detections = net.forward()\n    detections = detections[detections[:, :, :, 2] > 0.4]\n    face_detection_coordinates = []\n\n    # loop over the detections\n    for i in range(0, len(detections)):\n        box = detections[i, 3:7] * np.array([w, h, w, h])\n        (startX, startY, endX, endY) = box.astype(\"int\")\n        face_w = endX - startX\n        face_h = endY - startY\n        if face_w != 299:\n            offset = int((299 - face_w) / 2)\n            if startX - offset < 0:\n                startX = 0\n                endX = 299\n            elif endX + offset > w:\n                startX = w - 299\n                endX = w\n            else:\n                startX = startX - offset\n                endX = endX + offset\n                if endX - startX != 299:\n                    endX += (299 - (endX - startX))\n        if face_h != 299:\n            offset = int((299 - face_h) / 2)\n            if startY - offset < 0:\n                startY = 0\n                endY = 299\n            elif endY + offset > h:\n                startY = h - 299\n                endY = h\n\n            else:\n                startY = startY - offset\n                endY = endY + offset\n\n                if endY - startY != 299:\n                    endY += (299 - (endY - startY))\n        face_detection_coordinates.append(((startX, startY), (endX, endY)))\n    return face_detection_coordinates\n\n\ndef detect_video_test_set(video_path, frames_to_capture, net_ogj):\n    # capture the video into frames\n    list_frames = []\n    j = 0\n    frame_counter = 0\n    # max time allowed for while loop to run\n    time_out = time.time() + 30\n    while frame_counter < 12:\n        # last measure to abort lengthy processes\n        # break loop if run time takes over 30 seconds\n        if time.time() > time_out:\n            print(\"time out\")\n            # fill in the rest with duplicates\n            list_frames.extend([list_frames[0] for i in range(12 - frame_counter)])\n            break\n\n        if frame_counter > 12:\n            break\n\n        count = j\n        vid = cv2.VideoCapture(video_path)\n        while True:\n\n            ret, cap = vid.read()  # Capture frame-by-frame\n            if cap is not None:\n                # number of faces detected in frame\n                cr = image_face_detector(cap, net_ogj)\n                for i in range(len(cr)):\n                    frame = cap[int(cr[i][0][1]): int(cr[i][1][1]), int(cr[i][0][0]):int(cr[i][1][0])]\n                    list_frames.append(frame)\n                    frame_counter += 1\n                count = count + frames_to_capture\n                vid.set(1, count)\n            else:\n                vid.release()\n                break\n        j = j + 1\n        # if j == 35:\n        #     break\n        # else:\n        #     j += 1\n\n    return list_frames\n\n\n\ndef video_to_frames_test_videos(frames_interval, test_videos):\n    test_vids = test_videos\n\n    predictor_path = \"/kaggle/input/face-detection-text-models/shape_predictor_68_face_landmarks.dat\"\n    detector = dlib.get_frontal_face_detector()\n    predictor = dlib.shape_predictor(predictor_path)\n    net = cv2.dnn.readNetFromCaffe('/kaggle/input/face-detection-text-models/deploy.prototxt.txt',\n                                   '/kaggle/input/face-detection-text-models/res10_300x300_ssd_iter_140000.caffemodel')\n\n    # Inception V3 model for feature extraction\n    input_tensor = Input((299, 299, 3))\n    print('creating model')\n    base_mdl = InceptionV3(input_tensor=input_tensor, weights=None, include_top=True)\n    #get imagenet weights\n    print('loading weights')\n    base_mdl.load_weights('/kaggle/input/imagenet/inception_v3_weights_tf_dim_ordering_tf_kernels.h5')\n    # only use model up to last avg_pool\n    mdl = Model(inputs=base_mdl.input, outputs=base_mdl.get_layer('avg_pool').output)  # output size (None, 2048)\n\n    list_vid_sequence_features = []\n    # for each video\n    for path in tqdm(test_vids):\n        vid_name = path.split('/')[5]\n        try:\n            sequence = []\n            # get 300/frame_interval frames from the video\n            frames = detect_video_test_set(video_path=path, frames_to_capture=frames_interval, net_ogj=net)\n            if len(frames) < 12:\n                continue\n            for img in frames:  # feed sequence of frames in inception_v3 for feature extraction\n                x = np.expand_dims(img, axis=0)\n                x = preprocess_input(x)\n                features = mdl.predict(x)\n                sequence.append(features[0])\n\n            if len(sequence) < 12:\n                continue\n            elif len(sequence) > 12:\n                list_vid_sequence_features.append((vid_name,sequence[0:12]))\n            else:\n                list_vid_sequence_features.append((vid_name,sequence))\n        except Exception as err:\n            print(err)\n\n    return list_vid_sequence_features\n\n\ndef larger_range(model_pred, time):\n    return (((model_pred - 0.5) * time) + 0.5)\n\n\ndef prediction_pipline(X, models, two_times=False):\n    preds = []\n    for model in tqdm(models):\n        pred = model.predict([X])\n        preds.append(pred)\n    preds = sum(preds) / len(preds)\n    if two_times:\n        return larger_range(preds, 2)\n    else:\n        return preds\n\n\ndef define_model_lstm():\n    learning_model = Sequential()\n    learning_model.add(\n        LSTM(2048, input_shape=(12, 2048), dropout=0.5))  # input_shape = sequence length, feature vector length\n    learning_model.add(Dense(512, activation='relu'))\n    learning_model.add(Dropout(0.5))\n    learning_model.add(Dense(2, activation='softmax'))\n    learning_model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-4))\n\n    return learning_model\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1) go through test_videos, get paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_vid_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nfilenames = os.listdir(test_vid_dir)\nprediction_filenames = filenames\ntest_video_files = [test_vid_dir + x for x in filenames]  # get test video files paths","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2) get feature vectors of videos"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X = video_to_frames_test_videos(25, test_video_files)  # get feature vectors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load pretrained models and weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"# list_models = []\n# models_json = os.listdir('./models/')\n# weights_h5 = os.listdir('./weights/')\n# for model, weights in tqdm(zip(models_json, weights_h5), total=len(models_json)):\n#\n#     with open(model, 'r') as json_file:\n#         loaded_model_json = json_file.read()\n#         json_file.close()\n#     temp_model = model_from_json(loaded_model_json)\n#     temp_model.load_weights(weights)\n#     list_models.append(temp_model)\n\n\nwith open(\"/kaggle/input/final-model-cnn-lstm/best_model.json\", 'r') as json_file:\n    loaded_best_model_json = json_file.read()\n    json_file.close()\n\nfinal_model = model_from_json(loaded_best_model_json)\nfinal_model.load_weights('/kaggle/input/final-model-cnn-lstm/best_model_weights.h5')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_vids = []\ntest_X_vid_names = []\ntest_X_features = []\nfor tup in test_X:\n    test_X_features.append(tup[1])\n    test_X_vid_names.append(tup[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/deepfake-detection-challenge/sample_submission.csv')\ndf_test['label'] = 0.5\ntwo_times = False\nprediction_filenames = df_test['filename'].to_numpy()\npreds = prediction_pipline(test_X_features, [final_model], two_times=two_times).clip(0.35, 0.65)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for pred, name in zip(preds, prediction_filenames):\n    name = name.replace(test_vid_dir, '')\n    df_test.iloc[list(df_test['filename']).index(name), 1] = pred[1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(preds.clip(0.35, 0.65).mean())\nprint(scipy.stats.median_absolute_deviation(preds.clip(0.35, 0.65))[0])\nprint(preds[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_test.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}