{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/facenet-torch-model/facenet_pytorch-2.2.7-py3-none-any.whl\n\nfrom facenet_pytorch.models.inception_resnet_v1 import get_torch_home\ntorch_home = get_torch_home()\n\n# Copy model checkpoints to torch cache so they are loaded automatically by the package\n!mkdir -p $torch_home/checkpoints/\n!cp /kaggle/input/facenet-torch-pretrained/vggface2_DG3kwML46X.pt $torch_home/checkpoints/vggface2_DG3kwML46X.pt\n!cp /kaggle/input/facenet-torch-pretrained/vggface2_G5aNV2VSMn.pt $torch_home/checkpoints/vggface2_G5aNV2VSMn.pt\n\nimport os\nimport glob\nimport time\nimport torch\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm\nimport gc\nimport math\n\n# See github.com/timesler/facenet-pytorch:\nfrom facenet_pytorch import MTCNN, InceptionResnetV1, extract_face\n\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nprint(f'Running on device: {device}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"filenames = glob.glob('/kaggle/input/deepfake-detection-challenge/test_videos/*.mp4')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/deepfake-detection-challenge/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtcnn = MTCNN(margin=14, keep_all=True, factor=0.5, device=device).eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def face_crop(filepath, batch_size, detector = mtcnn):\n    \n    v_cap = cv2.VideoCapture(filepath)\n    v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    \n    sample = np.arange(0, v_len, 2)\n    \n    result_dict = {'ptype1' : False,\n                  'ptype2': False, \n                  'boxes' : [],\n                   'probs' : []}\n    frames = []\n    video_frames = []\n    \n    for j in range(v_len):\n        success = v_cap.grab()\n        success, frame = v_cap.retrieve()\n        if not success or j not in sample:\n            continue\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n\n        frames.append(frame)\n        video_frames.append(frame)\n\n        # When batch is full, detect faces and reset frame list\n        \n    for i in np.arange(0, len(frames), batch_size):\n        boxes, probs = mtcnn.detect(frames[i:np.minimum(i+batch_size, len(frames))])\n        result_dict['boxes'].extend(boxes)\n        result_dict['probs'].extend(probs)\n    if not result_dict['ptype2'] and len([0 for a in result_dict['boxes'] if (a is None or  a == [])]) > 0:\n        result_dict['ptype2'] = True\n    if not result_dict['ptype1'] and len([0 for a in result_dict['boxes'] if (a is not None and  len(a) > 1)]) > 0:\n        result_dict['ptype1'] = True\n\n    v_cap.release()\n\n    return result_dict, video_frames","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_overlap (box1, box2, frac = 0.5):\n        b1x1, b1y1, b1x2, b1y2 = box1[:]\n        b2x1, b2y1, b2x2, b2y2 = box2[:]\n        w1 = abs(b1x1-b1x2)\n        h1 = abs(b1y1-b1y2)\n    #     w2 = abs(b2x1-b2x2)\n    #     h2 = abs(b2y1-b2y2)\n        range_b2x = (b1x1 - frac*w1, b1x1 + frac*w1, b1x2 - frac*w1, b1x2 + frac*w1)\n        range_b2y = (b1y1 - frac*h1, b1y1 + frac*h1, b1y2 - frac*h1, b1y2 + frac*h1)\n        if range_b2x[0] < b2x1 < range_b2x[1] and range_b2x[2] < b2x2 < range_b2x[3] and \\\n            range_b2y[0] < b2y1 < range_b2y[1] and range_b2y[2] < b2y2 < range_b2y[3]:\n            return True\n        else:\n            return False\n\ndef get_known_faces(boxes):\n    boxes_int = [ [list(map(int, box)) for box in fboxes] if fboxes is not None else None for fboxes in boxes ] \n    known_faces_dict = {}\n    for i in range(0,len(boxes_int)):\n        if boxes_int[i]:\n            # exclude if box with negative number\n            temp_dict = {'face'+str(j): {i: box} for j, box in enumerate(boxes_int[i]) if (np.array(box) > 0).all() }\n                    \n            nkeys = len(known_faces_dict.keys())          \n            for _, tempv in temp_dict.items():\n                exist = False\n                for k, v in known_faces_dict.items():  \n                    if check_overlap(list(v.values())[-1], tempv[i]):\n                        known_faces_dict[k].update(tempv)\n                        exist = True\n                if exist == False:\n                    known_faces_dict.update({'face'+str(1+nkeys):tempv})              \n    return known_faces_dict\n\n\ndef filter_known_faces(known_faces_dict, len_thresh=51):\n    filtered_known_faces_dict = {k:v for k,v in known_faces_dict.items() if len(v)>len_thresh}\n    return filtered_known_faces_dict\n\ndef filter_known_faces_by_size(known_faces_dict, dim_thresh=51):\n    filtered_known_faces_dict = {}\n    for k,v in known_faces_dict.items():\n        bx_arr = np.array(list(v.values()))\n        x1, y1, x2, y2 = np.mean(bx_arr, axis=0) \n        if np.abs(x1-x2) > dim_thresh and np.abs(y1-y2) > dim_thresh:\n            filtered_known_faces_dict[k] = v\n    return filtered_known_faces_dict\n\n\ndef get_boxes_filter_ptype1(filtered_known_faces_dict, len_boxes):\n    box_filtered = [[] for _ in range(len_boxes)]\n    for key in filtered_known_faces_dict.keys():\n        for key2 in filtered_known_faces_dict[key].keys():\n            box_filtered[key2].append(filtered_known_faces_dict[key][key2]) \n    return box_filtered","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fill_missing_box(boxes_ori):\n    boxes_filled = boxes_ori\n    missing_idx = [i for i, box in enumerate(boxes_ori) if box == []]\n    processed = False\n    if len(missing_idx) == 1:\n        if missing_idx[0] == 0:\n            boxes_filled[0] = boxes_filled[1]\n        elif missing_idx[0] == len(boxes_ori)-1 :\n            boxes_filled[-1] = boxes_filled[-2]\n        else:\n            boxes_filled[missing_idx[0]] = (np.array(boxes_filled[missing_idx[0]-1]) + np.array(boxes_filled[missing_idx[0]+1])) // 2\n        processed = True\n    else:\n        print(\"Too many missing boxes, not processed\")\n    return boxes_filled, processed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Combine(torch.nn.Module):\n    def __init__(self, CNN_model):\n        super(Combine, self).__init__()\n        self.cnn = CNN_model\n        self.rnn = torch.nn.LSTM(512, 64, 1, batch_first=True)\n        self.fc = torch.nn.Linear(64,1)\n\n    def forward(self, x):\n        b, t, c, w, h = x.size()\n        c_in = x.view(b*t, c, w, h)\n        c_out = self.cnn(c_in)\n        r_in = c_out.view(c_out.shape[0]//50, 50, -1)\n        self.rnn.flatten_parameters()\n        r_out, _ = self.rnn(r_in)\n        fc_in = r_out[:,-1,:]\n        fc_out = self.fc(fc_in)\n        return torch.sigmoid(fc_out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet = InceptionResnetV1(pretrained=None, num_classes=8631, device=device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Combine(resnet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\nmodel = torch.nn.DataParallel(model)\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('/kaggle/input/cnn-rnn-trained/CNN_RNN_model_to_submit.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(checkpoint)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_sample_for_estimation(video_frames, boxes):\n    v_len = len(video_frames)\n    sample = np.arange(0, np.minimum(v_len-1,299), 6)\n    sample = sample + 3 #MAGIC numbers\n    faces = []\n    while sample.shape[0] < 50:\n        sample = np.hstack([sample, [v_len]])    \n    for j in range(v_len):\n        if j not in sample:\n            continue\n        frame = video_frames[j]\n        cx1, cy1, cx2, cy2 = list(map(int, boxes[j][0]))\n        marginx = (160 - (cx2 - cx1))//2\n        cx1 = cx1 - marginx - (160 - (cx2 - cx1))%2\n        cx2 = cx2 + marginx\n        marginy = (160 - (cy2 - cy1))//2\n        cy1 = cy1 - marginy - (160 - (cy2 - cy1))%2\n        cy2 = cy2 + marginy\n        cy1, pad_y1 = np.maximum(cy1, 0), np.maximum(0-cy1, 0)\n        cy2, pad_y2 = np.minimum(cy2, frame.shape[0]), np.maximum(cy2-frame.shape[0], 0)\n        cx1, pad_x1 = np.maximum(cx1, 0), np.maximum(0-cx1, 0)\n        cx2, pad_x2 = np.minimum(cx2, frame.shape[1]), np.maximum(cx2-frame.shape[1], 0)\n\n        padded_frame = np.pad(frame[cy1:cy2, cx1:cx2, :], ((pad_y1,pad_y2),(pad_x1,pad_x2), (0,0)), mode='constant' )\n        assert padded_frame.shape == (160, 160, 3), \\\n        'the coords are {}, {}, {},{}, frame shape {}'.format(cx1, cx2, cy1, cy2, frame[cy1:cy2, cx1:cx2, :].shape)\n\n        faces.append(padded_frame)\n\n    while len(faces) < 50:\n        faces.append(faces[-1])\n    assert len(faces) == 50, 'Not enough faces, len: {}, sameple: {}'.format(v_len, sample)\n    faces = np.moveaxis(np.array(faces), 3,1)   #.permute(0,3,1,2)\n    faces = (faces - 127.5) * 0.0078125\n\n    return faces.astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = {}\nfalse_counter = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for filename in tqdm(filenames):\n    with torch.no_grad():\n        result_dict, vid_frames = face_crop(filename, 64)\n    boxes_ori = result_dict['boxes']\n    known_faces_dict = get_known_faces(boxes_ori)\n    filtered_faces_1 = filter_known_faces(known_faces_dict)\n    filtered_faces_2 = filter_known_faces_by_size(filtered_faces_1)\n    boxes_filtered = get_boxes_filter_ptype1(filtered_faces_2, len(boxes_ori))\n    result_dict['boxes'] = boxes_filtered\n    if len([*filtered_faces_2]) == 1:\n        result_dict['ptype1'] = False\n    else:\n        result_dict['ptype1'] = True\n\n    if len([0 for a in result_dict['boxes'] if (a is None or  a == [])]) > 0:\n        result_dict['ptype2'] = True\n    else:\n        result_dict['ptype2'] = False \n    if result_dict['ptype2']==True and result_dict['ptype1']==False:               \n        boxes_ori = result_dict['boxes']\n        boxes_filled, processed = fill_missing_box(boxes_ori)\n        if processed:\n            result_dict['boxes'] = boxes_filled\n            result_dict['ptype2'] = False\n    if result_dict['ptype2']==True and result_dict['ptype1']==False:               \n        boxes_ori = result_dict['boxes']\n        boxes_filled, processed = fill_missing_box(boxes_ori)\n        if processed:\n            result_dict['boxes'] = boxes_filled\n            result_dict['ptype2'] = False\n    if result_dict['ptype2']==False and result_dict['ptype1']==False:\n        print('Estimating by model')\n        sample_to_train = build_sample_for_estimation(vid_frames, result_dict['boxes'])\n        with torch.no_grad():\n            prob = model(torch.from_numpy(sample_to_train.reshape(1,50,3,160,160)).type(torch.FloatTensor).to(device))\n            if torch.isnan(prob).item():\n                submission.append([os.path.basename(filename), 0.5])\n            else:\n                prob_val = prob.item()\n                if prob_val >= 0 and prob_val <= 1:\n                    submission[os.path.basename(filename)] =  prob_val\n                else:\n                    submission[os.path.basename(filename)] =  0.5\n    else:\n        false_counter += 1\n        submission[os.path.basename(filename)] =  0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fill_df(x):\n    if x not in submission.keys():\n        return 0.5\n    else:\n        val = submission[x]\n        if math.isnan(val):\n            return 0.5\n        else:\n            try:\n                val = float(val)\n                if val >0 and val<1:\n                    return val\n            except:\n                return 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['label'] = df['filename'].apply(lambda x: fill_df(x))\ndf['label'] = 1. - df['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(df.label, 20)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}