{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport sys\nsys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\nsys.path.insert(0, \"/kaggle/input/deepfakes-inference-demo\")\nsys.path.insert(0, \"/kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4\")\nsys.path.insert(0, \"/kaggle/input/facenetpytorch/facenet-pytorch-1.0.1/\")\n\nfrom models.mtcnn import MTCNN#, InceptionResnetV1, extract_face\nimport os\nfrom pretrainedmodels import se_resnext50_32x4d\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch import nn, optim\nimport torch.nn.functional as F\nimport os\nimport glob\nimport time\nimport torch\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom torchvision import transforms\n\nimport datetime\n\n# See github.com/timesler/facenet-pytorch:\nfrom concurrent.futures import ThreadPoolExecutor\nfrom albumentations.augmentations.transforms import ShiftScaleRotate, VerticalFlip, RandomBrightnessContrast, \\\n    GaussianBlur, CoarseDropout, Normalize\nfrom albumentations.pytorch import ToTensor\nfrom albumentations import Compose\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from torchvision.transforms import Normalize\nprint(\"PyTorch version:\", torch.__version__)\nprint(\"CUDA version:\", torch.version.cuda)\nprint(\"cuDNN version:\", torch.backends.cudnn.version())\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nnormalize_transform = Normalize(mean, std)\ngpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ngpu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_size = 224\nbatch_video = 2\nnum_frame = 3\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\ntrain_transform = Compose([  \n    #RandomBrightnessContrast(p=0.5, brightness_limit=0.5, contrast_limit=0.5),\n    Normalize(mean=mean, std=std, p=1),\n    ToTensor()\n])\ndevice = 'cuda:2' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NextVLAD"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install pytorchcv --quiet\ndevice = gpu\n#checkpoint = torch.load('se_resnext50.pth',map_location=device)\nnet = se_resnext50_32x4d(num_classes=1, pretrained=None).to(device)\nmodel = net.to(device)\n\nmodel = nn.Sequential(*list(model.children())[:-1])\nmodel[-1] = nn.Sequential(nn.AdaptiveAvgPool2d(1)).to(device)\nbase_model = model\n\n\ndef freeze_until(net, param_name):\n    found_name = False\n    for name, params in net.named_parameters():\n        if name == param_name:\n            found_name = True\n        params.requires_grad = found_name\n#freeze_until(base_model, \"#4.2.conv1.weight\") #freezen all layer\n\n\nclass SE_ContextGating(nn.Module):\n    def __init__(self, vlad_dim, hidden_size, drop_rate=0.5,gating_reduction=8):\n\n        super(SE_ContextGating, self).__init__()\n        \n        self.fc1 = nn.Linear(vlad_dim,hidden_size)\n        self.dropout = nn.Dropout(drop_rate)\n        self.bn1 = nn.BatchNorm1d(hidden_size)      \n        self.gate = torch.nn.Sequential(    \n            nn.Linear(hidden_size,hidden_size//gating_reduction),\n            nn.BatchNorm1d(hidden_size//gating_reduction),\n            nn.ReLU(),\n            \n            nn.Linear(hidden_size//gating_reduction,hidden_size),\n            nn.Sigmoid()\n        )\n               \n    def forward(self,x):\n        x = self.bn1(self.dropout(self.fc1(x)))\n        gate = self.gate(x)\n        activation = x * gate\n        return activation\n\nclass NextVLAD(nn.Module):\n    \"\"\"NetVLAD layer implementation\"\"\"\n\n    def __init__(self, num_clusters=64,expansion=2,group=8, dim=2048,num_class=1):\n\n        super(NextVLAD, self).__init__()\n        \n        self.num_clusters = num_clusters\n        self.expansion = expansion\n        self.group = group\n        self.dim = dim\n        self.bn1 = nn.BatchNorm1d(group*num_clusters)\n        self.bn2 = nn.BatchNorm1d(num_clusters*expansion*dim//group)\n        self.centroids1 = nn.Parameter(torch.rand(expansion*dim, group*num_clusters))\n        self.centroids2 = nn.Parameter(torch.rand(1,expansion*dim//group,num_clusters ))\n        self.fc1 = nn.Linear(dim,expansion*dim) \n        self.fc2 = nn.Linear(dim*expansion,group)\n        \n        self.cg = SE_ContextGating(num_clusters*expansion*dim//group,dim)\n        self.fc3 = nn.Linear(dim,num_class)\n\n    def forward(self, x): #2,4,2048,1,1\n        \n        max_frames = x.size(1)\n        x = x.view(x.size()[:3]) #2,4,2048\n        \n        x_3d = F.normalize(x, p=2, dim=2)  # across descriptor dim, torch.Size([2,4, 2048, 1, 1])\n        \n        vlads = []\n        for t in range(x_3d.size(0)):\n            x = x_3d[t, :, :] #4,2048\n            x = self.fc1(x) #expand, 4,2*2048\n            \n            #attention\n            attention = torch.sigmoid(self.fc2(x)) # 4,8\n            attention = attention.view(-1, max_frames*self.group, 1) \n            \n            feature_size = self.expansion * self.dim // self.group\n            #reshaped_input = tf.reshape(input, [-1, self.expansion * self.feature_size])\n            reshaped_input = x.view(-1,self.expansion *self.dim) # 4,2*2048\n            #activation = tf.matmul(reshaped_input, cluster_weights)\n            activation = torch.mm(reshaped_input, self.centroids1) # 4,8*32\n            activation = self.bn1(activation)\n            #activation = tf.reshape(activation, [-1, self.max_frames * self.groups, self.cluster_size])\n            activation = activation.view(-1,max_frames*self.group,self.num_clusters) # 1,32,32\n            #activation = tf.nn.softmax(activation, axis=-1)\n            activation = F.softmax(activation, dim=-1)  # 1,32,32\n            #activation = tf.multiply(activation, attention)\n            activation = activation * attention # 1,32,32\n            #a_sum = tf.sum(activation, -2, keep_dims=True)\n            a_sum = activation.sum(dim=-2, keepdim=True) #1,32,1\n            \n            #a = tf.multiply(a_sum, cluster_weights2)\n            a = a_sum * self.centroids2 # 1,512,32 (512=dim*expansion//group,32=clusters)\n            #activation = tf.transpose(activation, perm=[0, 2, 1])\n            activation = activation.permute(0, 2, 1) #1,32,1\n            #reshaped_input = tf.reshape(input, [-1, self.max_frames * self.groups, feature_size])\n            reshaped_input = x.view(-1,max_frames*self.group,feature_size) # 1,32,512\n            vlad = torch.bmm(activation, reshaped_input) # 1,32,512\n            #vlad = tf.transpose(vlad, perm=[0, 2, 1])\n            vlad = vlad.permute(0,2,1)\n            #vlad = tf.subtract(vlad, a)\n            vlad = vlad - a # 1,512,32\n            #vlad = tf.nn.l2_normalize(vlad, 1)\n            vlad = F.normalize(vlad,p=2,dim=1)\n            #vlad = tf.reshape(vlad, [-1, self.cluster_size * feature_size])\n            vlad = vlad.view(self.num_clusters*feature_size) #[1, 16384]\n            vlads.append(vlad)\n        vlads = torch.stack(vlads, dim=0)\n        vlads = self.bn2(vlads) #[2, 16384]\n        \n        x = self.cg(vlads) #SE Context Gating\n        x = self.fc3(x) \n\n        return x\n        \n    \n            \n    \nclass EmbedNet(nn.Module):\n    def __init__(self, base_model, net_vlad):\n        super(EmbedNet, self).__init__()\n        self.base_model = base_model\n        self.net_vlad = net_vlad\n\n    def forward(self, x):\n        frame_feature = []\n        for t in range(x.size(1)):\n            feature = self.base_model(x[:, t, :, :, :])\n            frame_feature.append(feature)\n        frame_feature = torch.stack(frame_feature, dim=0).transpose_(0, 1)\n        embedded_x = self.net_vlad(frame_feature)\n        \n        return embedded_x\ntorch.backends.cudnn.benchmark = True\ntorch.backends.cudnn.deterministic = True\n\n\nnext_vlad = NextVLAD(num_clusters=32,expansion=2,group=8, dim=2048)\nmodel = EmbedNet(model, next_vlad).to(device)\ncheckpoint = torch.load('/kaggle/input/se50nextv1/3.24_0.pth',map_location=device)\nmodel.load_state_dict(checkpoint)\n_ = model.eval()\ndel checkpoint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MTCNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Detector:\n    def __init__(self, net, image_size = 224):\n        self.net = net\n        self.image_size = image_size\n    def __call__(self, frames):\n        return self.net.detect(frames)\n    \n    def face_resize(self, face):\n        h, w = face.shape[:2]\n        if w != h:\n            size = max(w, h)\n            offset_w, offset_h = size - w, size - h\n            face = cv2.copyMakeBorder(face, offset_h//2, offset_h - offset_h//2, offset_w//2, offset_w - offset_w//2, cv2.BORDER_CONSTANT)\n        return cv2.resize(face, (self.image_size, self.image_size), cv2.INTER_LINEAR)\n        \n    def recover_in_original_frame(self, frames, boxes, resize):\n        boxes =  [[(box/resize).astype(int) for box in i] if i is not None else i for i in boxes]\n        return [[self.face_resize(frame[box[1]:box[3], box[0]:box[2]]) for box in boxes_perframe] if boxes_perframe is not None else None for frame, boxes_perframe in zip(frames, boxes)]\n    \n    def make_square(self, boxes, frame_size):\n        for i in range(len(boxes)):\n            boxes_perframe = []\n            if boxes[i] is not None:\n                for j in range(len(boxes[i])):\n                    box = boxes[i][j]\n                    w, h = (box[2] - box[0], box[3] - box[1])\n                    size = max(w, h)\n                    centre = int(box[0] + w/2), int(box[1] + h/2)\n\n                    xmin = np.clip(centre[0] - size//2, 0, frame_size[1])\n                    xmax = np.clip(centre[0] + size//2, 0, frame_size[1])\n                    ymin = np.clip(centre[1] - size//2, 0, frame_size[0])\n                    ymax = np.clip(centre[1] + size//2, 0, frame_size[0])\n\n                    boxes_perframe.append([xmin, ymin, xmax, ymax])\n                boxes_perframe = np.array(boxes_perframe)\n\n            else:\n                boxes_perframe = None\n                \n            boxes[i] = boxes_perframe\n        return boxes\n    \n    \n    \n    def add_margin(self, boxes, frame_size, margin = 0.2):\n        '''\n        margin is a number bewteen [0. 1] or a numpy array which contains 4 elements\n        [xmin, ymin, xmax, ymax]\n        '''\n        if isinstance(margin, float):\n            margin = np.array([margin] *4)\n        for i in range(len(boxes)):\n            boxes_perframe = []\n            if boxes[i] is not None:\n                for j in range(len(boxes[i])):\n                    box = boxes[i][j]\n                    w, h = (box[2] - box[0], box[3] - box[1])\n\n                    xmin = np.clip(box[0] - w*margin[0], 0, frame_size[1])\n                    xmax = np.clip(box[2] + w*margin[2], 0, frame_size[1])\n                    ymin = np.clip(box[1] - h*margin[1], 0, frame_size[0])\n                    ymax = np.clip(box[3] + h*margin[3], 0, frame_size[0])\n\n                    boxes_perframe.append([xmin, ymin, xmax, ymax])\n                boxes_perframe = np.array(boxes_perframe)\n            else:\n                boxes_perframe = None\n            boxes[i] = boxes_perframe\n        return boxes\n\n\n\nclass PathPipeline:\n    def __init__(self, root = './'):\n        self.root = root\n    def __call__(self, filename):\n        return f'{self.root}/{filename}'\n\nclass DetectionPipeline:\n    \"\"\"Pipeline class for detecting faces in the frames of a video file.\"\"\"\n    \n    def __init__(self, detector, n_frames=None, batch_size=60, resize=None, num = 10):\n        \"\"\"Constructor for DetectionPipeline class.\n        \n        Keyword Arguments:\n            n_frames {int} -- Total number of frames to load. These will be evenly spaced\n                throughout the video. If not specified (i.e., None), all frames will be loaded.\n                (default: {None})\n            batch_size {int} -- Batch size to use with MTCNN face detector. (default: {32})\n            resize {float} -- Fraction by which to resize frames from original prior to face\n                detection. A value less than 1 results in downsampling and a value greater than\n                1 result in upsampling. (default: {None})\n            num: num is the number of faces returned\n        \"\"\"\n        self.detector = detector\n        self.n_frames = n_frames\n        self.batch_size = batch_size\n        self.resize = resize\n        self.num = num\n    \n    def __call__(self, filename):\n        \"\"\"Load frames from an MP4 video and detect faces.\n\n        Arguments:\n            filename {str} -- Path to video.\n        \"\"\"\n        # Create video reader and find length\n        v_cap = cv2.VideoCapture(filename)\n        v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))\n\n        # Pick 'n_frames' evenly spaced frames to sample\n        \n        \n        \n        \n        if self.n_frames is None:\n            sample = np.arange(0, v_len)\n        else:\n            sample = np.arange(0, v_len - 1, self.n_frames).astype(int)\n\n        # Loop through frames\n        faces = []\n        probs = []\n        frames = []\n        frames_small = []\n        for j in range(v_len):\n            success = v_cap.grab()\n            if j in sample:\n                # Load frame\n                success, frame = v_cap.retrieve()\n                if not success:\n                    continue\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                frame_small = Image.fromarray(frame)\n                \n                # Resize frame to desired size\n                if self.resize is not None:\n                    frame_small = frame_small.resize([int(d * self.resize) for d in frame_small.size])\n                frames_small.append(frame_small)\n                \n                frames.append(frame)\n\n                # When batch is full, detect faces and reset frame list\n                if len(frames_small) % self.batch_size == 0 or j == sample[-1]:\n                    \n                    result = [list(zip(*sorted(zip(*i), key = lambda x:x[1], reverse = True)) if i[0] is not None else i)  for i in zip(*self.detector(frames_small))]\n                    boxes = [i[0] for i in result]\n                    W, H = frames_small[0].size\n                    boxes = self.detector.add_margin(boxes, (H, W), margin = [0.2, 0.4, 0.2, 0.2])\n                    boxes = self.detector.make_square(boxes, (H, W))\n                    \n                    faces = self.detector.recover_in_original_frame(frames, boxes, self.resize)\n                    faces = list(filter(None,faces))\n                    faces = [train_transform(image=np.array(face[0]))['image'] for face in faces]\n\n                    frames_small = []\n                    frames = []\n\n        v_cap.release()\n        return faces","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtcnn = MTCNN(image_size = 224, margin=40, post_process=False, keep_all=False, factor=0.5, device=device).eval()\ndetector = DetectionPipeline(detector=Detector(mtcnn), n_frames=6, batch_size=54, resize=0.25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" ## Prediction loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def FaceExtractor(video_path,detector):\n    faces = detector(os.path.join(test_dir,video_path))\n    faces = faces[:((len(faces)//num_frame)//batch_video)*num_frame*batch_video]\n    faces = torch.stack(faces,dim=0).view(-1,num_frame,3,224,224)\n    return faces\ndef predict_on_video(video_path):\n    try:\n        predicts = []\n        faces = FaceExtractor(video_path,detector) #torch.Size([16, 3, 3, 224, 224])\n        if len(faces)>0:\n            for i in range(faces.size(0)//batch_video):         \n                batch = faces[batch_video*i:batch_video*i+batch_video].to(device)\n                predict = torch.sigmoid(model(batch)).cpu().detach().numpy()\n                if (predict>0.5).sum() == batch_video:\n                    return predict.mean()\n                predicts.extend(predict)\n            return np.mean(predicts)\n    except Exception as e:\n        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n\n    return 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\n\ndef predict_on_video_set(videos, num_workers):\n    def process_file(i):\n        filename = videos[i]\n        y_pred = predict_on_video(os.path.join(test_dir, filename))\n        return y_pred\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        predictions = ex.map(process_file, range(len(videos)))\n\n    return list(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Speed test\n\nThe leaderboard submission must finish within 9 hours. With 4000 test videos, that is `9*60*60/4000 = 8.1` seconds per video. So if the average time per video is greater than ~8 seconds, the kernel will be too slow!"},{"metadata":{"trusted":true},"cell_type":"code","source":"speed_test = True  # you have to enable this manually\n\nif speed_test:\n    start_time = time.time()\n    speedtest_videos = test_videos[:1]\n    predictions = predict_on_video_set(speedtest_videos, num_workers=4)\n    elapsed = time.time() - start_time\n    print(\"Elapsed %f sec. Average per video: %f sec.\" % (elapsed, elapsed / len(speedtest_videos)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make the submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = predict_on_video_set(test_videos, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\n#submission_df['label'] = submission_df['label'].clip(0.9,0.1)\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}