{"cells":[{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"import os\n# Custom packages\nimport sys\n# Standalone packages\nfrom concurrent.futures import ThreadPoolExecutor\nfrom pathlib import Path\nfrom time import time\n\nimport scipy\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nfrom scipy.special import expit\n\nsys.path.append('/kaggle/input/faceutils/')\nsys.path.append('/kaggle/input/efficientnet-pytorch-master/EfficientNet-PyTorch-master/')\n\nfrom blazeface import BlazeFace\nfrom blazeface import FaceExtractor\nfrom blazeface import VideoReader\nfrom efficientnet_pytorch import EfficientNet\nfrom isplutils.utils import extract_bb\n\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Network definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\n\nclass FeatureExtractor(nn.Module):\n    \"\"\"\n    Abstract class to be extended when supporting features extraction.\n    It also provides standard normalized and parameters\n    \"\"\"\n\n    def features(self, x: torch.Tensor) -> torch.Tensor:\n        raise NotImplementedError\n\n    def get_trainable_parameters(self):\n        return self.parameters()\n\n    @staticmethod\n    def get_normalizer():\n        return transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    \nclass EfficientNetAutoAtt(EfficientNet):\n    def init_att(self, model: str):\n        \"\"\"\n        Initialize attention\n        :param model: efficientnet-bx, x \\in {0,..,7}\n        :return:\n        \"\"\"\n        if model == 'efficientnet-b0':\n            self.att_block_idx = 4\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=40, out_channels=1)\n        elif model == 'efficientnet-b1':\n            self.att_block_idx = 6\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=40, out_channels=1)\n        elif model == 'efficientnet-b2':\n            self.att_block_idx = 7\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=48, out_channels=1)\n        elif model == 'efficientnet-b3':\n            self.att_block_idx = 7\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=48, out_channels=1)\n        elif model == 'efficientnet-b4':\n            self.att_block_idx = 9\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=56, out_channels=1)\n        elif model == 'efficientnet-b5':\n            self.att_block_idx = 12\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=64, out_channels=1)\n        elif model == 'efficientnet-b6':\n            self.att_block_idx = 14\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=72, out_channels=1)\n        elif model == 'efficientnet-b7':\n            self.att_block_idx = 17\n            self.attconv = nn.Conv2d(kernel_size=1, in_channels=80, out_channels=1)\n        else:\n            raise ValueError('Model not valid: {}'.format(model))\n\n    def get_attention(self, x: torch.Tensor) -> torch.Tensor:\n\n        # Placeholder\n        att = None\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(x)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n            if idx == self.att_block_idx:\n                att = torch.sigmoid(self.attconv(x))\n                break\n\n        return att\n\n    def extract_features(self, x: torch.Tensor) -> torch.Tensor:\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(x)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n            if idx == self.att_block_idx:\n                att = torch.sigmoid(self.attconv(x))\n                x = x * att\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\nclass EfficientNetGenAutoAtt(FeatureExtractor):\n    def __init__(self, model: str):\n        super(EfficientNetGenAutoAtt, self).__init__()\n\n        self.efficientnet = EfficientNetAutoAtt.from_name(model)\n        self.efficientnet.init_att(model)\n        self.classifier = nn.Linear(self.efficientnet._conv_head.out_channels, 1)\n        del self.efficientnet._fc\n\n    def features(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.efficientnet.extract_features(x)\n        x = self.efficientnet._avg_pooling(x)\n        x = x.flatten(start_dim=1)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.efficientnet._dropout(x)\n        x = self.classifier(x)\n        return x\n\n    def get_attention(self, x: torch.Tensor) -> torch.Tensor:\n        return self.efficientnet.get_attention(x)\n        \nclass EfficientNetAutoAttB4(EfficientNetGenAutoAtt):\n    def __init__(self):\n        super(EfficientNetAutoAttB4, self).__init__(model='efficientnet-b4')\n\nclass SiameseTuning(FeatureExtractor):\n    def __init__(self, feat_ext: FeatureExtractor, num_feat: int):\n        super(SiameseTuning, self).__init__()\n        self.feat_ext = feat_ext()\n        if not hasattr(self.feat_ext, 'features'):\n            raise NotImplementedError('The provided feature extractor needs to provide a features() method')\n\n        self.classifier = nn.Sequential(\n            nn.BatchNorm1d(num_features=num_feat),\n            nn.Linear(in_features=num_feat, out_features=1),\n        )\n\n    def features(self, x):\n        x = self.feat_ext.features(x)\n        return x\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        with torch.no_grad():\n            x = self.features(x)\n        x = self.classifier(x)\n        return x\n\n    def get_trainable_parameters(self):\n        return self.classifier.parameters()\n        \nclass EfficientNetAutoAttB4ST(SiameseTuning):\n    def __init__(self):\n        super(EfficientNetAutoAttB4ST, self).__init__(feat_ext=EfficientNetAutoAttB4, num_feat=1792)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initializations"},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"# Parameters\n\n\"\"\"\nModel\nSee dataset description\n\"\"\"\n\n# Kaggle\ninput_folder = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nmodel_paths = [\n    '/kaggle/input/efficientnetautoattb4st-folds-weights/net-EfficientNetAutoAttB4ST_size-224_face-scale_split-fourtyfolders-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_foldslow/it000350.pth',\n    '/kaggle/input/efficientnetautoattb4st-folds-weights/net-EfficientNetAutoAttB4ST_size-224_face-scale_split-fourtyfolders1-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_foldslow/it000300.pth',\n    '/kaggle/input/efficientnetautoattb4st-folds-weights/net-EfficientNetAutoAttB4ST_size-224_face-scale_split-fourtyfolders2-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_foldslow/it000300.pth',\n    '/kaggle/input/efficientnetautoattb4st-folds-weights/net-EfficientNetAutoAttB4ST_size-224_face-scale_split-fourtyfolders3-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_foldslow/it000400.pth',\n    '/kaggle/input/efficientnetautoattb4st-folds-weights/net-EfficientNetAutoAttB4ST_size-224_face-scale_split-fourtyfolders4-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_foldslow/it000300.pth',\n    '/kaggle/input/efficientnetautoattb4-folds-weights/net-EfficientNetAutoAttB4_size-224_face-scale_split-fourtyfolders-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_ffpp-True_fold/it003000.pth',\n    '/kaggle/input/efficientnetautoattb4-folds-weights/net-EfficientNetAutoAttB4_size-224_face-scale_split-fourtyfolders1-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_ffpp-True_fold/it026000.pth',\n    '/kaggle/input/efficientnetautoattb4-folds-weights/net-EfficientNetAutoAttB4_size-224_face-scale_split-fourtyfolders2-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_ffpp-True_fold/it017500.pth',\n    '/kaggle/input/efficientnetautoattb4-folds-weights/net-EfficientNetAutoAttB4_size-224_face-scale_split-fourtyfolders3-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_ffpp-True_fold/it009000.pth',\n    '/kaggle/input/efficientnetautoattb4-folds-weights/net-EfficientNetAutoAttB4_size-224_face-scale_split-fourtyfolders4-oneface_subset-original_aug-1.00_trainloss-bce_optim-adam_ffpp-True_fold/it009000.pth',\n]\nblazeface_anchors_path = \"/kaggle/input/faceutils/blazeface/anchors.npy\"\nblazeface_weights_path = \"/kaggle/input/faceutils/blazeface/blazeface.pth\"\n\nframes_per_video = 72\nnum_workers = 8\n\nsubmission_path = 'submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"# Instantiate nets\npatch_size = 224\n\nnets = []\nfor model_path in model_paths:\n    if 'net-EfficientNetAutoAttB4ST_' in model_path:\n        net = EfficientNetAutoAttB4ST()\n    elif 'net-EfficientNetAutoAttB4_' in model_path:\n        net = EfficientNetAutoAttB4()\n    else:\n        raise ValueError('Unknown model for: {}'.format(model_path))\n    net = net.eval().to(device)\n    net.load_state_dict(torch.load(model_path, map_location='cpu')['net'])\n    nets.append(net)\ndel net #Avoid human errors in using net, if possible\n\nfacedet = BlazeFace().eval().to(device)\nfacedet.load_weights(blazeface_weights_path)\nfacedet.load_anchors(blazeface_anchors_path)\n\nvideo_reader = VideoReader()\nvideo_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)\nface_extractor = FaceExtractor(video_read_fn, facedet)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"# Transformers\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\npatch_size_load = patch_size\nloading_transformations = [\n    A.LongestMaxSize(max_size=patch_size_load, always_apply=True),\n    A.PadIfNeeded(min_height=patch_size_load, min_width=patch_size_load,\n                  border_mode=cv2.BORDER_CONSTANT, value=0),\n]\nfinal_transformations = [\n    A.Normalize(mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                ),\n    ToTensorV2(),\n]\n\nval_transformer = A.Compose(loading_transformations +  final_transformations)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"# Index files\nvideo_paths = sorted(Path(input_folder).glob('*.mp4'))\ndf = pd.DataFrame({'path': map(str, video_paths)}, index=[p.name for p in video_paths])\nprint('Found {} video files'.format(len(df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from threading import Semaphore\nnet_lock = Semaphore(1)\n\ndef process_video(name,debug:bool=False):\n    path = os.path.join(input_folder, name)\n    locked_by_me = False\n    try:\n        if debug:\n            t0 = time()\n            \n        # Find the faces for N frames in the video.\n        frames = face_extractor.process_video(path)\n        \n        # Only look at one face per frame.\n#         face_extractor.keep_only_best_face(frames)\n\n        # save blazeface scores for each frame\n        blzf_score_list = list()\n        faces_list = list()\n        for i in range(len(frames)):\n\n            frame_im = Image.fromarray(frames[i]['frame'])\n            scores = frames[i]['scores']\n            if len(scores) == 0:\n#                 print(‘Found empy score vector’)\n                continue\n    \n            if not np.any(np.array(scores) > 0.9):\n                \n                idx_max = np.argmax(scores)               \n                detection = frames[i]['detections'][idx_max]\n                face_im = extract_bb(frame=frame_im,\n                                        bb=(detection[1],detection[0],detection[3],detection[2]),\n                                        scale='scale',\n                                        size=patch_size_load)                    \n                faces_list.append(np.asarray(face_im))            \n                blzf_score_list.append([scores[idx_max]])\n                \n            else:\n                                \n                idx_max = np.where(np.array(scores) > 0.9)\n                if debug:\n                    print('idx_max {}'.format(idx_max))\n                \n                for detection in frames[i]['detections'][idx_max]:\n                    face_im = extract_bb(frame=frame_im,\n                                            bb=(detection[1],detection[0],detection[3],detection[2]),\n                                            scale='scale',\n                                            size=patch_size_load)                    \n                    faces_list.append(np.asarray(face_im))            \n                \n                blzf_score_list.append([scores[j] for j in idx_max[0]])\n       \n        if debug:\n            t1 = time()\n            print('Face detection: {:.3f}s'.format(t1-t0))\n\n        if debug:\n            t2 = time()\n            print('Face extraction: {:.3f}s'.format(t2-t1))\n        \n        faces_tensor = torch.stack([val_transformer(image=face)['image'] for face in faces_list])\n        faces_tensor = faces_tensor.to(device)\n        \n        if debug:\n            t3 = time()\n            print('Face transformation: {:.3f}s'.format(t3-t2))\n    \n        \n        net_lock.acquire()\n        locked_by_me = True\n        with torch.no_grad():\n            S = torch.zeros((len(nets), len(faces_list)))\n            for idx,net in enumerate(nets):\n                net_y_pred = net(faces_tensor)\n                S[idx, :] = net_y_pred.squeeze()\n        net_lock.release()\n        locked_by_me = False\n        \n        if debug:\n            t4 = time()\n            print('Prediction: {:.3f}s'.format(t4-t3))\n        \n        \n        # aggregation of the face scores\n        S = S.numpy()\n        cnt = 0\n        score_video = []\n        print(len(faces_list))\n        for i in range(len(blzf_score_list)):\n            blz_score = blzf_score_list[i]\n            score_frame = S[:, cnt:cnt+len(blz_score)]\n            score_out_frame = np.max(score_frame, axis=1)\n            for idx_net, score_net in enumerate(score_out_frame):\n                if expit(score_net) <= 0.5:\n                    score_out_frame[idx_net] = np.min(score_frame[idx_net, :])\n            score_video.append(score_out_frame.mean())\n            cnt = cnt + len(blz_score)\n        # final video score: average of all the frames\n        score = expit(np.mean(score_video))\n            \n    except Exception as e:\n        print('Prediction error on video %s: %s' % (name, str(e)))\n        score = 0.5\n        if locked_by_me:\n            net_lock.release()\n    \n    return name, score","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"if len(df) == 400:\n    # Process a video at random\n#     out = process_video(np.random.choice(df.index),debug=True)\n    out = process_video('wixbuuzygv.mp4',debug=True)\n    print(out)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n","is_executing":false},"trusted":true},"cell_type":"code","source":"def predict(video_paths: list):\n    video_paths = list(video_paths)\n    if num_workers == 0:\n        predictions = []\n        for name in tqdm(video_paths):\n            predictions.append(process_video(name))\n    else:\n        with ThreadPoolExecutor(max_workers=num_workers) as ex:\n            predictions = list(tqdm(ex.map(process_video, video_paths),total=len(video_paths)))\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"if len(df) == 400:\n    speedtest = True\n    test_samples = num_workers * 2\n    if speedtest:\n        t0 = time()\n        predictions = predict(df.index[:num_workers*2])\n\n        t1 = time()\n        elapsed = t1 - t0\n        print('Elapsed time for {:d} samples: {:.2f}s'.format(test_samples, elapsed))\n        print('Average speed: {:.2f}s/video'.format(elapsed / test_samples))\n        print(\n            'Forecasted prediction on small test set: {:.0f}min'.format(elapsed / test_samples * len(video_paths) / 60))\n        print('Forecasted prediction on full test set: {:.1f}h'.format(elapsed / test_samples * 4000 / 3600))","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"# Predict on all\npredictions = predict(df.index)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"df['label'] = 0.5\nfor name, video_pred in predictions:\n    df.loc[name, 'label'] = video_pred","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"df_out = df.loc[:, ['label', ]]\ndf_out.index.name = 'filename'\n\n# Save submission\ndf_out.to_csv(submission_path, index=True)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"df_out.hist(bins=50);","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"print('Num records: {}'.format(len(df_out)))\nprint('Min: {}'.format(df_out['label'].min()))\nprint('Max: {}'.format(df_out['label'].max()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":2},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython2","version":"2.7.6"},"pycharm":{"stem_cell":{"cell_type":"raw","source":[],"metadata":{"collapsed":false}}}},"nbformat":4,"nbformat_minor":4}