{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfrom glob import glob\n\nfor path in glob('../input/*/*'):\n    print(path)\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import random\nimport re\nfrom copy import deepcopy\nfrom typing import Union, List, Tuple, Optional, Callable\nfrom collections import OrderedDict, defaultdict\nimport math\n\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torchvision import transforms, models\nfrom torchvision.transforms import Normalize\nfrom tqdm import tqdm\nfrom sklearn.cluster import DBSCAN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_H, TARGET_W = 224, 224\nFRAMES_PER_VIDEO = 30\nTEST_VIDEOS_PATH = '../input/deepfake-detection-challenge/test_videos'\nNN_MODEL_PATHS = [\n    '../input/kdold-deepfake-effb2/fold0-effb2-000epoch.pt',\n    '../input/kdold-deepfake-effb2/fold0-effb2-001epoch.pt',\n    '../input/kdold-deepfake-effb2/fold0-effb2-002epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-000epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-001epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-002epoch.pt',\n    \n    '../input/kdold-deepfake-effb2/fold1-effb2-000epoch.pt',\n    '../input/kdold-deepfake-effb2/fold1-effb2-001epoch.pt',\n    '../input/kdold-deepfake-effb2/fold1-effb2-002epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-000epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-001epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-002epoch.pt',\n    \n    '../input/kdold-deepfake-effb2/fold2-effb2-000epoch.pt',\n    '../input/kdold-deepfake-effb2/fold2-effb2-001epoch.pt',\n    '../input/kdold-deepfake-effb2/fold2-effb2-002epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-000epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-001epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-002epoch.pt',\n\n    '../input/kdold-deepfake-effb2/fold3-effb2-000epoch.pt',\n    '../input/kdold-deepfake-effb2/fold3-effb2-001epoch.pt',\n    '../input/kdold-deepfake-effb2/fold3-effb2-002epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-000epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-001epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-002epoch.pt',\n\n    '../input/kdold-deepfake-effb2/fold4-effb2-000epoch.pt',\n    '../input/kdold-deepfake-effb2/fold4-effb2-001epoch.pt',\n    '../input/kdold-deepfake-effb2/fold4-effb2-002epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-000epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-001epoch.pt',\n    '../input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-002epoch.pt',\n]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = False\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"/kaggle/input/face-detector\")\n\nfrom face_detector import FaceDetector\nfrom face_detector.utils import VideoReader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pytorchefficientnet/EfficientNet-PyTorch-master > /dev/null\n\nfrom efficientnet_pytorch import EfficientNet\n\ndef get_net():\n    net = EfficientNet.from_name('efficientnet-b2')\n    net._fc = nn.Linear(in_features=net._fc.in_features, out_features=2, bias=True)\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, df):\n        self.video_paths = df['video_path']\n        self.filenames = df.index\n        self.face_dr = FaceDetector(frames_per_video=FRAMES_PER_VIDEO)\n\n        mean = [0.485, 0.456, 0.406]\n        std = [0.229, 0.224, 0.225]\n        self.normalize_transform = Normalize(mean, std)\n        \n        self.video_reader = VideoReader()\n        self.video_read_fn = lambda x: self.video_reader.read_frames(x, num_frames=FRAMES_PER_VIDEO)\n\n    def __len__(self):\n        return self.filenames.shape[0]\n\n    def __getitem__(self, idx):\n        video_path = self.video_paths[idx]\n        filename = self.filenames[idx]\n        \n        my_frames, my_idxs = self.video_read_fn(video_path)\n        faces = self.face_dr.get_faces(\n            my_frames, my_idxs,\n            0.7, 0.7, 0.7, 0.6\n        )\n\n        n = len(faces)\n\n        video = torch.zeros((n, 3, TARGET_H, TARGET_W))\n        for i, face in enumerate(faces[:n]):\n            face = 255 - face\n            face = face.astype(np.float32)/255.\n            face = torch.tensor(face)\n            face = face.permute(2,0,1)\n            face = self.normalize_transform(face)\n            video[i] = face\n\n        return filename, video","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nvideos = []\nfor video_path in glob(os.path.join(TEST_VIDEOS_PATH, '*.mp4')):\n    videos.append({'filename': video_path.split('/')[-1], 'video_path': video_path})\n    \ndf = pd.DataFrame(videos).set_index('filename')\n\nvideos = None\ndel videos\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage import io\n\nfor filename, video in DatasetRetriever(df[:1]):\n    break\n    \nio.imshow(1 - video.permute(0,2,3,1).numpy()[5,:,:,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DeepFakePredictor:\n\n    def __init__(self):\n        self.models = [self.prepare_model(get_net(), path) for path in NN_MODEL_PATHS]\n        self.models_count = len(self.models)\n\n    def predict(self, dataset):\n        result = []\n        \n        with torch.no_grad():\n            for filename, video in dataset:\n                video = video.to(self.device, dtype=torch.float32)\n                try:\n                    label = self.predict_ensemble(video)\n                except Exception as e:\n                    print(f'Warning! {e}, {type(e)}')\n                    label = 0.5\n\n                result.append({\n                    'filename': filename,\n                    'label': label,\n                })\n\n        return pd.DataFrame(result).set_index('filename')\n\n    def prepare_model(self, model, path):\n        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n        model.to(self.device);\n\n        if torch.cuda.is_available():\n            model = model.cuda()\n            \n        if torch.cuda.is_available():\n            checkpoint = torch.load(path)\n        else:\n            checkpoint = torch.load(path, map_location=torch.device('cpu'))\n            \n        model.load_state_dict(checkpoint['model_state_dict'])\n        model.eval()\n        print(f'Model prepared. Device is {self.device}')\n        return model\n    \n    @staticmethod\n    def net_forward(net, inputs):\n        bs = inputs.size(0)\n        # Convolution layers\n        x = net.extract_features(inputs)\n        # Pooling and final linear layer\n        x = net._avg_pooling(x)\n        emb = x.view(bs, -1)\n        x = net._dropout(emb)\n        x = net._fc(x)\n        return emb, x\n    \n    def postprocess(self, embs, predictions):\n        clusters = defaultdict(list)\n        for prediction, cluster_id in zip(predictions, DBSCAN(eps=1.2, min_samples=1).fit_predict(embs)):\n            clusters[cluster_id].append(prediction)\n        sorted_clusters = sorted(clusters.items(), key=lambda x: -len(x[1]))\n        if len(sorted_clusters) < 2:\n            return sorted_clusters[0][1]\n        if len(sorted_clusters[1][1]) / len(predictions) > 0.25:\n            return sorted_clusters[0][1] + sorted_clusters[1][1]\n        return sorted_clusters[0][1]\n    \n    def predict_ensemble(self, video):\n        embs, predictions = 0, 0\n        for model in self.models:\n            emb, prediction = self.net_forward(model, video)\n            predictions += prediction / self.models_count\n            embs += emb / self.models_count\n\n        predictions = nn.functional.softmax(predictions, dim=1).data.cpu().numpy()[:,1]\n        embs = embs.cpu().numpy()\n        \n        predictions = self.postprocess(embs, predictions)\n        return np.mean(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"deep_fake_predictor = DeepFakePredictor()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\n\ndef process_dfs(df, num_workers=2):\n    def process_df(sub_df):\n        dataset = DatasetRetriever(sub_df)\n        result = deep_fake_predictor.predict(dataset)\n        return result\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        results = ex.map(process_df, np.split(df, num_workers))\n\n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nimport time\n\ncount = df.shape[0]\n\n\ntime_start = time.time()\nresults = process_dfs(df[:count])\ndtime = time.time() - time_start\n\nprint(f'[speed]:', round(dtime / count, 2), 'sec/video')\nprint(f'[sum_time]:', f'~{round(dtime / count * 4000 / 60)}', 'min')\n\nresult = pd.concat(list(results))\nresult","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}