{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport gc\nimport cv2\nimport glob\nimport time\nimport copy\nfrom tqdm import tqdm_notebook as tqdm\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import models, transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trained_weights_path = '../input/deepfake-model-weights/vgg19_ep5_20191219.pth'\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = '../input/deepfake-detection-challenge/test_videos'\nos.listdir(test_dir)[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_from_mov(video_file, show_img=False):\n    # https://note.nkmk.me/python-opencv-videocapture-file-camera/\n    cap = cv2.VideoCapture(video_file)\n    frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    \n    image_list = []\n    for i in range(frames):\n        _, image = cap.read()\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image_list.append(image)\n    cap.release()\n\n    if show_img:\n        fig, ax = plt.subplots(1,1, figsize=(15, 15))\n        ax.imshow(image[0])\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)\n        ax.title.set_text(f\"FRAME 0: {video_file.split('/')[-1]}\")\n        plt.grid(False)\n        \n    return image_list\n\ndef detect_face(img):\n    # Add Dataset \"Haarcascades\"\n    face_cascade = cv2.CascadeClassifier('../input/haarcascades/haarcascade_frontalface_alt.xml')\n    face_crops = face_cascade.detectMultiScale(img, scaleFactor=1.3, minNeighbors=5)\n    \n    if len(face_crops) == 0:\n        return []\n    \n    crop_imgs = []\n    for i in range(len(face_crops)):\n        x = face_crops[i][0]\n        y = face_crops[i][1]\n        w = face_crops[i][2]\n        h = face_crops[i][3]\n        #x,y,w,h=ratio*x,ratio*y,ratio*w,ratio*h\n        crop_imgs.append(img[y:y+h, x:x+w])\n    return crop_imgs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Resize(object):\n    def __init__(self, size=300):\n        self.size = size\n\n    def __call__(self, image):\n        image = cv2.resize(image, (self.size, self.size))\n        return image\n\n# Data Augumentation\nclass ImageTransform():\n    def __init__(self, resize):\n        self.data_transform = {\n            'test': transforms.Compose([\n                Resize(resize),\n                transforms.ToTensor(),\n            ])\n        }\n        \n    def __call__(self, img, phase):\n        return self.data_transform[phase](img)\n\n\nclass DeepfakeDataset(Dataset):\n    def __init__(self, file_list, transform=None, phase='test'):\n        self.file_list = file_list\n        self.transform = transform\n        self.phase = phase\n        \n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self, idx):\n        \n        mov_path = self.file_list[idx]\n        # first frame image only\n        image = get_img_from_mov(mov_path, show_img=False)[0]\n        # FaceCrop\n        image = detect_face(image)[0]\n        # Transform\n        image = self.transform(image, self.phase)\n        \n        return image, mov_path","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Model"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py\n__all__ = [\n    'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',\n    'vgg19_bn', 'vgg19',\n]\n\n\nmodel_urls = {\n    'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth',\n    'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth',\n    'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth',\n    'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth',\n    'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth',\n    'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth',\n    'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth',\n    'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth',\n}\n\n\nclass VGG(nn.Module):\n\n    def __init__(self, features, num_classes=1000, init_weights=True):\n        super(VGG, self).__init__()\n        self.features = features\n        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))\n        self.classifier = nn.Sequential(\n            nn.Linear(512 * 7 * 7, 4096),\n            nn.ReLU(True),\n            nn.Dropout(),\n            nn.Linear(4096, 4096),\n            nn.ReLU(True),\n            nn.Dropout(),\n            nn.Linear(4096, num_classes),\n        )\n        if init_weights:\n            self._initialize_weights()\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.classifier(x)\n        return x\n\n    def _initialize_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n            elif isinstance(m, nn.Linear):\n                nn.init.normal_(m.weight, 0, 0.01)\n                nn.init.constant_(m.bias, 0)\n\n\ndef make_layers(cfg, batch_norm=False):\n    layers = []\n    in_channels = 3\n    for v in cfg:\n        if v == 'M':\n            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n        else:\n            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)\n            if batch_norm:\n                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n            else:\n                layers += [conv2d, nn.ReLU(inplace=True)]\n            in_channels = v\n    return nn.Sequential(*layers)\n\n\ncfgs = {\n    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n    'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n    'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],\n    'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],\n}\n\n\ndef _vgg(arch, cfg, batch_norm, pretrained, progress, **kwargs):\n    if pretrained:\n        kwargs['init_weights'] = False\n    model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch],\n                                              progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\n\ndef vgg11(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 11-layer model (configuration \"A\") from\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg11', 'A', False, pretrained, progress, **kwargs)\n\n\ndef vgg11_bn(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 11-layer model (configuration \"A\") with batch normalization\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg11_bn', 'A', True, pretrained, progress, **kwargs)\n\n\ndef vgg13(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 13-layer model (configuration \"B\")\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg13', 'B', False, pretrained, progress, **kwargs)\n\n\ndef vgg13_bn(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 13-layer model (configuration \"B\") with batch normalization\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg13_bn', 'B', True, pretrained, progress, **kwargs)\n\n\ndef vgg16(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 16-layer model (configuration \"D\")\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg16', 'D', False, pretrained, progress, **kwargs)\n\n\ndef vgg16_bn(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 16-layer model (configuration \"D\") with batch normalization\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg16_bn', 'D', True, pretrained, progress, **kwargs)\n\n\ndef vgg19(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 19-layer model (configuration \"E\")\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg19', 'E', False, pretrained, progress, **kwargs)\n\n\ndef vgg19_bn(pretrained=False, progress=True, **kwargs):\n    r\"\"\"VGG 19-layer model (configuration 'E') with batch normalization\n    `\"Very Deep Convolutional Networks For Large-Scale Image Recognition\" <https://arxiv.org/pdf/1409.1556.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _vgg('vgg19_bn', 'E', True, pretrained, progress, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = vgg19()\nmodel.classifier[6] = nn.Linear(in_features=4096, out_features=2)\nmodel.load_state_dict(torch.load(trained_weights_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_file = [os.path.join(test_dir, path) for path in os.listdir(test_dir)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_file[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction\npred_list = []\npath_list = []\n\nwith torch.no_grad():\n    for mov_path in tqdm(test_file):\n        pred = 0\n        try:\n            # first frame image only\n            img = get_img_from_mov(mov_path, show_img=False)\n            # All Frame Image\n            # If face cannot be detected, prediction is 0.5\n            _img = img[0]\n            # FaceCrop\n            _img = detect_face(_img)[0]\n            # Transform\n            _img = ImageTransform(resize=224)(_img, 'test')\n\n            _img = _img.unsqueeze(0)\n            _img = _img.to(device)\n            model.to(device)\n            model.eval()\n\n            output = model(_img)\n            pred += F.softmax(output, dim=1)[:, 1].tolist()[0]\n            \n            del img, _img\n            gc.collect()\n        except:\n            pred += 0.5\n        \n        pred_list.append(pred)\n        path_list.append(mov_path.split('/')[-1])\n        \ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Submission\nres = pd.DataFrame({\n    'filename': path_list,\n    'label': pred_list,\n})\n\nres.sort_values(by='filename', ascending=True, inplace=True)\n\nres.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}