{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/facenet-pytorch-vggface2/facenet_pytorch-2.0.1-py3-none-any.whl\n!mkdir -p /root/.cache/torch/checkpoints/\n!cp ../input/facenet-pytorch-vggface2/20180402-114759-vggface2-logits.pth /root/.cache/torch/checkpoints/vggface2_DG3kwML46X.pt\n!cp ../input/facenet-pytorch-vggface2/20180402-114759-vggface2-features.pth /root/.cache/torch/checkpoints/vggface2_G5aNV2VSMn.pt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport gc\nfrom glob import glob\nimport json\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport random\nimport cv2\nfrom PIL import Image, ImageDraw\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict, deque\nimport sys\n\nimport torch\nfrom torch.nn import Module\nfrom torch import nn\nfrom torch import optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.utils.data import DataLoader, Dataset, Subset\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nfrom facenet_pytorch import MTCNN, InceptionResnetV1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_path = '../input/deepfake-detection-challenge/sample_submission.csv'\ntrain_video_path = '../input/deepfake-detection-challenge/train_sample_videos'\ntest_video_path = '../input/deepfake-detection-challenge/test_videos'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train video"},{"metadata":{"trusted":true},"cell_type":"code","source":"list_train = glob(os.path.join(train_video_path, '*.mp4'))\nprint(f'Sum video in train: {len(list_train)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test video"},{"metadata":{"trusted":true},"cell_type":"code","source":"list_test = glob(os.path.join(test_video_path, '*.mp4'))\nprint(f'Sum video in test: {len(list_test)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train json"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json = glob(os.path.join(train_video_path, '*.json'))\nwith open(train_json[0], 'rt') as file:\n    train = json.load(file)\n    \ntrain_df = pd.DataFrame()\ntrain_df['file'] = train.keys()\n\nlabel = [i['label'] for i in train.values() if isinstance(i, dict)]\ntrain_df['label'] = label\n\nsplit = [i['split'] for i in train.values() if isinstance(i, dict)]\ntrain_df['split'] = split\n\noriginal = [i['original'] for i in train.values() if isinstance(i, dict)]\ntrain_df['original'] = original\n\ntrain_df['original'] = train_df['original'].fillna(train_df['file'])\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = train_df.iloc[:50, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real = train_df[train_df['label']=='REAL']\nreal.reset_index(inplace=True, drop=True)\nfake = train_df[train_df['label']=='FAKE']\nfake.reset_index(inplace=True, drop=True)\n\nplt.figure(figsize=(15,8))\nax = sns.countplot(y=label, data=train_df)\n\nfor p in ax.patches:\n    ax.annotate('{:.2f}%'.format(100*p.get_width()/train_df.shape[0]), (p.get_x() + p.get_width() + 0.02, p.get_y() + p.get_height()/2))\n    \nplt.title('Distribution of label', size=25, color='b')    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"original_same = train_df.pivot_table(values=['file'], columns=['label'], index=['original'], fill_value=0, aggfunc='count')\noriginal_same = original_same[(original_same[('file', 'FAKE')] != 0) & (original_same[('file', 'REAL')] != 0)]\n\nprint(f'Number of file having both FAKE and REAL: {len(original_same)}')\noriginal_same","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'] = train_df['label'].apply(lambda x: 1 if x=='FAKE' else 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display some video"},{"metadata":{"trusted":true},"cell_type":"code","source":"def box_mtcnn(frame, landmarks=True):\n    mtcnn = MTCNN(margin=14, keep_all=True, device=device, factor=0.5).to(device)\n    if landmarks:\n        boxes, scores, landmarks = mtcnn.detect(frame, landmarks=landmarks)\n        return boxes, scores, landmarks\n    else:\n        boxes, scores = mtcnn.detect(frame, landmarks=landmarks)\n        return boxes, scores","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MTCNN with FaceNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_video(df, number_frame=5, number_video=3):\n    \n    color = ['b', 'g', 'r']\n    for index in range(number_video):\n        \n        index_random = random.randint(0, len(df))\n        video = df.loc[index_random, 'file']\n        \n        if video in os.listdir(train_video_path):\n            video_path = os.path.join(train_video_path, video)\n            cap = cv2.VideoCapture(video_path)\n            \n            fig, axes = plt.subplots(number_frame, 2, figsize=(20, 20))\n            \n            frame_index = 0\n            ax_ix = 0\n            while True:\n                    \n                ret, frame = cap.read()\n                \n                if cv2.waitKey(1) & 0xFF == 27:\n                    break\n                \n                if ret:                    \n                    \n                    if frame_index%24==0:\n                        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                        image = Image.fromarray(image)\n                        boxes, scores = box_mtcnn(image, False)\n                        if scores[0]:\n                            if boxes is not None:\n                                box = boxes[scores.argmax()]\n                                frame_crop = image.crop(box)\n                                frame_crop = np.array(frame_crop)\n\n                                for i in range(3):\n                                    hist = cv2.calcHist([frame_crop], [i], None, [256], [0, 256])\n                                    axes[ax_ix, 1].plot(hist, color=color[i])\n\n\n                                axes[ax_ix, 0].imshow(frame_crop)\n                                axes[ax_ix, 0].xaxis.set_visible(False)\n                                axes[ax_ix, 0].yaxis.set_visible(False)\n                                axes[ax_ix, 0].set_title(f'Frame: {frame_index}')\n                                ax_ix += 1\n\n                                fig.tight_layout()\n\n                                fig.suptitle(video, color='b', size=20, y=1)\n\n                                if ax_ix == number_frame:\n                                    break\n                                                                \n                else:\n                    break\n                    \n                \n                frame_index += 1          \n        \ndisplay_video(fake)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_video(real)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display image with MTCNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"def transfer(label):\n    \n    if label==0:\n        return \"real\"\n    else:\n        return \"fake\"\n    \ndef display_mtcnn(number_frame=3, number_video=2):\n    \n    fake_real = original_same[(original_same[('file', 'FAKE')] == 1) & (original_same[('file', 'REAL')] == 1)].index.tolist()                \n    original_images = random.sample(fake_real, number_video)\n    \n    for original_image in original_images:\n        real_video = train_df[(train_df['label']==0) & (train_df['original']==original_image)]['file'].values[0]\n        fake_video = train_df[(train_df['label']==1) & (train_df['original']==original_image)]['file'].values[0]\n\n        if (real_video in os.listdir(train_video_path)) and (fake_video in os.listdir(train_video_path)):\n            real_path = os.path.join(train_video_path, real_video)\n            fake_path = os.path.join(train_video_path, fake_video)\n\n\n\n            fig, axes = plt.subplots(number_frame, 2, figsize=(20, 20))\n\n            for ind, path in enumerate([real_path, fake_path]):\n\n                cap = cv2.VideoCapture(path)\n                frame_index = 0\n                ax_ix = 0\n                \n                while True:\n                    ret, frame = cap.read()\n\n                    if cv2.waitKey(1) & 0xFF == 27:\n                        break\n\n                    if ret:                    \n\n                        if frame_index%24==0:\n                            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                            frame = Image.fromarray(frame)\n                            boxes, scores = box_mtcnn(frame, False)\n                            box = boxes[scores.argmax()]\n                            frame_crop = frame.crop(box)\n                            \n                            boxes, scores, landmarks = box_mtcnn(frame_crop)\n                            if landmarks is not None:\n                                landmark = landmarks[scores.argmax()]\n                                axes[ax_ix, ind].scatter(landmark[:, 0], landmark[:, 1], c='red', s=8)\n\n                                axes[ax_ix, ind].imshow(frame_crop)\n                                axes[ax_ix, ind].xaxis.set_visible(False)\n                                axes[ax_ix, ind].yaxis.set_visible(False)\n                                axes[ax_ix, ind].set_title(f'Frame: {frame_index}_{transfer(ind)}')\n\n                                fig.tight_layout()\n                                ax_ix += 1\n\n                                if ax_ix == number_frame:\n                                    break\n\n                    else:\n                        break\n                    \n                    frame_index+=1\n                    \n            fig.suptitle(original_image, color='b', size=20, y=1)\n\n\ndisplay_mtcnn(number_frame=3, number_video=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creat dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class VideoDataset(Dataset):\n    \n    def __init__(self, df, path_video, num_frame=5, is_train=True):\n        super(VideoDataset, self).__init__()\n        \n        self.df = df\n        self.num_frame = num_frame\n        self.is_train = is_train\n        self.path_video = path_video\n        \n        index_list = deque()\n        for index in tqdm(range(len(self.df))):\n            \n            video_name = self.df.loc[index, 'file']\n            video_path = os.path.join(self.path_video, video_name)\n            \n            if self.landmark_mtcnn(video_path) is not None:\n                index_list.append(index)\n                \n        index_list = list(index_list)\n        self.df = self.df[self.df.index.isin(index_list)]\n        self.df.reset_index(inplace=True, drop=True)\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        video_name = self.df.loc[idx, 'file']\n        video_path = os.path.join(self.path_video, video_name)\n        list_landmark = self.landmark_mtcnn(video_path)\n        \n        if self.is_train:\n            label = self.df.loc[idx, 'label']\n            return torch.from_numpy(list_landmark), torch.tensor(label, dtype=torch.float)\n        else:\n            return video_name, torch.from_numpy(list_landmark)\n        \n        \n    def landmark_mtcnn(self, video_path):\n\n        cap = cv2.VideoCapture(video_path)\n        frame_index = 0\n\n        list_landmark = deque()\n        \n        while len(list_landmark) < 10*self.num_frame:\n            \n                \n            ret, frame = cap.read()\n\n            if cv2.waitKey(1) & 0xFF == 27:\n                break\n\n            if ret:                    \n                if frame_index % 24 == 0:\n                    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                    frame = Image.fromarray(frame)\n                    boxes, scores, landmarks = box_mtcnn(frame)\n\n                    if scores[0]:\n                        index_max = np.argmax(scores)\n                        landmark = landmarks[index_max]\n\n                        list_landmark.extend(landmark.flatten())\n\n            else:\n                break\n\n            frame_index+=1\n        \n        list_landmark = list(list_landmark)\n        if len(list_landmark) == 10*self.num_frame:\n            list_landmark = np.array(list_landmark).reshape(self.num_frame, 10)\n\n            return list_landmark\n        return None\n    \n    \ndataset = VideoDataset(train_df, train_video_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_size = 0.2\nindex_split = int(len(dataset)*test_size)\nlist_index = (list(range(len(dataset))))\nrandom.shuffle(list_index)\n\ntrain_idx = list_index[index_split:]\nval_idx = list_index[:index_split]\n\ntrain_dataset = Subset(dataset, train_idx)\nval_dataset = Subset(dataset, val_idx)\n\ntrain_ld = DataLoader(train_dataset, batch_size=8, shuffle=True)\nval_ld = DataLoader(val_dataset, batch_size=8, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Module"},{"metadata":{"trusted":true},"cell_type":"code","source":"class swish(Module):\n  \n  def __init__(self):\n    super(swish, self).__init__()\n    \n    self.sigmoid = nn.Sigmoid()\n    \n  def forward(self, x):\n    return x*self.sigmoid(x)\n\n\nclass small_model(Module):\n    def __init__(self, num_class=1):\n        super(small_model, self).__init__()\n        \n        self.conv = nn.Sequential(nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n                                  nn.BatchNorm2d(32),\n                                  nn.ReLU(inplace=True),\n                                  nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n                                  nn.BatchNorm2d(64),\n                                  nn.ReLU(inplace=True),\n                                  nn.Conv2d(64, 64, kernel_size=1, stride=1, padding=0),\n                                  nn.BatchNorm2d(64),\n                                  nn.Dropout(0.2))\n        \n        self.fc = nn.Sequential(nn.Linear(64*10*5, 128),\n                                nn.ReLU(inplace=True),\n                                nn.BatchNorm1d(128),\n                                nn.Dropout(0.2),\n                                nn.Linear(128, num_class),\n                                nn.Sigmoid())\n        \n        \n    def forward(self, x):\n        \n        x = self.conv(x)\n        x = torch.flatten(x, 1)        \n        x = self.fc(x)\n        \n        return x\n    \nmodel = small_model().to(device)\nmodel.eval()    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer(object):\n    \n    def __init__(self, model):\n        \n        self.model = model\n        self.creation = nn.MSELoss()\n        \n        self.optimizer = optim.AdamW([      \n            {'params': model.conv.parameters(), 'lr': 1e-4},\n            {'params': model.fc.parameters(), 'lr': 1e-3}], lr=0.001)\n        \n        self.scheduler = ReduceLROnPlateau(self.optimizer, mode='min', factor=0.15)\n        \n    def train_process(self, train_ld, val_ld, epochs):\n        score_max = 0\n        check_step = 0\n        loss_min = 1\n        check_number = 13\n        \n        self.model.train()\n        for epoch in range(epochs):\n            train_loss, val_loss = 0, 0\n            for crop, label in tqdm(train_ld):                \n                crop = crop.unsqueeze(1)\n                crop, label = crop.float().to(device), label.to(device)\n                                \n                self.optimizer.zero_grad()\n                output = self.model(crop).squeeze(1)\n                loss = self.creation(output, label)\n                loss.backward()\n                                \n                self.optimizer.step()\n                self.scheduler.step(train_loss)\n                train_loss += loss.item()\n                \n                del crop, label\n            \n            train_loss = train_loss/len(train_ld)\n            torch.cuda.empty_cache()\n            \n            gc.collect()\n            \n            self.model.eval()\n            \n            val_score = 0\n            with torch.no_grad():\n                for crop, label in tqdm(val_ld):\n                    crop = crop.unsqueeze(1)\n                    crop, label = crop.float().to(device), label.to(device)                    \n                    \n                    output = self.model(crop).squeeze(1)\n                    loss = self.creation(output, label)\n                    val_loss += loss.item()\n                    val_score += torch.sum((output>0.5).float() == label).item()/len(label)\n                \n                val_loss = val_loss/len(val_ld)\n                val_score = val_score/len(val_ld)\n                \n            self.scheduler.step(val_loss)\n            \n            if val_score > score_max:\n                print(f'Epoch: {epoch}, train loss: {train_loss:.5f}, val_loss: {val_loss:.5f}.\\nValidation score increased from {score_max:.5f} to {val_score:.5f}')\n                score_max = val_score\n                loss_min = val_loss\n                torch.save(self.model.state_dict(), 'model.pth')\n                print('Saving model!')\n                check_step = 0\n                \n            elif val_score == score_max:\n                if val_loss < loss_min:\n                    print(f'Epoch: {epoch}, train loss: {train_loss:.5f}, val_loss: {val_loss:.5f}, val_score: {val_score:.5f}.\\nValidation loss decreased from {loss_min:.5f} to {val_loss:.5f}')\n                    loss_min = val_loss\n                    torch.save(self.model.state_dict(), 'model.pth')\n                    print('Saving model!')\n                    check_step = 0\n                else:\n                    check_step += 1\n                    print(f'Epoch: {epoch}, train loss: {train_loss:.5f}, val_loss: {val_loss:.5f}, val_score: {val_score:.5f}.\\nModel not improve in {str(check_step)} step')\n                    if check_step > check_number:\n                        print('Stop trainning!')\n                        break\n            else:\n                check_step += 1\n                print(f'Epoch: {epoch}, train loss: {train_loss:.5f}, val_loss: {val_loss:.5f}.\\nValidation score not increased from {val_score:.5f} in {str(check_step)} step')\n                \n                if check_step > check_number:\n                    print('Stop trainning!')\n                    break\n                    \ntrainer = Trainer(model)        \ntrainer.train_process(train_ld=train_ld, val_ld=val_ld, epochs=20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}