{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Siamese Training\n\n* Training is done with 2 phases / models\n* Phase 1 trains a CNN to output embeddings that have high pairwise distance between fake and real images and low distance between fake and fake\n* Phase 2 adds a linear layer to above CNN to do binary classification\n* It is not clear how / if backpropagation will work when the same model object is used to backpropagate the loss from ouput of 2 forward passes for each pair.\n\nhttps://github.com/radekosmulski/whale/blob/master/siamese_network_prototype.ipynb\n\n*Note: This did not yield good results; the contrastive loss was converging to the margin value and not reducing towards 0. I abandoned it after very few experiments. But it could have worked after a few more tweaks - who knows.*\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport cv2\nimport glob\nimport fastai\nimport PIL\nimport torch\nimport glob\nfrom functools import partial\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nfrom fastai.basics import *\nfrom fastai.vision import learner\n\nfrom tqdm import tqdm\nimport torch\nimport torchvision\nfrom torchvision import transforms\n\n\ntqdm.pandas()\n\nINPUT_PATH = \"../input/deepfake-detection-challenge\"\nVERBOSE = True\nEPS = 1e-5\nRUN_NOTEBOOK=False\nFACES_PATH = 'faces'\n\nos.makedirs(FACES_PATH, exist_ok=True)\nfrom dfdc_face_extractor import *\nfrom dfdc_fastai_reusables import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"pair_df = pd.read_csv(f'../input/fakereal-pairs-in-dfdc-test-videos/dfdc_test_video_pairs.csv')\n\nextractor = DFDCVideoFaceExtractor(backend='CV2')\npair_df = pair_df[:20]\nfor index, row in tqdm(pair_df.iterrows(), total=len(pair_df)):\n    video_filename = row[\"filename\"]\n    basename, _ = basename_and_ext(video_filename)\n    video_path=f'{INPUT_PATH}/test_videos/{video_filename}'\n    extractor.extract_faces(video_path, seq_length=10,stride=1, faces_path=\"faces\", margin=1)\n\n    video_filename = row[\"original\"]\n    basename, _ = basename_and_ext(video_filename)\n    video_path=f'{INPUT_PATH}/test_videos/{video_filename}'\n    extractor.extract_faces(video_path, seq_length=10,stride=1, faces_path=\"faces\", margin=1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmean, std = torch.tensor(imagenet_stats)\n\nclass SiamesePair(ItemBase):\n    def __init__(self, img1, img2): ## These should of Image type\n        self.img1, self.img2 = img1, img2\n        self.obj, self.data = (img1, img2), [\n            (img1.data-mean[...,None,None])/std[...,None,None],\n            (img2.data-mean[...,None,None])/std[...,None,None]\n        ]\n    def apply_tfms(self, tfms,*args, **kwargs):\n        self.img1 = self.img1.apply_tfms(tfms, *args, **kwargs)\n        self.img2 = self.img2.apply_tfms(tfms, *args, **kwargs)\n        self.data = [(self.img1.data-mean[...,None,None])/std[...,None,None],\n                     (self.img2.data-mean[...,None,None])/std[...,None,None]\n                    ]\n        return self\n    def __repr__(self): return f'{self.__class__.__name__} {self.img1.shape, self.img2.shape}'\n    def to_one(self):\n        return Image(mean[...,None,None]+torch.cat(self.data,-1)*std[...,None,None])\n\nnormalize = partial(transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225]))\n\ndenormalize = partial(transforms.Normalize(\n   mean=[-0.485/0.229, -0.456/0.224, -0.406/0.225],\n   std=[1/0.229, 1/0.224, 1/0.225]\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SiameseImageList(ImageList):\n    @classmethod\n    def from_df(cls, df,**kwargs):\n        return cls(items=range(len(df)),inner_df=df, **kwargs)\n\n    def to_img(self, path):\n        m = Image.open(path)\n        return m\n        \n    def get(self, i):\n        row = self.inner_df.iloc[i]\n        first = row['first']\n        first = mp4_to_glob(first)\n        # Randomly selects one face per video\n        first_files = glob.glob(first)\n        first = random.choice(first_files)\n        \n        second=row['second']\n        second = mp4_to_glob(second)\n        second_files = glob.glob(second)\n        # Do not include first file if both sets are from the same fake video\n        # second_files = [f for f in second_files if f != first ]\n        second = random.choice(second_files)\n        first = super().open(first)\n        second = super().open(second)\n        return SiamesePair(first, second)\n\n    def reconstruct(self, t):\n        return SiamesePair(mean[...,None,None]+t[0]*std[...,None,None], \n                            mean[...,None,None]+t[1]*std[...,None,None])\n    \n    def show_xys(self, xs, ys, figsize:Tuple[int,int]=(9,10), **kwargs):\n        rows = int(math.sqrt(len(xs)))\n        fig, axs = plt.subplots(rows,rows,figsize=figsize)\n        for i, ax in enumerate(axs.flatten() if rows > 1 else [axs]):\n            xs[i].to_one().show(ax=ax, y=ys[i], **kwargs)\n        plt.tight_layout()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_siamese_data(bs=4, faces_path='faces', tfms=[[],[]] ):\n    unlike_df = pd.read_csv(f'../input/fakereal-pairs-in-dfdc-test-videos/dfdc_test_video_pairs.csv')\n    files = os.listdir(faces_path)\n    videos = [jpg_to_mp4name(f) for f in files]\n    \n    unlike_df = unlike_df[unlike_df['filename'].isin(videos)].copy()\n    unlike_df = unlike_df[unlike_df['original'].isin(videos)].copy()\n    unlike_df.rename(columns={'filename':'first', 'original': 'second'}, inplace=True)\n    unlike_df['label']=1\n    unlike_df['is_valid']=False\n    unlike_df['is_valid'].iloc[:5] = True\n\n    like_df = unlike_df.copy()\n    like_df['second']=like_df['first']\n    like_df['label']=0\n    \n    siamese_df=pd.concat([unlike_df, like_df], axis=0)\n    siamese_df.reset_index(inplace=True)\n    siamese_df.drop(columns=['index'], inplace=True)\n    siamese_df = siamese_df.sample(frac=1)\n    databunch = SiameseImageList.from_df(siamese_df)\\\n                     .split_from_df(col='is_valid')\\\n                     .label_from_df(cols='label')\\\n                     .transform(tfms, size=(224,224))\\\n                    .databunch(bs=4)\n    \n    return databunch\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = get_dfdc_transforms()\ndata = get_siamese_data(tfms=tfms)\ndata.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss Function\n\nhttps://www.kaggle.com/c/quora-question-pairs/discussion/33631\n\nhttps://discuss.pytorch.org/t/triplet-loss-in-pytorch/30634/3"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SiameseLoss(nn.Module):\n    \"\"\"\n    Custom loss designed to decrease distance between predictions for like pairs and\\\n    increase distance between predictions for unlike pairs.\n    \"\"\"\n    def __init__(self, margin=5, eps=1e-3):\n        super(SiameseLoss, self).__init__()\n        self.margin=margin\n        self.eps=eps\n        \n    def bce_loss(self, p,n, target, size_average=True):\n        ps = torch.sigmoid(p)[:,-1]\n        pt = torch.ones_like(p)\n        ns = torch.sigmoid(n)[:,-1]\n        nt = target.unsqueeze(-1)\n        logps = torch.log(ps)\n        logns = torch.log(1-ns)\n        d1 = -pt*logps\n        d2 = -(1-nt)*logns\n        d = d1+d2\n        return d.mean() if size_average else d.sum()\n        \n    def contrastive_loss(self, p,n, target, size_average=True):\n        euclidean_distance = F.pairwise_distance(p, n, keepdim = True)\n        tgt = target.unsqueeze(-1).float()\n        term1 = (tgt) * torch.pow(euclidean_distance, 2)\n        term2 = (1-tgt) * torch.pow(torch.clamp(self.margin - euclidean_distance, min=0.0), 2)\n        t = torch.cat((tgt, euclidean_distance, term1, term2),dim=1)\n        loss_contrastive = torch.mean(term1 + term2)\n        return loss_contrastive\n    \n    def forward(self, siamese_out, target, size_average=True):\n        p, n = siamese_out[0], siamese_out[1]\n        return self.contrastive_loss(p,n, target, size_average=size_average)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Siamese Model\n\nhttps://github.com/afitts/kaggle/blob/master/competitions/humpback-whale/siamese-with-fast-ai.ipynb"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SiameseNet(nn.Module):\n    def __init__(self, arch=models.resnet50, lin_ftrs=[256], emb_sz=128,ps=0., bn_final=True):\n        super(SiameseNet, self).__init__()\n        self.arch, self.emb_sz = arch, emb_sz\n        self.lin_ftrs, self.ps, self.bn_final = lin_ftrs, ps, bn_final\n        self.body = learner.create_body(self.arch, True, learner.cnn_config(self.arch)['cut'])\n        self.head = learner.create_head(num_features_model(self.body)*2, self.emb_sz, self.lin_ftrs, self.ps,self.bn_final)\n        self.cnn = nn.Sequential(self.body, self.head)\n                                  \n    def forward(self, fake, original):\n        p = self.cnn(fake)\n        n = self.cnn(original)\n        return p, n\n\n    def get_embedding(self, x):\n        return self.cnn(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training Phase 1\nmodel_dir = 'models/dfdc-siamese'\nos.makedirs(model_dir, exist_ok=True)\ndata = get_siamese_data()\nmodel = SiameseNet()\n\nloss_func = SiameseLoss()\nsiamese_1_learn = Learner(data,\n                model,\n                loss_func=loss_func,\n                model_dir=model_dir)\n\nsiamese_1_learn.fit(1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training Phase 2\nsiamese_cnn = siamese_1_learn.model.cnn\nhead = bn_drop_lin(128,1,True,p=0.25)\nsiamese_2_net = nn.Sequential(siamese_cnn, *head)\n\n\nmodel_dir = 'models/dfdc-siamese-2'\nos.makedirs(model_dir, exist_ok=True)\ndata = get_deepfakeimagelist_data()\n\n\nsiamese_2_learn = Learner(data,\n                model=siamese_2_net,\n                loss_func=BCEWithLogitsFlat(),\n                metrics=[DFDCAUROC(),RealLoss(),FakeLoss()],\n                model_dir=model_dir)\nsiamese_2_learn.split( lambda m: m[1])\nsiamese_2_learn.unfreeze()\nsiamese_2_learn.freeze_to(1)\nsiamese_2_learn.fit(1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}