{"metadata":{"colab":{"name":"Deepfake_Detection.ipynb","provenance":[],"collapsed_sections":["AaU3SeKMz_qm","XsefoEdR1gHt"],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","language_info":{"name":""},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":735991,"sourceType":"datasetVersion","datasetId":379454},{"sourceId":893807,"sourceType":"datasetVersion","datasetId":451078}],"dockerImageVersionId":29845,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Simple baseline binary classifier using FFHQ dataset to balance the data\nThis kernal shows a simple training pipeline. I'm sure a lot can be improved upon.  \nView this kernal for inference and submission: https://www.kaggle.com/greatgamedota/xception-binary-classifier-inference\n\nThanks to:  \n[@unkownhihi](https://www.kaggle.com/unkownhihi) for dataset and corresponding kernal: https://www.kaggle.com/unkownhihi/starter-kernel-with-cnn-model-ll-lb-0-69235  \n[@humananalog](https://www.kaggle.com/humananalog) for inference kernal: https://www.kaggle.com/humananalog/inference-demo\n\nLink to my FFHQ dataset: https://www.kaggle.com/greatgamedota/ffhq-face-data-set\n\nUpdate 1: Fixed data leak when balancing data and added more augmentations","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nfrom tqdm import tqdm,trange\nfrom sklearn.model_selection import train_test_split\nimport sklearn.metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"YjwlB710mIH_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setup Data","metadata":{"id":"AaU3SeKMz_qm"}},{"cell_type":"code","source":"df_train0 = pd.read_json('../input/deepfake/metadata0.json')\ndf_train1 = pd.read_json('../input/deepfake/metadata1.json')\ndf_train2 = pd.read_json('../input/deepfake/metadata2.json')\ndf_train3 = pd.read_json('../input/deepfake/metadata3.json')\ndf_train4 = pd.read_json('../input/deepfake/metadata4.json')\ndf_train5 = pd.read_json('../input/deepfake/metadata5.json')\ndf_train6 = pd.read_json('../input/deepfake/metadata6.json')\ndf_train7 = pd.read_json('../input/deepfake/metadata7.json')\ndf_train8 = pd.read_json('../input/deepfake/metadata8.json')\ndf_train9 = pd.read_json('../input/deepfake/metadata9.json')\ndf_train10 = pd.read_json('../input/deepfake/metadata10.json')\ndf_train11 = pd.read_json('../input/deepfake/metadata11.json')\ndf_train12 = pd.read_json('../input/deepfake/metadata12.json')\ndf_train13 = pd.read_json('../input/deepfake/metadata13.json')\ndf_train14 = pd.read_json('../input/deepfake/metadata14.json')\ndf_train15 = pd.read_json('../input/deepfake/metadata15.json')\ndf_train16 = pd.read_json('../input/deepfake/metadata16.json')\ndf_train17 = pd.read_json('../input/deepfake/metadata17.json')\ndf_train18 = pd.read_json('../input/deepfake/metadata18.json')\ndf_train19 = pd.read_json('../input/deepfake/metadata19.json')\ndf_train20 = pd.read_json('../input/deepfake/metadata20.json')\ndf_train21 = pd.read_json('../input/deepfake/metadata21.json')\ndf_train22 = pd.read_json('../input/deepfake/metadata22.json')\ndf_train23 = pd.read_json('../input/deepfake/metadata23.json')\ndf_train24 = pd.read_json('../input/deepfake/metadata24.json')\ndf_train25 = pd.read_json('../input/deepfake/metadata25.json')\ndf_train26 = pd.read_json('../input/deepfake/metadata26.json')\ndf_train27 = pd.read_json('../input/deepfake/metadata27.json')\ndf_train28 = pd.read_json('../input/deepfake/metadata28.json')\ndf_train29 = pd.read_json('../input/deepfake/metadata29.json')\ndf_train30 = pd.read_json('../input/deepfake/metadata30.json')\ndf_train31 = pd.read_json('../input/deepfake/metadata31.json')\ndf_train32 = pd.read_json('../input/deepfake/metadata32.json')\ndf_train33 = pd.read_json('../input/deepfake/metadata33.json')\ndf_train34 = pd.read_json('../input/deepfake/metadata34.json')\ndf_train35 = pd.read_json('../input/deepfake/metadata35.json')\ndf_train36 = pd.read_json('../input/deepfake/metadata36.json')\ndf_train37 = pd.read_json('../input/deepfake/metadata37.json')\ndf_train38 = pd.read_json('../input/deepfake/metadata38.json')\ndf_train39 = pd.read_json('../input/deepfake/metadata39.json')\ndf_train40 = pd.read_json('../input/deepfake/metadata40.json')\ndf_train41 = pd.read_json('../input/deepfake/metadata41.json')\ndf_train42 = pd.read_json('../input/deepfake/metadata42.json')\ndf_train43 = pd.read_json('../input/deepfake/metadata43.json')\ndf_train44 = pd.read_json('../input/deepfake/metadata44.json')\ndf_train45 = pd.read_json('../input/deepfake/metadata45.json')\ndf_train46 = pd.read_json('../input/deepfake/metadata46.json')\ndf_val1 = pd.read_json('../input/deepfake/metadata47.json')\ndf_val2 = pd.read_json('../input/deepfake/metadata48.json')\ndf_val3 = pd.read_json('../input/deepfake/metadata49.json')\ndf_trains = [df_train0 ,df_train1, df_train2, df_train3, df_train4,\n             df_train5, df_train6, df_train7, df_train8, df_train9,df_train10,\n            df_train11, df_train12, df_train13, df_train14, df_train15,df_train16, \n            df_train17, df_train18, df_train19, df_train20, df_train21, df_train22, \n            df_train23, df_train24, df_train25, df_train26, df_train27, df_train28, \n            df_train29, df_train30, df_train31, df_train32, df_train33, df_train34,\n            df_train34, df_train35, df_train36, df_train37, df_train38, df_train39,\n            df_train40, df_train41, df_train42, df_train43, df_train44, df_train45,\n            df_train46]\ndf_vals=[df_val1, df_val2, df_val3]\nnums = list(range(len(df_trains)+1))\nLABELS = ['REAL','FAKE']\nval_nums=[47, 48, 49]","metadata":{"id":"K55cUb_0yTfH","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(num,x):\n    num=str(num)\n    if len(num)==2:\n        path='../input/deepfake/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    else:\n        path='../input/deepfake/DeepFake0'+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    if not os.path.exists(path):\n       raise Exception\n    return path\npaths=[]\ny=[]\nfor df_train,num in tqdm(zip(df_trains,nums),total=len(df_trains)):\n    images = list(df_train.columns.values)\n    for x in images:\n        try:\n            paths.append(get_path(num,x))\n            y.append(LABELS.index(df_train[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass\n\nval_paths=[]\nval_y=[]\nfor df_val,num in tqdm(zip(df_vals,val_nums),total=len(df_vals)):\n    images = list(df_val.columns.values)\n    for x in images:\n        try:\n            val_paths.append(get_path(num,x))\n            val_y.append(LABELS.index(df_val[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass","metadata":{"id":"FSPvZdzbzKd5","outputId":"fe78b0b0-aab7-4dfd-d33c-0af316b60f04","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_img(path):\n    return cv2.cvtColor(cv2.imread(path),cv2.COLOR_BGR2RGB)\n\ndef shuffle(X,y):\n    new_train=[]\n    for m,n in zip(X,y):\n        new_train.append([m,n])\n    random.shuffle(new_train)\n    X,y=[],[]\n    for x in new_train:\n        X.append(x[0])\n        y.append(x[1])\n    return X,y\n\nimport random\ndef get_random_sampling(paths, y, val_paths, val_y):\n  real=[]\n  fake=[]\n  for m,n in zip(paths,y):\n      if n==0:\n          real.append(m)\n      else:\n          fake.append(m)\n  # fake=random.sample(fake,len(real))\n  paths,y=[],[]\n  for x in real:\n      paths.append(x)\n      y.append(0)\n  for x in fake:\n      paths.append(x)\n      y.append(1)\n\n  real=[]\n  fake=[]\n  for m,n in zip(val_paths,val_y):\n      if n==0:\n          real.append(m)\n      else:\n          fake.append(m)\n  # fake=random.sample(fake,len(real))\n  val_paths,val_y=[],[]\n  for x in real:\n      val_paths.append(x)\n      val_y.append(0)\n  for x in fake:\n      val_paths.append(x)\n      val_y.append(1)\n\n  X=[]\n  for img in tqdm(paths):\n      X.append(read_img(img))\n  val_X=[]\n  for img in tqdm(val_paths):\n      val_X.append(read_img(img))\n\n  # Balance with ffhq dataset\n  ffhq = os.listdir('../input/ffhq-face-data-set/thumbnails128x128')\n  X_ = []\n  for file in tqdm(ffhq):\n    im = read_img(f'../input/ffhq-face-data-set/thumbnails128x128/{file}')\n    im = cv2.resize(im, (150,150))\n    X_.append(im)\n  random.shuffle(X_)\n\n  for i in range(64773 - 12130):\n    X.append(X_[i])\n    y.append(0)\n  del X_[0:64773 - 12130]\n  for i in range(6108 - 1258):\n    val_X.append(X_[i])\n    val_y.append(0)\n\n  X, y = shuffle(X,y)\n  val_X, val_y = shuffle(val_X,val_y)\n\n  return X, val_X, y, val_y","metadata":{"id":"QXIIa5A-zfa3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"id":"HmvRDCqmaa_i"}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\nclass ImageDataset(Dataset):\n    def __init__(self, X, y, training=True, transform=None):\n        self.X = X\n        self.y = y\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        img = self.X[idx]\n\n        if self.transform is not None:\n          res = self.transform(image=img)\n          img = res['image']\n        \n        img = np.rollaxis(img, 2, 0)\n        # img = np.array(img).astype(np.float32) / 255.\n\n        labels = self.y[idx]\n        labels = np.array(labels).astype(np.float32)\n        return [img, labels]","metadata":{"id":"7KNA5r-7afVp","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"-xvk_DhD1iUn"}},{"cell_type":"code","source":"!pip install pytorchcv --quiet\nfrom pytorchcv.model_provider import get_model\nmodel = get_model(\"xception\", pretrained=True)\n# model = get_model(\"resnet18\", pretrained=True)\nmodel = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer","metadata":{"id":"dcIJUiPx1DgP","outputId":"d8024ba7-ca65-4aed-c2c8-042a5274fdf4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n# model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))","metadata":{"id":"jGr9EuSX1ZYI","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Head(torch.nn.Module):\n  def __init__(self, in_f, out_f):\n    super(Head, self).__init__()\n    \n    self.f = nn.Flatten()\n    self.l = nn.Linear(in_f, 512)\n    self.d = nn.Dropout(0.75)\n    self.o = nn.Linear(512, out_f)\n    self.b1 = nn.BatchNorm1d(in_f)\n    self.b2 = nn.BatchNorm1d(512)\n    self.r = nn.ReLU()\n\n  def forward(self, x):\n    x = self.f(x)\n    x = self.b1(x)\n    x = self.d(x)\n\n    x = self.l(x)\n    x = self.r(x)\n    x = self.b2(x)\n    x = self.d(x)\n\n    out = self.o(x)\n    return out","metadata":{"id":"EEVBeVoW1cJX","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FCN(torch.nn.Module):\n  def __init__(self, base, in_f):\n    super(FCN, self).__init__()\n    self.base = base\n    self.h1 = Head(in_f, 1)\n  \n  def forward(self, x):\n    x = self.base(x)\n    return self.h1(x)\n\nmodel = FCN(model, 2048)","metadata":{"id":"FRyOSBXy1wim","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install torchtoolbox --quiet\n# from torchtoolbox.tools import summary\n\n# model.cuda()\n# summary(model, torch.rand((1, 3, 150, 150)).cuda())","metadata":{"id":"OPA6IyUJ1yxU","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Functions","metadata":{"id":"pZv7D2KQ2YBk"}},{"cell_type":"code","source":"def criterion1(pred1, targets):\n  l1 = F.binary_cross_entropy(F.sigmoid(pred1), targets)\n  return l1\n\ndef train_model(epoch, optimizer, scheduler=None, history=None):\n    model.train()\n    total_loss = 0\n    \n    t = tqdm(train_loader)\n    for i, (img_batch, y_batch) in enumerate(t):\n        img_batch = img_batch.cuda().float()\n        y_batch = y_batch.cuda().float()\n\n        optimizer.zero_grad()\n\n        out = model(img_batch)\n        loss = criterion1(out, y_batch)\n\n        total_loss += loss\n        t.set_description(f'Epoch {epoch+1}/{n_epochs}, LR: %6f, Loss: %.4f'%(optimizer.state_dict()['param_groups'][0]['lr'],total_loss/(i+1)))\n\n        if history is not None:\n          history.loc[epoch + i / len(X), 'train_loss'] = loss.data.cpu().numpy()\n          history.loc[epoch + i / len(X), 'lr'] = optimizer.state_dict()['param_groups'][0]['lr']\n\n        loss.backward()\n        optimizer.step()\n        if scheduler is not None:\n          scheduler.step()\n\ndef evaluate_model(epoch, scheduler=None, history=None):\n    model.eval()\n    loss = 0\n    pred = []\n    real = []\n    with torch.no_grad():\n        for img_batch, y_batch in val_loader:\n            img_batch = img_batch.cuda().float()\n            y_batch = y_batch.cuda().float()\n\n            o1 = model(img_batch)\n            l1 = criterion1(o1, y_batch)\n            loss += l1\n            \n            for j in o1:\n              pred.append(F.sigmoid(j))\n            for i in y_batch:\n              real.append(i.data.cpu())\n    \n    pred = [p.data.cpu().numpy() for p in pred]\n    pred2 = pred\n    pred = [np.round(p) for p in pred]\n    pred = np.array(pred)\n    acc = sklearn.metrics.recall_score(real, pred, average='macro')\n\n    real = [r.item() for r in real]\n    pred2 = np.array(pred2).clip(0.1, 0.9)\n    kaggle = sklearn.metrics.log_loss(real, pred2)\n\n    loss /= len(val_loader)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()\n    \n    if scheduler is not None:\n      scheduler.step(loss)\n\n    print(f'Dev loss: %.4f, Acc: %.6f, Kaggle: %.6f'%(loss,acc,kaggle))\n    \n    return loss","metadata":{"id":"Jc3QTjqj2XkJ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataloaders","metadata":{"id":"XAhEFXSVKsyr"}},{"cell_type":"code","source":"X, val_X, y, val_y = get_random_sampling(paths, y, val_paths, val_y)\n\nprint('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","metadata":{"id":"n25nfarz8Gfi","outputId":"9961d38f-8461-4535-ef15-2a3d06ae64cd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations\nfrom albumentations.augmentations.transforms import ShiftScaleRotate, HorizontalFlip, Normalize, RandomBrightnessContrast, MotionBlur, Blur, GaussNoise, JpegCompression\ntrain_transform = albumentations.Compose([\n                                          ShiftScaleRotate(p=0.3, scale_limit=0.25, border_mode=1, rotate_limit=25),\n                                          HorizontalFlip(p=0.2),\n                                          RandomBrightnessContrast(p=0.3, brightness_limit=0.25, contrast_limit=0.5),\n                                          MotionBlur(p=.2),\n                                          GaussNoise(p=.2),\n                                          JpegCompression(p=.2, quality_lower=50),\n                                          Normalize()\n])\nval_transform = albumentations.Compose([\n                                          Normalize()\n])\n\ntrain_dataset = ImageDataset(X, y, transform=train_transform)\nval_dataset = ImageDataset(val_X, val_y, transform=val_transform)","metadata":{"id":"kfCLL0pt9Vh-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nrow, ncol = 5, 6\nfig, axes = plt.subplots(nrow, ncol, figsize=(20, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_dataset[i]\n    image = np.rollaxis(image, 0, 3)\n    image = image*std + mean\n    image = np.clip(image, 0., 1.)\n    ax.imshow(image)\n    ax.set_title(f'label: {label}')","metadata":{"id":"P0Z_BWFJ-E5A","outputId":"db70092d-f2b0-4e17-fdfe-ffbb32bd9630","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"id":"zbaUUqLIKwst"}},{"cell_type":"code","source":"import gc\n\nhistory = pd.DataFrame()\nhistory2 = pd.DataFrame()\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nbest = 1e10\nn_epochs = 20\nbatch_size = 128\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\nval_loader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n\nmodel = model.cuda()\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=0.001)\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, mode='min', factor=0.7, verbose=True, min_lr=1e-5)\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    train_model(epoch, optimizer, scheduler=None, history=history)\n    \n    loss = evaluate_model(epoch, scheduler=scheduler, history=history2)\n    \n    if loss < best:\n      best = loss\n      print(f'Saving best model...')\n      torch.save(model.state_dict(), f'model.pth')","metadata":{"id":"RJmdT2spBEU1","outputId":"bd51ed41-f0b2-49ec-8fa1-7c890243b78d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2.plot()","metadata":{"id":"vtSjo9DYtoEL","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## View this kernal for inference and submission: https://www.kaggle.com/greatgamedota/xception-binary-classifier-inference","metadata":{}}]}