{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Binary classifier with Xception using FFHQ dataset for upsampling\n  \n  \nThis notebook utilize existing Kaggle dataset that consists of a single extract frame from each video for the entire training dataset, in addtion FFHQ dataset is used to balance to uneven distribution between real and fake, consider reduce the number of epoch during training as it takes little over 8 hours with GPU. "},{"metadata":{"id":"YjwlB710mIH_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nfrom tqdm import tqdm,trange\nfrom sklearn.model_selection import train_test_split\nimport sklearn.metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"id":"AaU3SeKMz_qm","colab_type":"text"},"cell_type":"markdown","source":"# Setup Data"},{"metadata":{"id":"K55cUb_0yTfH","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"df_train0 = pd.read_json('../input/deepfake/metadata0.json')\ndf_train1 = pd.read_json('../input/deepfake/metadata1.json')\ndf_train2 = pd.read_json('../input/deepfake/metadata2.json')\ndf_train3 = pd.read_json('../input/deepfake/metadata3.json')\ndf_train4 = pd.read_json('../input/deepfake/metadata4.json')\ndf_train5 = pd.read_json('../input/deepfake/metadata5.json')\ndf_train6 = pd.read_json('../input/deepfake/metadata6.json')\ndf_train7 = pd.read_json('../input/deepfake/metadata7.json')\ndf_train8 = pd.read_json('../input/deepfake/metadata8.json')\ndf_train9 = pd.read_json('../input/deepfake/metadata9.json')\ndf_train10 = pd.read_json('../input/deepfake/metadata10.json')\ndf_train11 = pd.read_json('../input/deepfake/metadata11.json')\ndf_train12 = pd.read_json('../input/deepfake/metadata12.json')\ndf_train13 = pd.read_json('../input/deepfake/metadata13.json')\ndf_train14 = pd.read_json('../input/deepfake/metadata14.json')\ndf_train15 = pd.read_json('../input/deepfake/metadata15.json')\ndf_train16 = pd.read_json('../input/deepfake/metadata16.json')\ndf_train17 = pd.read_json('../input/deepfake/metadata17.json')\ndf_train18 = pd.read_json('../input/deepfake/metadata18.json')\ndf_train19 = pd.read_json('../input/deepfake/metadata19.json')\ndf_train20 = pd.read_json('../input/deepfake/metadata20.json')\ndf_train21 = pd.read_json('../input/deepfake/metadata21.json')\ndf_train22 = pd.read_json('../input/deepfake/metadata22.json')\ndf_train23 = pd.read_json('../input/deepfake/metadata23.json')\ndf_train24 = pd.read_json('../input/deepfake/metadata24.json')\ndf_train25 = pd.read_json('../input/deepfake/metadata25.json')\ndf_train26 = pd.read_json('../input/deepfake/metadata26.json')\ndf_train27 = pd.read_json('../input/deepfake/metadata27.json')\ndf_train28 = pd.read_json('../input/deepfake/metadata28.json')\ndf_train29 = pd.read_json('../input/deepfake/metadata29.json')\ndf_train30 = pd.read_json('../input/deepfake/metadata30.json')\ndf_train31 = pd.read_json('../input/deepfake/metadata31.json')\ndf_train32 = pd.read_json('../input/deepfake/metadata32.json')\ndf_train33 = pd.read_json('../input/deepfake/metadata33.json')\ndf_train34 = pd.read_json('../input/deepfake/metadata34.json')\ndf_train35 = pd.read_json('../input/deepfake/metadata35.json')\ndf_train36 = pd.read_json('../input/deepfake/metadata36.json')\ndf_train37 = pd.read_json('../input/deepfake/metadata37.json')\ndf_train38 = pd.read_json('../input/deepfake/metadata38.json')\ndf_train39 = pd.read_json('../input/deepfake/metadata39.json')\ndf_train40 = pd.read_json('../input/deepfake/metadata40.json')\ndf_train41 = pd.read_json('../input/deepfake/metadata41.json')\ndf_train42 = pd.read_json('../input/deepfake/metadata42.json')\ndf_train43 = pd.read_json('../input/deepfake/metadata43.json')\ndf_train44 = pd.read_json('../input/deepfake/metadata44.json')\ndf_train45 = pd.read_json('../input/deepfake/metadata45.json')\ndf_train46 = pd.read_json('../input/deepfake/metadata46.json')\ndf_val1 = pd.read_json('../input/deepfake/metadata47.json')\ndf_val2 = pd.read_json('../input/deepfake/metadata48.json')\ndf_val3 = pd.read_json('../input/deepfake/metadata49.json')\ndf_trains = [df_train0 ,df_train1, df_train2, df_train3, df_train4,\n             df_train5, df_train6, df_train7, df_train8, df_train9,df_train10,\n            df_train11, df_train12, df_train13, df_train14, df_train15,df_train16, \n            df_train17, df_train18, df_train19, df_train20, df_train21, df_train22, \n            df_train23, df_train24, df_train25, df_train26, df_train27, df_train28, \n            df_train29, df_train30, df_train31, df_train32, df_train33, df_train34,\n            df_train34, df_train35, df_train36, df_train37, df_train38, df_train39,\n            df_train40, df_train41, df_train42, df_train43, df_train44, df_train45,\n            df_train46]\ndf_vals=[df_val1, df_val2, df_val3]\nnums = list(range(len(df_trains)+1))\nLABELS = ['REAL','FAKE']\nval_nums=[47, 48, 49]","execution_count":null,"outputs":[]},{"metadata":{"id":"FSPvZdzbzKd5","colab_type":"code","outputId":"fe78b0b0-aab7-4dfd-d33c-0af316b60f04","colab":{"base_uri":"https://localhost:8080/"},"trusted":true},"cell_type":"code","source":"def get_path(num,x):\n    num=str(num)\n    if len(num)==2:\n        path='../input/deepfake/DeepFake'+num+'/DeepFake'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    else:\n        path='../input/deepfake/DeepFake0'+num+'/DeepFake0'+num+'/' + x.replace('.mp4', '') + '.jpg'\n    if not os.path.exists(path):\n       raise Exception\n    return path\npaths=[]\ny=[]\nfor df_train,num in tqdm(zip(df_trains,nums),total=len(df_trains)):\n    images = list(df_train.columns.values)\n    for x in images:\n        try:\n            paths.append(get_path(num,x))\n            y.append(LABELS.index(df_train[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass\n\nval_paths=[]\nval_y=[]\nfor df_val,num in tqdm(zip(df_vals,val_nums),total=len(df_vals)):\n    images = list(df_val.columns.values)\n    for x in images:\n        try:\n            val_paths.append(get_path(num,x))\n            val_y.append(LABELS.index(df_val[x]['label']))\n        except Exception as err:\n            #print(err)\n            pass","execution_count":null,"outputs":[]},{"metadata":{"id":"QXIIa5A-zfa3","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def read_img(path):\n    return cv2.cvtColor(cv2.imread(path),cv2.COLOR_BGR2RGB)\n\ndef shuffle(X,y):\n    new_train=[]\n    for m,n in zip(X,y):\n        new_train.append([m,n])\n    random.shuffle(new_train)\n    X,y=[],[]\n    for x in new_train:\n        X.append(x[0])\n        y.append(x[1])\n    return X,y\n\nimport random\ndef get_random_sampling(paths, y, val_paths, val_y):\n  real=[]\n  fake=[]\n  for m,n in zip(paths,y):\n      if n==0:\n          real.append(m)\n      else:\n          fake.append(m)\n  # fake=random.sample(fake,len(real))\n  paths,y=[],[]\n  for x in real:\n      paths.append(x)\n      y.append(0)\n  for x in fake:\n      paths.append(x)\n      y.append(1)\n\n  real=[]\n  fake=[]\n  for m,n in zip(val_paths,val_y):\n      if n==0:\n          real.append(m)\n      else:\n          fake.append(m)\n  # fake=random.sample(fake,len(real))\n  val_paths,val_y=[],[]\n  for x in real:\n      val_paths.append(x)\n      val_y.append(0)\n  for x in fake:\n      val_paths.append(x)\n      val_y.append(1)\n\n  X=[]\n  for img in tqdm(paths):\n      X.append(read_img(img))\n  val_X=[]\n  for img in tqdm(val_paths):\n      val_X.append(read_img(img))\n\n  # Balance with ffhq dataset\n  ffhq = os.listdir('../input/ffhq-face-data-set/thumbnails128x128')\n  X_ = []\n  for file in tqdm(ffhq):\n    im = read_img(f'../input/ffhq-face-data-set/thumbnails128x128/{file}')\n    im = cv2.resize(im, (150,150))\n    X_.append(im)\n  random.shuffle(X_)\n\n  for i in range(64773 - 12130):\n    X.append(X_[i])\n    y.append(0)\n  del X_[0:64773 - 12130]\n  for i in range(6108 - 1258):\n    val_X.append(X_[i])\n    val_y.append(0)\n\n  X, y = shuffle(X,y)\n  val_X, val_y = shuffle(val_X,val_y)\n\n  return X, val_X, y, val_y","execution_count":null,"outputs":[]},{"metadata":{"id":"HmvRDCqmaa_i","colab_type":"text"},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"id":"7KNA5r-7afVp","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\nclass ImageDataset(Dataset):\n    def __init__(self, X, y, training=True, transform=None):\n        self.X = X\n        self.y = y\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        img = self.X[idx]\n\n        if self.transform is not None:\n          res = self.transform(image=img)\n          img = res['image']\n        \n        img = np.rollaxis(img, 2, 0)\n        # img = np.array(img).astype(np.float32) / 255.\n\n        labels = self.y[idx]\n        labels = np.array(labels).astype(np.float32)\n        return [img, labels]","execution_count":null,"outputs":[]},{"metadata":{"id":"-xvk_DhD1iUn","colab_type":"text"},"cell_type":"markdown","source":"# Model"},{"metadata":{"id":"dcIJUiPx1DgP","colab_type":"code","outputId":"d8024ba7-ca65-4aed-c2c8-042a5274fdf4","colab":{"base_uri":"https://localhost:8080/","height":52},"trusted":true},"cell_type":"code","source":"#Trying out different pre-trained models\n\n!pip install pytorchcv --quiet\nfrom pytorchcv.model_provider import get_model\n\n#model = get_model(\"xception\", pretrained=True)\nmodel = get_model(\"resnet101\", pretrained=True)\n#model = get_model(\"InceptionV4\", pretrained=True)\n\n\nmodel = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer","execution_count":null,"outputs":[]},{"metadata":{"id":"jGr9EuSX1ZYI","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\nmodel[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))","execution_count":null,"outputs":[]},{"metadata":{"id":"EEVBeVoW1cJX","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class Head(torch.nn.Module):\n  def __init__(self, in_f, out_f):\n    super(Head, self).__init__()\n    \n    self.f = nn.Flatten()\n    self.l = nn.Linear(in_f, 512)\n    self.d = nn.Dropout(0.50)\n    self.o = nn.Linear(512, out_f)\n    self.b1 = nn.BatchNorm1d(in_f)\n    self.b2 = nn.BatchNorm1d(512)\n    self.r = nn.ReLU()\n\n  def forward(self, x):\n    x = self.f(x)\n    x = self.b1(x)\n    x = self.d(x)\n\n    x = self.l(x)\n    x = self.r(x)\n    x = self.b2(x)\n    x = self.d(x)\n\n    out = self.o(x)\n    return out","execution_count":null,"outputs":[]},{"metadata":{"id":"FRyOSBXy1wim","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class FCN(torch.nn.Module):\n  def __init__(self, base, in_f):\n    super(FCN, self).__init__()\n    self.base = base\n    self.h1 = Head(in_f, 1)\n  \n  def forward(self, x):\n    x = self.base(x)\n    return self.h1(x)\n\nmodel = FCN(model, 2048)\n\n#model = FCN(model, 1536)","execution_count":null,"outputs":[]},{"metadata":{"id":"OPA6IyUJ1yxU","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# !pip install torchtoolbox --quiet\n# from torchtoolbox.tools import summary\n\n# model.cuda()\n# summary(model, torch.rand((1, 3, 150, 150)).cuda())","execution_count":null,"outputs":[]},{"metadata":{"id":"pZv7D2KQ2YBk","colab_type":"text"},"cell_type":"markdown","source":"# Train Functions"},{"metadata":{"id":"Jc3QTjqj2XkJ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def criterion1(pred1, targets):\n  l1 = F.binary_cross_entropy(F.sigmoid(pred1), targets)\n  return l1\n\ndef train_model(epoch, optimizer, scheduler=None, history=None):\n    model.train()\n    total_loss = 0\n    \n    t = tqdm(train_loader)\n    for i, (img_batch, y_batch) in enumerate(t):\n        img_batch = img_batch.cuda().float()\n        y_batch = y_batch.cuda().float()\n\n        optimizer.zero_grad()\n\n        out = model(img_batch)\n        loss = criterion1(out, y_batch)\n\n        total_loss += loss\n        t.set_description(f'Epoch {epoch+1}/{n_epochs}, LR: %6f, Loss: %.4f'%(optimizer.state_dict()['param_groups'][0]['lr'],total_loss/(i+1)))\n\n        if history is not None:\n          history.loc[epoch + i / len(X), 'train_loss'] = loss.data.cpu().numpy()\n          history.loc[epoch + i / len(X), 'lr'] = optimizer.state_dict()['param_groups'][0]['lr']\n\n        loss.backward()\n        optimizer.step()\n        if scheduler is not None:\n          scheduler.step()\n\ndef evaluate_model(epoch, scheduler=None, history=None):\n    model.eval()\n    loss = 0\n    pred = []\n    real = []\n    with torch.no_grad():\n        for img_batch, y_batch in val_loader:\n            img_batch = img_batch.cuda().float()\n            y_batch = y_batch.cuda().float()\n\n            o1 = model(img_batch)\n            l1 = criterion1(o1, y_batch)\n            loss += l1\n            \n            for j in o1:\n              pred.append(F.sigmoid(j))\n            for i in y_batch:\n              real.append(i.data.cpu())\n    \n    pred = [p.data.cpu().numpy() for p in pred]\n    pred2 = pred\n    pred = [np.round(p) for p in pred]\n    pred = np.array(pred)\n    acc = sklearn.metrics.recall_score(real, pred, average='macro')\n\n    real = [r.item() for r in real]\n    pred2 = np.array(pred2).clip(0.1, 0.9)\n    kaggle = sklearn.metrics.log_loss(real, pred2)\n\n    loss /= len(val_loader)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()\n    \n    if scheduler is not None:\n      scheduler.step(loss)\n\n    print(f'Dev loss: %.4f, Acc: %.6f, Kaggle: %.6f'%(loss,acc,kaggle))\n    \n    return loss","execution_count":null,"outputs":[]},{"metadata":{"id":"XAhEFXSVKsyr","colab_type":"text"},"cell_type":"markdown","source":"# Dataloaders"},{"metadata":{"id":"n25nfarz8Gfi","colab_type":"code","outputId":"9961d38f-8461-4535-ef15-2a3d06ae64cd","colab":{"base_uri":"https://localhost:8080/","height":141},"trusted":true},"cell_type":"code","source":"X, val_X, y, val_y = get_random_sampling(paths, y, val_paths, val_y)\n\nprint('There are '+str(y.count(1))+' fake train samples')\nprint('There are '+str(y.count(0))+' real train samples')\nprint('There are '+str(val_y.count(1))+' fake val samples')\nprint('There are '+str(val_y.count(0))+' real val samples')","execution_count":null,"outputs":[]},{"metadata":{"id":"kfCLL0pt9Vh-","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import albumentations\nfrom albumentations.augmentations.transforms import ShiftScaleRotate, HorizontalFlip, Normalize, RandomBrightnessContrast, MotionBlur, Blur, GaussNoise, JpegCompression\ntrain_transform = albumentations.Compose([\n                                          ShiftScaleRotate(p=0.3, scale_limit=0.25, border_mode=1, rotate_limit=25),\n                                          HorizontalFlip(p=0.2),\n                                          RandomBrightnessContrast(p=0.3, brightness_limit=0.25, contrast_limit=0.5),\n                                          MotionBlur(p=.2),\n                                          GaussNoise(p=.2),\n                                          JpegCompression(p=.2, quality_lower=50),\n                                          Normalize()\n])\nval_transform = albumentations.Compose([\n                                          Normalize()\n])\n\ntrain_dataset = ImageDataset(X, y, transform=train_transform)\nval_dataset = ImageDataset(val_X, val_y, transform=val_transform)","execution_count":null,"outputs":[]},{"metadata":{"id":"P0Z_BWFJ-E5A","colab_type":"code","outputId":"db70092d-f2b0-4e17-fdfe-ffbb32bd9630","colab":{"base_uri":"https://localhost:8080/","height":499},"trusted":true},"cell_type":"code","source":"nrow, ncol = 5, 6\nfig, axes = plt.subplots(nrow, ncol, figsize=(20, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_dataset[i]\n    image = np.rollaxis(image, 0, 3)\n    image = image*std + mean\n    image = np.clip(image, 0., 1.)\n    ax.imshow(image)\n    ax.set_title(f'label: {label}')","execution_count":null,"outputs":[]},{"metadata":{"id":"RJmdT2spBEU1","colab_type":"code","outputId":"bd51ed41-f0b2-49ec-8fa1-7c890243b78d","colab":{"base_uri":"https://localhost:8080/","height":405},"trusted":true},"cell_type":"code","source":"import gc\n\nhistory = pd.DataFrame()\nhistory2 = pd.DataFrame()\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nbest = 1e10\nn_epochs = 20\nbatch_size = 64\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\nval_loader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n\nmodel = model.cuda()\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=0.001)\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, mode='min', factor=0.7, verbose=True, min_lr=1e-5)\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    train_model(epoch, optimizer, scheduler=None, history=history)\n    \n    loss = evaluate_model(epoch, scheduler=scheduler, history=history2)\n    \n    if loss < best:\n      best = loss\n      print(f'Saving best model...')\n      torch.save(model.state_dict(), f'model.pth')","execution_count":null,"outputs":[]},{"metadata":{"id":"zbaUUqLIKwst","colab_type":"text"},"cell_type":"markdown","source":"# Train"},{"metadata":{"id":"vtSjo9DYtoEL","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"history.plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## View this kernal for inference and submission: https://www.kaggle.com/greatgamedota/xception-binary-classifier-inference"}],"metadata":{"colab":{"name":"Deepfake_Detection.ipynb","provenance":[],"collapsed_sections":["AaU3SeKMz_qm","XsefoEdR1gHt"],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":4}