{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Transformer Based Model\n\nI seen several notebooks working with LSTM. This is my inference notebook using Transformer.\n\nIt uses the TRANSFORMER model without position embedding.\nUnfortuantely I came up with this just a couple days ago and wasn't able to properly train it.\nNow I ran out of time and credits on AWS. :D\n\nI'm also using faces of 128x128 because it needs so much memory for training.\n\nThis was based on the work from Limerobot\nhttps://www.kaggle.com/c/data-science-bowl-2019/discussion/127891\n\nIt was a lot of fun to work on this competition. Thanks to Kaggle, Google, AWS and many others."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/deppfake-montez-2020","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\n# Install facenet-pytorch\n!pip install /kaggle/input/deppfake-montez-2020/facenet_pytorch-2.2.9-py3-none-any.whl\n!pip install /kaggle/input/deppfake-montez-2020/timm-0.1.18-py3-none-any.whl\n!pip install /kaggle/input/deppfake-montez-2020/pytorch_transformers-1.2.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport torch\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import DataLoader\nfrom pytorch_transformers.modeling_bert import BertConfig, BertEncoder\nimport torch.nn as nn\nfrom facenet_pytorch import MTCNN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Thanks to limerobot: https://www.kaggle.com/c/data-science-bowl-2019/discussion/127891\n\nclass TransfomerModel(nn.Module):\n    def __init__(self, cfg, cnn_model):\n        super(TransfomerModel, self).__init__()\n        self.cfg = cfg\n        self.n_cnn_features = cfg.n_cnn_features\n        self.cnn_model = nn.Sequential(\n            cnn_model,\n            torch.nn.AdaptiveAvgPool2d(1),\n            nn.Dropout(cfg.dropout)\n        )\n\n        self.config = BertConfig( \n            3, # not used\n            hidden_size=cfg.hidden_size,\n            num_hidden_layers=cfg.nlayers,\n            num_attention_heads=cfg.nheads,\n            intermediate_size=cfg.hidden_size,\n            hidden_dropout_prob=cfg.dropout,\n            attention_probs_dropout_prob=cfg.dropout,\n        )\n        self.encoder = BertEncoder(self.config) \n        \n        def get_reg():\n            return nn.Sequential(\n            nn.Linear(cfg.hidden_size, cfg.hidden_size),\n            nn.LayerNorm(cfg.hidden_size),\n            nn.Dropout(cfg.dropout),\n            nn.ReLU(),\n            nn.Linear(cfg.hidden_size, cfg.hidden_size),\n            nn.LayerNorm(cfg.hidden_size),\n            nn.Dropout(cfg.dropout),\n            nn.ReLU(),\n            nn.Linear(cfg.hidden_size, 1), #, cfg.target_size\n            nn.Sigmoid()\n        )\n        self.reg_layer = get_reg()\n       \n    def forward(self, x, mask):\n        batch_size, n_frames, n_channels, height, width = x.size() # e.g. 16 x 10 x 3 x 224 x 224\n        assert n_channels == 3, f\"Expecting 3 channels but got {n_channels}\"\n        x = x.reshape(batch_size*n_frames, n_channels, height, width) # (16*10) x 3 x 224 x 224 x = self.cnn_model(x) # (32*64) x 2048\n        x = self.cnn_model(x)\n        x = torch.squeeze(x)\n        #print(x.shape)\n        x = x.view(batch_size, n_frames, self.n_cnn_features)\n        \n        \n        extended_attention_mask = mask.unsqueeze(1).unsqueeze(2)\n        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility\n        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0\n        head_mask = [None] * self.config.num_hidden_layers\n        \n        encoded_layers = self.encoder(x, extended_attention_mask, head_mask=head_mask)\n        sequence_output = encoded_layers[-1]\n        sequence_output = sequence_output[:, -1]\n \n        pred_y = self.reg_layer(sequence_output)\n        return pred_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    learning_rate=1.0e-4\n    batch_size=4\n    seq_len=15\n    frame_size = 128\n    print_freq=100\n    test_freq=1\n    start_epoch=0\n    num_train_epochs=10\n    warmup_steps=30\n    max_grad_norm=1000\n    gradient_accumulation_steps=1\n    weight_decay=0.01\n    dropout=0.2\n    emb_size=100\n    hidden_size=1536\n    nlayers=2\n    nheads=8\n    device=device\n    seed=7\n    n_cnn_features=1536","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load('/kaggle/input/deppfake-montez-2020/transformer11_train30_2.pkl', map_location=torch.device(device))\nmodel.eval()\nprint('Model Loaded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n\ntest_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtcnn = MTCNN(keep_all=True, margin=14, device=device) # margin actually has no effect","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get faces from video frames list\ndef get_faces(vframes):\n    cropped_faces = []\n    \n    with torch.no_grad():\n        faces, probs = mtcnn.detect(vframes)\n\n        count = 0\n        for j, faces_per_frame in enumerate(faces):\n            if faces_per_frame is None:\n                continue\n\n            for i, face in enumerate(faces_per_frame):\n                x1, x2, y1, y2 = round(face[0]), round(face[2]), round(face[1]), round(face[3])\n                width = abs(x2 - x1)\n                height = abs(y2 - y1)\n                if width < 50 or height < 50 or probs[j][i] < 0.90:\n                    continue\n\n                face_box = np.zeros(4, dtype=int)\n                # width\n                face_box[0] = max(x1, 0)\n                face_box[2] = min(face_box[0] + width, vframes[j].shape[1])\n\n                # height\n                face_box[1] = max(y1, 0)\n                face_box[3] = min(face_box[1] + height, vframes[j].shape[0])\n\n                crop_img = vframes[j][face_box[1]:face_box[3], face_box[0]:face_box[2]]\n                cropped_faces.append(crop_img)\n\n    return cropped_faces","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get faces from filename\ndef capture_face(name, n_frames1, n_frames2, min_num_faces):\n    faces = []\n    filename = os.path.join(test_dir, name)\n    #print(\"Analyzing file: \" + filename)\n\n    try:\n        vframes1 = []\n        vframes2 = []\n        v_cap = cv2.VideoCapture(filename)\n        v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))\n\n        # Pick evenly spaced frames to sample\n        # use 2 sample intervales, preferably prime numbers e.g. 17 and 11\n        sample1 = np.linspace(0, v_len - 1, n_frames1).round().astype(int)\n        sample2 = np.linspace(0, v_len - 1, n_frames2).round().astype(int)\n        for j in range(v_len):\n            ret = v_cap.grab()\n            if j in sample1:\n                ret, vframe = v_cap.retrieve()\n                vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n                vframes1.append(vframe)\n            else:\n                if j in sample2:\n                    ret, vframe = v_cap.retrieve()\n                    vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)\n                    vframes2.append(vframe)\n        v_cap.release()\n    \n        faces = get_faces(vframes1)\n        del vframes1\n        \n        if (len(faces) < min_num_faces):\n            #print('Number of faces smaller than min: {0} < {1}'.format(len(faces), min_num_faces))\n            faces2 = get_faces(vframes2)\n            del vframes2\n            \n            faces += faces2\n            #print('New number of faces: {0}'.format(len(faces)))\n        \n    except:\n        v_cap.release()\n\n    return faces        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom torch.utils import data\n\nclass Dataset_Transform_Test(data.Dataset):\n    def __init__(self, files, seq_len, size):\n        self.files = files\n        self.size = size\n        self.seq_len = seq_len\n\n    def get_faces(self, file):\n        X = torch.tensor([])\n        list_tensors = []\n        try:\n            faces = capture_face(file, 17, 11, seq_len)\n            for face in faces:\n                img = pil2tensor(face.astype(np.float32), np.float32) / 255\n                img = vision.Image(img)\n                img.resize(self.size)\n                list_tensors.append(img.data)\n                if len(list_tensors) == self.seq_len:\n                    break\n            if len(list_tensors) > 0:\n                X = torch.stack(list_tensors, dim=0)\n        except:\n            print('failed to get faces for {0}'.format(file))\n        return X\n    \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, index):\n        file = self.files[index]\n\n        X = torch.FloatTensor(self.seq_len, 3, self.size, self.size).zero_()\n        X_temp = self.get_faces(file)\n        seq_len = min(self.seq_len, len(X_temp))\n        mask = torch.ByteTensor(self.seq_len).zero_()\n        if (seq_len > 0):\n            X[:seq_len] = X_temp[:seq_len]\n            mask[:seq_len] = 1\n        else:\n            print('mask is empty')\n        \n        return X.to(device), mask.to(device) #, y.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seq_len = 15\nsize = 128\nbatch_size=4\ntest_db = Dataset_Transform_Test(test_videos, seq_len, size)\n#test_loader = DataLoader(test_db, batch_size=batch_size, shuffle=False, pin_memory=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clip_value(value):\n    #print(value)\n    return max(min(value, 0.9), 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nxs = []\nmasks = []\nsubmission = []\nbatch_file = []\nbatch_num = 0\ncount = 0\nfor i, (x, mask) in enumerate(test_db):\n\n    if mask[5] == 0:\n        submission.append([test_videos[count], 0.5])\n        count += 1\n        continue\n    batch_file.append(test_videos[i])\n    xs.append(x)\n    masks.append(mask)\n    batch_num += 1\n    \n    # continue until batch or end\n    if batch_num != batch_size and i < len(test_db):\n        continue\n    \n    x = torch.stack(xs, dim=0)\n    mask = torch.stack(masks, dim=0)\n    x, mask = x.to(device), mask.to(device)        \n\n    with torch.no_grad():        \n        pred = model(x, mask)\n        for j, proc_file in enumerate(batch_file):\n            submission.append([test_videos[count], pred[j][0].cpu().item()]) #clip_value(pred[j][0].cpu().item())]) #\n            count += 1\n    batch_num = 0\n    batch_file = []\n    xs = []\n    masks = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(submission, columns=['filename', 'label'])\nsubmission.sort_values('filename').to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission.label, 20)\nplt.show()\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}