{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport sys\nsys.path.insert(0, \"/kaggle/input/deepfake\")\nsys.path.insert(0, \"/kaggle/input/testpy\")\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport math\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.utils.data as data\nimport torchvision\nfrom torch.autograd import Variable\nfrom functions import *\nimport pandas as pd\n\nprint(torch.cuda.is_available())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = [\"../input/deepfake-detection-challenge/test_videos\"]  # define UCF-101 RGB data path\n\n# encoder model and rnn classification model\ncnn_path = \"/kaggle/input/resnet50/cnn_encoder_epoch_211.pth\"\nrnn_path = \"/kaggle/input/resnet50/rnn_decoder_epoch_211.pth\"\n# EncoderCNN architecture\nCNN_fc_hidden1, CNN_fc_hidden2 = 1024, 768\nCNN_embed_dim = 512   # latent dim extracted by 2D CNN\nres_size = 224        # ResNet image size\ndropout_p = 0.0       # dropout probability\n\n# DecoderRNN architecture\nRNN_hidden_layers = 3\nRNN_hidden_nodes = 512\nRNN_FC_dim = 256\n\n# training parameters\nk = 2             # number of target category\nepochs = 201        # training epochs\nbatch_size = 32 \nlearning_rate = 1e-3\nlog_interval = 1   # interval for displaying training info\n\n# Select which frame to begin & end in videos\n# begin_frame, end_frame, skip_frame = 1, 29, 1\nframe_len = 39 # the number of frames used to input network for each video.\n\n\n##############################################################################################\n# dirs_0      -- paths of videos\n# v_id        -- id of test video\n# net         -- retinaface net work\n# transform   -- the function of image preprocessing\n# device      -- GPU \n# frames      -- the index of obtained frames\n# length_frames -- number of frames\n# batch_size  -- batch_size == 1\n# is_multi    -- True (multiple faces in one video)\n\ndef get_data(dirs_0, v_id, net, transform, device, frames, length_frames, batch_size, is_multi):\n    results = []\n    res_y = []\n    net.eval()\n    net = net.to(device)\n    for id in range(0, batch_size):\n        # Select sample\n\n        dirs = dirs_0\n        folder = dirs[v_id] # get the video path\n\n        videoCapture = cv2.VideoCapture(folder)  # read video\n\n        fNUMS = videoCapture.get(cv2.CAP_PROP_FRAME_COUNT) # frame number\n\n        skip_f = -1  # obtain frames from the first frame\n\n        success, frame = videoCapture.read()\n        n_frame = 0\n\n        res_list = []\n        index = 0\n        is_null = True\n        nums_temp = 0\n        sizes = frame.shape[0:2]\n        ranges = min(sizes)/30.0  # used to produce multiple faces in each video\n        res = []\n        stackw = []\n        stackh = []\n        indexs = []\n       \n        while success :\n\n            if n_frame > skip_f:\n\n                img_raw = frame\n\n                img = np.float32(img_raw)\n\n                # testing scale\n                target_size = 1600\n                max_size = 2150\n                im_shape = img.shape\n                im_size_min = np.min(im_shape[0:2])\n                im_size_max = np.max(im_shape[0:2])\n                resize = float(target_size) / float(im_size_min)\n                # prevent bigger axis from being more than max_size:\n                if np.round(resize * im_size_max) > max_size:\n                    resize = float(max_size) / float(im_size_max)\n                resize = 1\n\n                if resize != 1:\n                    img = cv2.resize(img, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR)\n                im_height, im_width, _ = img.shape\n                scale = torch.Tensor([img.shape[1], img.shape[0], img.shape[1], img.shape[0]])\n                img -= (104, 117, 123)\n                img = img.transpose(2, 0, 1)\n                img = torch.from_numpy(img).unsqueeze(0)\n                img = img.cuda()\n                scale = scale.cuda()\n\n            \n                loc, conf, landms = net(img)  # forward pass\n                priorbox = PriorBox(cfg, image_size=(im_height, im_width))\n                priors = priorbox.forward()\n                priors = priors.cuda()\n                prior_data = priors.data\n                boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])\n                boxes = boxes * scale / resize\n                boxes = boxes.cpu().numpy()\n                scores = conf.squeeze(0).data.cpu().numpy()[:, 1]\n                landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])\n                scale1 = torch.Tensor([img.shape[3], img.shape[2], img.shape[3], img.shape[2],\n                                       img.shape[3], img.shape[2], img.shape[3], img.shape[2],\n                                       img.shape[3], img.shape[2]])\n                scale1 = scale1.cuda()\n                landms = landms * scale1 / resize\n                landms = landms.cpu().numpy()\n\n                # ignore low scores\n                inds = np.where(scores > 0.02)[0]\n                boxes = boxes[inds]\n                landms = landms[inds]\n                scores = scores[inds]\n\n                # keep top-K before NMS\n                order = scores.argsort()[::-1]\n                # order = scores.argsort()[::-1][:args.top_k]\n                boxes = boxes[order]\n                landms = landms[order]\n                scores = scores[order]\n\n                # do NMS\n                dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)\n                keep = py_cpu_nms(dets, 0.4)\n                # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)\n                dets = dets[keep, :]\n                landms = landms[keep]\n\n                dets = np.concatenate((dets, landms), axis=1)\n\n                for b in dets:\n                    if b[4] < 0.7:\n                        continue\n                    b = list(map(int, b))\n                    for j in range(0, 4):\n                        if b[j] < 0:\n                            b[j] = 0\n\n                    w = (b[2]-b[0])/2+b[0]\n                    h = (b[3]-b[1])/2+b[1]\n                    if index == 0:\n                        stackw.append(w)\n                        stackh.append(h)\n                        crop_img = img_raw[b[1]:b[3], b[0]:b[2]]\n                        crop_img = Image.fromarray(cv2.cvtColor(crop_img,cv2.COLOR_BGR2RGB)) # bgr to rgb\n                        temp = transform(crop_img)\n                        res.append([])\n                        res[0].append(temp)\n                        nums_temp = nums_temp + 1\n                        indexs.append(1)\n                        index = index + 1\n                    else:\n                        w_ = [stackw[j]-w for j in range(0, len(stackw))]\n                        h_ = [stackh[j]-h for j in range(0, len(stackh))]\n                        d2 = [w_[j]**2 for j in range(0, len(w_))]\n                        d2_ = [h_[j]**2 for j in range(0, len(h_))]\n                        d2 = [d2[k]+d2_[k] for k in range(0, len(d2))]\n                        d = [math.sqrt(s) for s in d2]\n                        r = min(d)\n                        l = d.index(r)\n                        if r < ranges:\n                            crop_img = img_raw[b[1]:b[3], b[0]:b[2]]\n                            crop_img = Image.fromarray(cv2.cvtColor(crop_img,cv2.COLOR_BGR2RGB))\n\n                            temp = transform(crop_img)\n                            res[l].append(temp)\n                            nums_temp = nums_temp + 1\n                            stackw[l] = w\n                            stackh[l] = h\n                            indexs[l] = indexs[l] + 1\n                        else:\n                            stackw.append(w)\n                            stackh.append(h)\n                            crop_img = img_raw[b[1]:b[3], b[0]:b[2]]\n                            crop_img = Image.fromarray(cv2.cvtColor(crop_img,cv2.COLOR_BGR2RGB))\n\n                            temp = transform(crop_img)\n                            res.append([])\n                            res[-1].append(temp)\n\n                            nums_temp = nums_temp + 1\n                            indexs.append(1)\n\n                if indexs != []:\n                    if max(indexs) == length_frames:\n                        break\n            success, frame = videoCapture.read()\n            n_frame = n_frame + 1\n\n        videoCapture.release()\n        if not is_multi:\n            for i in range(0, len(indexs)):\n                if indexs[i] == length_frames:\n                    results.append(torch.stack(res[i], dim=0))\n                    break\n        else:\n            for i in range(0, len(indexs)):\n                if indexs[i] == length_frames:\n                    is_null = False\n                    results.append(torch.stack(res[i], dim=0))\n    \n\n    return torch.stack(results, dim=0), is_null\n\n\n#############################################################################\n# model      -- cnn_encoder and rnn_decoder\n# device     -- GPU\n# X          -- input data\n\ndef validation(model, device, X):\n    # set model as testing mode\n    cnn_encoder, rnn_decoder = model\n    cnn_encoder.eval()\n    rnn_decoder.eval()\n\n    test_loss = 0\n    all_y = []\n    all_y_pred = []\n    with torch.no_grad():\n        if True:\n            X = X.to(device)\n\n            output = rnn_decoder(cnn_encoder(X))\n\n            y_pred = output.max(1, keepdim=True)[1]  # (y_pred != output) get the index of the max log-probability\n\n            return output, y_pred\n\n\ndef check_keys(model, pretrained_state_dict):\n    ckpt_keys = set(pretrained_state_dict.keys())\n    model_keys = set(model.state_dict().keys())\n    used_pretrained_keys = model_keys & ckpt_keys\n    unused_pretrained_keys = ckpt_keys - model_keys\n    missing_keys = model_keys - ckpt_keys\n    print('Missing keys:{}'.format(len(missing_keys)))\n    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))\n    print('Used keys:{}'.format(len(used_pretrained_keys)))\n    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'\n    return True\n\n\ndef remove_prefix(state_dict, prefix):\n    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''\n    print('remove prefix \\'{}\\''.format(prefix))\n    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x\n    return {f(key): value for key, value in state_dict.items()}\n\n\ndef load_model(model, pretrained_path, load_to_cpu):\n    print('Loading pretrained model from {}'.format(pretrained_path))\n    if load_to_cpu:\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)\n    else:\n        device = torch.cuda.current_device()\n        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))\n    if \"state_dict\" in pretrained_dict.keys():\n        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')\n    else:\n        pretrained_dict = remove_prefix(pretrained_dict, 'module.')\n    check_keys(model, pretrained_dict)\n    model.load_state_dict(pretrained_dict, strict=False)\n    return model\n\n\n# Detect devices\nuse_cuda = torch.cuda.is_available()                   # check if GPU exists\ndevice = torch.device(\"cuda\" if use_cuda else \"cpu\")   # use CPU or GPU\n\n# Data loading parameters\n# params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n\n\n\ntransform = transforms.Compose([transforms.Resize([res_size, res_size]),\n                                transforms.ToTensor(),\n                                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n\n\nselected_frames = []\nfor i in range(0,frame_len):\n    selected_frames.append(i)\n\n\n# Create model\ncnn_encoder = ResCNNEncoder(fc_hidden1=CNN_fc_hidden1, fc_hidden2=CNN_fc_hidden2, drop_p=dropout_p, CNN_embed_dim=CNN_embed_dim).to(device)\nrnn_decoder = DecoderRNN(CNN_embed_dim=CNN_embed_dim, h_RNN_layers=RNN_hidden_layers, h_RNN=RNN_hidden_nodes, \n                         h_FC_dim=RNN_FC_dim, drop_p=dropout_p, num_classes=k).to(device)\n\ncnn_encoder.load_state_dict(torch.load(cnn_path))\nrnn_decoder.load_state_dict(torch.load(rnn_path))\n\n\n# Parallelize model to multiple GPUs\nif torch.cuda.device_count() > 1:\n    print(\"Using\", torch.cuda.device_count(), \"GPUs!\")\n    cnn_encoder = nn.DataParallel(cnn_encoder)\n    rnn_decoder = nn.DataParallel(rnn_decoder)\n\n    # Combine all EncoderCNN + DecoderRNN parameters\n    crnn_params = list(cnn_encoder.module.fc1.parameters()) + list(cnn_encoder.module.bn1.parameters()) + \\\n                  list(cnn_encoder.module.fc2.parameters()) + list(cnn_encoder.module.bn2.parameters()) + \\\n                  list(cnn_encoder.module.fc3.parameters()) + list(rnn_decoder.parameters())\n\nelif torch.cuda.device_count() == 1:\n    print(\"Using\", torch.cuda.device_count(), \"GPU!\")\n    # Combine all EncoderCNN + DecoderRNN parameters\n    crnn_params = list(cnn_encoder.fc1.parameters()) + list(cnn_encoder.bn1.parameters()) + \\\n                  list(cnn_encoder.fc2.parameters()) + list(cnn_encoder.bn2.parameters()) + \\\n                  list(cnn_encoder.fc3.parameters()) + list(rnn_decoder.parameters())\n\noptimizer = torch.optim.Adam(crnn_params, lr=learning_rate)\n\n\n# record training process\nepoch_train_losses = []\nepoch_train_scores = []\nepoch_test_losses = []\nepoch_test_scores = []\n\n\nis_multi=True\n# cfg = cfg_re50\ncfg = cfg_mnet\nnet = RetinaFace(cfg=cfg, phase = 'test')        \n\n# net = load_model(net,'models/Resnet50_Final.pth', False) # when cfg = cfg_re50\nnet = load_model(net,'models/mobilenet0.25_Final.pth', False)  # when cfg = cfg_mnet\n\nnet.eval()\n\ndata_path = data_path\n\ndirs_0 = []\ndirs_1 = []\nfor i in range(0, len(data_path)):\n\n    dirs = os.listdir(data_path[i])\n    dir_0 = []\n    dir_1 = []\n    for s in dirs:\n        if s[-4:] != '.mp4':\n            continue\n        dir_0.append(s)                # video_name = s[:-4]\n\n    dirs_t = [os.path.join(data_path[i], t) for t in dir_0]\n    dirs_0 = dirs_0 + dirs_t\n\n\nlength_0 = len(dirs_0)\ntransform = transform\nframes = selected_frames\nlength_frames = len(frames)\nbatch_size = 1\n\n\ndirs = dirs_0 + dirs_1\nlength = len(dirs_0)\ntotal = 0\ncorre = 0\nrang = 0.35  # classification range\n\nfilenames=[]\nlabels = []\n\nfor it in range(0, length):\n    X, is_null = get_data(dirs_0, it, net, transform, device, frames, length_frames, batch_size, is_multi)\n    # print(X.shape)\n    if is_null:\n        path, filename = os.path.split(dirs_0[it])\n        filenames.append(filename)\n        labels.append(0.999)\n        continue\n    losses, pred = validation([cnn_encoder, rnn_decoder], device, X)\n    # print(it, F.softmax(losses), pred)\n    probs = F.softmax(losses)\n    probs = probs.cpu().numpy()\n    maxs = 0\n    mins = 1\n    print(dirs_0[it])\n    path, filename = os.path.split(dirs_0[it])\n    filenames.append(filename)\n    print('Have '+str(len(probs))+' faces in the video')\n    \n    for i in range(0, len(probs)):\n        temp = probs[i][0]\n        if temp > rang:\n            print('Fake probability of '+str(i+1)+' face is:', temp)\n            if temp > maxs:\n                maxs = temp\n        else:\n            print('Fake probability of '+str(i+1)+' face is:', temp)\n            if temp < mins:\n                mins = temp\n        \n    if maxs > 0:\n        print('*** The fake probability of the video is', maxs, ', therefore it is a fake video ***')\n        labels.append(maxs)\n    else:\n        print('*** The fake probability of the video is', mins, ', therefore it is a real video ***')\n        labels.append(mins)\ndataframe=pd.DataFrame({'filename':filenames,'label':labels})\nnew_df=dataframe.sort_values(by=\"filename\", ascending=True)\nnew_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}