{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install facenet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport time\nimport copy\nimport tqdm\nimport torchvision\nimport torch.utils.data\nimport torch.nn as nn\nimport torch.optim as optim\nfrom PIL import Image\nfrom facenet_pytorch import InceptionResnetV1\nfrom torch.utils.tensorboard import SummaryWriter\nimport matplotlib.pyplot as plt\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nprint(os.getcwd())\nfor dirname, dirs, _ in os.walk('../input'):\n    for d in dirs:\n        print(os.path.join(dirname, d))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"I try to train the fake classifier using transform learning\n\nFirst lets define my dataset and data loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transform the image in the way facenet_pytorch does\nclass ImgTransform:\n    def __call__(self, x):\n        np_img = np.array(x).astype(np.float32)\n        np_img = (np_img - 127.5) / 128.0\n        np_img = np.transpose(np_img, axes=[2, 0, 1])\n        return np_img\n\n\n# ImageFolder returns 0 in case of fake and 1 in case of real. We want the opposite\nclass LabelTransform:\n    def __call__(self, x):\n        return np.array((x+1) & 1, dtype=np.float32)\n\n\nimg_transfrom = ImgTransform()\nlabel_transform = LabelTransform()\n\ntrain_dataset = torchvision.datasets.ImageFolder('../input/dfdc-faces-of-the-train-sample/train', transform=img_transfrom, target_transform=label_transform)\nvalid_dataset = torchvision.datasets.ImageFolder('../input/dfdc-faces-of-the-train-sample/validation', transform=img_transfrom, target_transform=label_transform)\n\n# Lets look at an example of image loaded\nimg, lbl = train_dataset[0]\nimg = np.transpose(img, axes=[1, 2, 0])\npil_img = Image.fromarray(((img+1)*128).astype(np.uint8))\ndisplay(pil_img)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, 32, shuffle=True)\nvalid_loader = torch.utils.data.DataLoader(valid_dataset, 32, shuffle=False)\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets load the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet = InceptionResnetV1(pretrained='vggface2', classify=True, num_classes=1).eval()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following code was taken from https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html. I changed it a little for my needs."},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, train_criterion, valid_criterion, optimizer, scheduler=None, num_epochs=25):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_loss = 10000.0\n\n    dataset_sizes = {'train': len(train_dataset), 'val': len(valid_dataset)}\n    dataloaders = {'train': train_loader, 'val': valid_loader}\n    loos_func = {'train': train_criterion, 'val': valid_criterion}\n    epoch_losses = {'train': [], 'val': []}\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()  # Set model to evaluate mode\n\n            running_loss = 0.0\n\n            # Iterate over data.\n            # for inputs, labels in tqdm.tqdm(dataloaders[phase]):\n            for inputs, labels in dataloaders[phase]:\n                labels = labels.reshape(len(labels), 1)\n                torch.cuda.empty_cache()\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    loss = loos_func[phase](outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                n_real = (labels == 0).sum()\n                n_fake = (labels == 1).sum()\n                avg_factor = n_real+n_fake*loos_func[phase].pos_weight\n                avg_loss = loss / avg_factor\n                running_loss += avg_loss.item() * inputs.size(0)\n                if phase == 'train' and scheduler is not None:\n                    scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_losses[phase].append(epoch_loss)\n\n            print('{} Loss: {:.4f}'.format(phase, epoch_loss))\n\n            # deep copy the model\n            if phase == 'val' and epoch_loss < best_loss:\n                best_loss = epoch_loss\n                best_model_wts = copy.deepcopy(model.state_dict())\n                torch.save(model.state_dict(), 'resnet_params.pth')\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Loss: {:4f}'.format(best_loss))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, epoch_losses['train'],  epoch_losses['val']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the history of the loss values in training epoches\ndef plot_history(train_history, val_history):\n    epoches = [i for i in range(len(train_history))]\n    plt.plot(epoches, train_history, label='train')\n    plt.plot(epoches, val_history, label='validation')\n    plt.legend()\n    plt.xlabel('epoch numbers')\n    plt.ylabel('loss value')\n    plt.xticks(epoches)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"According to facenet_pytorch help, the last layer of resnet has random weights.\nFirsts, lets train the model so that only the last layer change its weight to refine them to something reasonable"},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet = resnet.to(device)\nresnet_layers = [module for module in resnet.modules()]\nfor param in resnet.parameters():\n    param.requires_grad = False\nactive_params = list(resnet_layers[-1].parameters())\nfor param in active_params:\n    param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_w = torch.Tensor([20699/73154]).to(device) # There are 73154 fake examples and only 20699 in the training set\nval_w = torch.Tensor([6029/24765]).to(device) # There are 24765 fake examples and only 6029 in the validation set\ntrain_criterion = nn.BCEWithLogitsLoss(pos_weight=train_w, reduction='sum')\nvalid_criterion = nn.BCEWithLogitsLoss(pos_weight=val_w, reduction='sum')\noptimizer = optimizer_conv = optim.SGD(active_params, lr=0.001, momentum=0.9)\nresnet, train_history, val_history = train_model(resnet, train_criterion, valid_criterion, optimizer, num_epochs=5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's visualize the traiaing history\nplot_history(train_history, val_history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now add more layers to the training and save the last layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"active_params = []\nfor layer in resnet_layers[-3:]:\n    active_params += layer.parameters()\nfor param in active_params:\n    param.requires_grad = True\noptimizer = optimizer_conv = optim.SGD(active_params, lr=0.000025, momentum=0.9)\nresnet, train_history, test_history = train_model(resnet, train_criterion, valid_criterion, optimizer, num_epochs=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's visualize the traiaing history\nplot_history(train_history, val_history)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}