{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import print_function\nimport cv2\nimport torch\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\n\nfrom PIL import Image,ImageFilter\nimport matplotlib.pyplot as plt\n\nimport torchvision.transforms as transforms\nimport torchvision.models as models\n\nimport copy\n\n# Make all plots shown immediately\nplt.ion()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-29T12:11:35.692344Z","iopub.execute_input":"2022-01-29T12:11:35.692631Z","iopub.status.idle":"2022-01-29T12:11:35.701768Z","shell.execute_reply.started":"2022-01-29T12:11:35.692600Z","shell.execute_reply":"2022-01-29T12:11:35.700922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style=\"background-color:#A50034; font-family:segoeui; font-size:200%; text-align:center; border-radius: 15px 50px;\"> 1. Introduction </h1>\n\n![img](https://upload.wikimedia.org/wikipedia/commons/a/a2/Mona_lisa_the_starry_night_o_lbfgs_i_content_h_720_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg)\n\nNeural style transfer (NST) refers to a class of software algorithms that manipulate digital images, or videos, in order to adopt the appearance or visual style of another image. NST algorithms are characterized by their use of deep neural networks for the sake of image transformation. Common uses for NST are the creation of artificial artwork from photographs, for example by transferring the appearance of famous paintings to user-supplied photographs. Several notable mobile apps use NST techniques for this purpose, including DeepArt and Prisma. This method has been used by artists and designers around the globe to develop new artwork based on existent style(s)(Wiki).\n<br>\n**How NST Works**\n<br>\n![nst](https://cdn.hackernoon.com/hn-images/1*4Pm-XwKTevANbKtiI_RQ5w.png)\n\n","metadata":{}},{"cell_type":"code","source":"# desired size of the output image\nimport cv2\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\nimsize = 1024\n\nloader = transforms.Compose([\n    transforms.Resize((imsize,imsize)),  # scale imported image (both dimenstions)\n    transforms.ToTensor()])  # transform it into a torch tensor\n\n\ndef image_loader(image_name,blur=False):\n    image = Image.open(image_name)\n    \n    if blur:\n        # Use for the style image - smoother transfer\n        image = image.filter(ImageFilter.GaussianBlur(radius=4))\n    \n    image = loader(image).unsqueeze(0)\n    return image.to(device, torch.float)\n\n\nstyle_img = image_loader(\"../input/tensorflow-great-barrier-reef/train_images/video_1/10011.jpg\",blur=True)\n\nprint(style_img.size())\n\ncontent_img = image_loader(\"../input/cotsst/COTS_edited-1.jpg\")\nprint(content_img.size())\n\n\nassert style_img.size() == content_img.size() # Ensure if you play with images sizes","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:39.808671Z","iopub.execute_input":"2022-01-29T12:11:39.808938Z","iopub.status.idle":"2022-01-29T12:11:39.948150Z","shell.execute_reply.started":"2022-01-29T12:11:39.808909Z","shell.execute_reply":"2022-01-29T12:11:39.947365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unloader = transforms.ToPILImage()  # reconvert into PIL image\n\n\ndef imshow(tensor, title=None):\n    image = tensor.cpu().clone()  # we clone the tensor to not do changes on it\n    image = image.squeeze(0)      # remove the fake batch dimension\n    image = unloader(image)\n    plt.imshow(image)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001) # pause a bit so that plots are updated\n\n\nplt.figure()\nimshow(style_img, title='Style Image')\n\nplt.figure()\nimshow(content_img, title='Content Image')","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:40.384640Z","iopub.execute_input":"2022-01-29T12:11:40.385053Z","iopub.status.idle":"2022-01-29T12:11:41.344270Z","shell.execute_reply.started":"2022-01-29T12:11:40.385017Z","shell.execute_reply":"2022-01-29T12:11:41.343477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style=\"background-color:#A50034; font-family:segoeui; font-size:200%; text-align:center; border-radius: 15px 50px;\"> 2. Network Preparation </h1>\nLet`s reproduce the pytorch example of the style transfer. Additionally we will explore the network structure","metadata":{}},{"cell_type":"code","source":"class ContentLoss(nn.Module):\n    def __init__(self,target,):\n        super(ContentLoss, self).__init__()\n        self.target = target.detach()\n        \n    def forward(self,input):\n        self.loss=F.mse_loss(input,self.target)\n        return input\n        \ndef gram_matrix(input):\n    a, b, c, d = input.size()  # a=batch size(=1)\n    # b=number of feature maps\n    # (c,d)=dimensions of a f. map (N=c*d)\n\n    features = input.view(a * b, c * d)  # resise F_XL into \\hat F_XL\n\n    G = torch.mm(features, features.t())  # compute the gram product\n\n    return G.div(a * b * c * d)\n\nclass StyleLoss(nn.Module):\n\n    def __init__(self, target_feature):\n        super(StyleLoss, self).__init__()\n        self.target = gram_matrix(target_feature).detach()\n\n    def forward(self, input):\n        G = gram_matrix(input)\n        self.loss = F.mse_loss(G, self.target)\n        return input","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:41.345567Z","iopub.execute_input":"2022-01-29T12:11:41.345792Z","iopub.status.idle":"2022-01-29T12:11:41.357735Z","shell.execute_reply.started":"2022-01-29T12:11:41.345762Z","shell.execute_reply":"2022-01-29T12:11:41.357093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get VGG16 model as exaple\ncnn = models.vgg16(pretrained=True).features.to(device).eval()\ncnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)\ncnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:41.852679Z","iopub.execute_input":"2022-01-29T12:11:41.852988Z","iopub.status.idle":"2022-01-29T12:11:43.368237Z","shell.execute_reply.started":"2022-01-29T12:11:41.852957Z","shell.execute_reply":"2022-01-29T12:11:43.367528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Visualize Network Structure\ntry:\n    from torchviz import make_dot\nexcept:\n    !pip install torchviz\n    from torchviz import make_dot\n#Generate some data for visualization\ny_cnn = cnn(content_img)\n    \nmake_dot(y_cnn, params=dict(cnn.named_parameters()), show_attrs=True, show_saved=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:43.369845Z","iopub.execute_input":"2022-01-29T12:11:43.370140Z","iopub.status.idle":"2022-01-29T12:11:43.476836Z","shell.execute_reply.started":"2022-01-29T12:11:43.370104Z","shell.execute_reply":"2022-01-29T12:11:43.476052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Normalization(nn.Module):\n    def __init__(self, mean, std):\n        super(Normalization, self).__init__()\n        # .view the mean and std to make them [C x 1 x 1] so that they can\n        # directly work with image Tensor of shape [B x C x H x W].\n        # B is batch size. C is number of channels. H is height and W is width.\n        self.mean = torch.tensor(mean).view(-1, 1, 1)\n        self.std = torch.tensor(std).view(-1, 1, 1)\n\n    def forward(self, img):\n        # normalize img\n        return (img - self.mean) / self.std","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:11:43.478518Z","iopub.execute_input":"2022-01-29T12:11:43.479410Z","iopub.status.idle":"2022-01-29T12:11:43.486997Z","shell.execute_reply.started":"2022-01-29T12:11:43.479367Z","shell.execute_reply":"2022-01-29T12:11:43.486072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can select the layer to recover\ncontent_layers_default = ['conv_5']\nstyle_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']\n\ndef get_style_model_and_losses(cnn, normalization_mean, normalization_std,\n                               style_img, content_img,\n                               content_layers=content_layers_default,\n                               style_layers=style_layers_default):\n    # normalization \n    normalization = Normalization(normalization_mean, normalization_std).to(device)\n\n    # just in order to have an iterable access to or list of content/syle\n    # losses\n    content_losses = []\n    style_losses = []\n\n    # assuming that cnn is a nn.Sequential, so we make a new nn.Sequential\n    # to put in modules that are supposed to be activated sequentially\n    model = nn.Sequential(normalization)\n\n    i = 0  # increment every time we see a conv\n    for layer in cnn.children():\n        if isinstance(layer, nn.Conv2d):\n            i += 1\n            name = 'conv_{}'.format(i)\n        elif isinstance(layer, nn.ReLU):\n            name = 'relu_{}'.format(i)\n            # The in-place version doesn't play very nicely with the ContentLoss\n            # and StyleLoss we insert below. So we replace with out-of-place\n            # ones here.\n            layer = nn.ReLU(inplace=False)\n        elif isinstance(layer, nn.MaxPool2d):\n            name = 'pool_{}'.format(i)\n        elif isinstance(layer, nn.BatchNorm2d):\n            name = 'bn_{}'.format(i)\n        else:\n            raise RuntimeError('Unrecognized layer: {}'.format(layer.__class__.__name__))\n\n        model.add_module(name, layer)\n\n        if name in content_layers:\n            # add content loss:\n            target = model(content_img).detach()\n            content_loss = ContentLoss(target)\n            model.add_module(\"content_loss_{}\".format(i), content_loss)\n            content_losses.append(content_loss)\n\n        if name in style_layers:\n            # add style loss:\n            target_feature = model(style_img).detach()\n            style_loss = StyleLoss(target_feature)\n            model.add_module(\"style_loss_{}\".format(i), style_loss)\n            style_losses.append(style_loss)\n\n    # now we trim off the layers after the last content and style losses\n    for i in range(len(model) - 1, -1, -1):\n        if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):\n            break\n\n    model = model[:(i + 1)]\n\n    return model, style_losses, content_losses","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:22:50.679681Z","iopub.execute_input":"2022-01-29T12:22:50.679957Z","iopub.status.idle":"2022-01-29T12:22:50.691819Z","shell.execute_reply.started":"2022-01-29T12:22:50.679927Z","shell.execute_reply":"2022-01-29T12:22:50.691161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate white noise\ninput_img = torch.randn(content_img.data.size(),device=device)\n\n\n# Input Image\nplt.figure()\nimshow(input_img, title='Input Image')","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:22:51.305670Z","iopub.execute_input":"2022-01-29T12:22:51.306226Z","iopub.status.idle":"2022-01-29T12:22:51.674048Z","shell.execute_reply.started":"2022-01-29T12:22:51.306186Z","shell.execute_reply":"2022-01-29T12:22:51.673373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_input_optimizer(input_img):\n    # this line to show that input is a parameter that requires a gradient\n    optimizer = optim.LBFGS([input_img])\n    return optimizer","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:22:52.321960Z","iopub.execute_input":"2022-01-29T12:22:52.322280Z","iopub.status.idle":"2022-01-29T12:22:52.327462Z","shell.execute_reply.started":"2022-01-29T12:22:52.322242Z","shell.execute_reply":"2022-01-29T12:22:52.326542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style=\"background-color:#A50034; font-family:segoeui; font-size:200%; text-align:center; border-radius: 15px 50px;\"> 3. Training </h1>\n","metadata":{}},{"cell_type":"code","source":"def run_style_transfer(cnn, normalization_mean, normalization_std,\n                       content_img, style_img, input_img, num_steps=500,\n                       style_weight=100000, content_weight=1):\n    \"\"\"Run the style transfer.\"\"\"\n    print('Building the style transfer model..')\n    model, style_losses, content_losses = get_style_model_and_losses(cnn,\n        normalization_mean, normalization_std, style_img, content_img)\n\n    # We want to optimize the input and not the model parameters so we\n    # update all the requires_grad fields accordingly\n    input_img.requires_grad_(True)\n    model.requires_grad_(False)\n\n    optimizer = get_input_optimizer(input_img)\n\n    print('Optimizing..')\n    run = [0]\n    while run[0] <= num_steps:\n\n        def closure():\n            # correct the values of updated input image\n            with torch.no_grad():\n                input_img.clamp_(0, 1)\n\n            optimizer.zero_grad()\n            model(input_img)\n            style_score = 0\n            content_score = 0\n\n            for sl in style_losses:\n                style_score += sl.loss\n            for cl in content_losses:\n                content_score += cl.loss\n\n            style_score *= style_weight\n            content_score *= content_weight\n\n            loss = style_score + content_score\n            loss.backward()\n\n            run[0] += 1\n            if run[0] % 50 == 0:\n                print(\"run {}:\".format(run))\n                print('Style Loss : {:4f} Content Loss: {:4f}'.format(\n                    style_score.item(), content_score.item()))\n                print()\n\n            return style_score + content_score\n\n        optimizer.step(closure)\n\n    # a last correction...\n    with torch.no_grad():\n        input_img.clamp_(0, 1)\n\n    return input_img","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:24:52.929641Z","iopub.execute_input":"2022-01-29T12:24:52.929920Z","iopub.status.idle":"2022-01-29T12:24:52.940500Z","shell.execute_reply.started":"2022-01-29T12:24:52.929887Z","shell.execute_reply":"2022-01-29T12:24:52.939525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std,\n                            content_img, style_img, input_img)\n\nplt.figure()\nimshow(output, title='Output Image')\n\nplt.ioff()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:24:53.107540Z","iopub.execute_input":"2022-01-29T12:24:53.107944Z","iopub.status.idle":"2022-01-29T12:26:00.616717Z","shell.execute_reply.started":"2022-01-29T12:24:53.107899Z","shell.execute_reply":"2022-01-29T12:26:00.616090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters can be optimized If you liked please upvote ;) \n","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:17:44.130890Z","iopub.execute_input":"2022-01-29T12:17:44.131732Z","iopub.status.idle":"2022-01-29T12:17:44.141622Z","shell.execute_reply.started":"2022-01-29T12:17:44.131686Z","shell.execute_reply":"2022-01-29T12:17:44.140913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}