{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":19205,"databundleVersionId":1011244,"sourceType":"competition"},{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"}],"dockerImageVersionId":30005,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Hello everyone!**\n\nOne way to generate new paintings is to **transfer the style of the target artist (Claude Monet) to already existings images of real life**. Here I randomly picked a real life image from Open Images Dataset and converted it using the painting style of Monet.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport numpy as np\nimport time\nfrom PIL import Image\n%matplotlib inline","metadata":{"id":"EuYCyA5gl4E4","execution":{"iopub.status.busy":"2023-11-19T11:52:51.119385Z","iopub.execute_input":"2023-11-19T11:52:51.119742Z","iopub.status.idle":"2023-11-19T11:52:55.798424Z","shell.execute_reply.started":"2023-11-19T11:52:51.119711Z","shell.execute_reply":"2023-11-19T11:52:55.797435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the function below we design a function to load images. We convert the image to the required dimension so that no matter which dimension the photo we upload is in, it will be resized accordingly in order to work.","metadata":{"id":"e5zHhFi4YhVe"}},{"cell_type":"code","source":"# A function to load the input images and set its dimensions to 1024 x 768\ndef load_image(image_path):\n    max_dim=512\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_image(img, channels=3)# decodes the image into a tensor\n    img = tf.image.convert_image_dtype(img, tf.float32)\n\n    shape = tf.cast(tf.shape(img)[:-1], tf.float32)\n    long_dim = max(shape)\n    scale = max_dim / long_dim\n    new_shape = tf.cast(shape * scale, tf.int32)\n\n    img = tf.image.resize(img, new_shape)\n    img = img[tf.newaxis, :]# broadcasting the image array so that it has a batch dimension\n\n    return img","metadata":{"id":"Ak8NXP86mWq5","execution":{"iopub.status.busy":"2023-11-19T11:54:54.709737Z","iopub.execute_input":"2023-11-19T11:54:54.710064Z","iopub.status.idle":"2023-11-19T11:54:54.718500Z","shell.execute_reply.started":"2023-11-19T11:54:54.710036Z","shell.execute_reply":"2023-11-19T11:54:54.717487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imshow(image, title=None):\n    if(len(image.shape) > 3):# suppose dim is like 1,2,4,2,2,1... it removes the ones so that only 3 values remain W,H,c\n        image=np.squeeze(image, axis=0)\n    plt.imshow(image)\n    if(title):# if there's a title mention it\n        plt.title(title)","metadata":{"id":"YPoIYOtzrasS","execution":{"iopub.status.busy":"2023-11-19T11:55:19.259719Z","iopub.execute_input":"2023-11-19T11:55:19.260071Z","iopub.status.idle":"2023-11-19T11:55:19.265856Z","shell.execute_reply.started":"2023-11-19T11:55:19.260037Z","shell.execute_reply":"2023-11-19T11:55:19.264870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see the images\ncontent_img=load_image('../input/gan-getting-started/photo_jpg/00068bc07f.jpg')\nstyle_img=load_image('../input/gan-getting-started/monet_jpg/2f90c99e10.jpg')\n\nplt.figure(figsize=(12,12))\nplt.subplot(1, 2, 1)\nimshow(content_img, 'Base Target Image')\nplt.subplot(1, 2, 2)\nimshow(style_img, 'Style image')","metadata":{"id":"02kdlfJet2Ji","outputId":"dbb90c51-b0d0-4890-91f7-7eba455b1129","execution":{"iopub.status.busy":"2023-11-19T11:55:34.269709Z","iopub.execute_input":"2023-11-19T11:55:34.270067Z","iopub.status.idle":"2023-11-19T11:55:36.795191Z","shell.execute_reply.started":"2023-11-19T11:55:34.270032Z","shell.execute_reply":"2023-11-19T11:55:36.794120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(content_img.shape)\nprint(style_img.shape)","metadata":{"id":"vVFKspAUGTDd","outputId":"2bfb1ed3-1d95-4146-b071-5b71cd0de82d","execution":{"iopub.status.busy":"2023-11-19T11:55:45.204812Z","iopub.execute_input":"2023-11-19T11:55:45.205150Z","iopub.status.idle":"2023-11-19T11:55:45.210027Z","shell.execute_reply.started":"2023-11-19T11:55:45.205120Z","shell.execute_reply":"2023-11-19T11:55:45.209082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the content image representation and load the model\nx=tf.keras.applications.vgg19.preprocess_input(content_img*255)# needs preprocessing for the model to be initialized\nx=tf.image.resize(x, (256,256))# the vgg19 model takes images in 256\nvgg_model=tf.keras.applications.VGG19(include_top=False, weights='imagenet')\nvgg_model.trainable=False\nvgg_model.summary()","metadata":{"id":"acSjybfw5k35","outputId":"67972759-d0c5-42b7-e955-62fb69dfabd7","execution":{"iopub.status.busy":"2023-11-19T11:55:50.189415Z","iopub.execute_input":"2023-11-19T11:55:50.189824Z","iopub.status.idle":"2023-11-19T11:55:53.912228Z","shell.execute_reply.started":"2023-11-19T11:55:50.189789Z","shell.execute_reply":"2023-11-19T11:55:53.911348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Chooose the content and style layers\ncontent_layers=['block4_conv2']\nstyle_layers = ['block1_conv1',\n                'block2_conv1',\n                'block3_conv1', \n                'block4_conv1', \n                'block5_conv1']","metadata":{"id":"ULHAsnwRgcfG","execution":{"iopub.status.busy":"2023-11-19T11:57:26.489185Z","iopub.execute_input":"2023-11-19T11:57:26.489540Z","iopub.status.idle":"2023-11-19T11:57:26.494201Z","shell.execute_reply.started":"2023-11-19T11:57:26.489506Z","shell.execute_reply":"2023-11-19T11:57:26.493237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build the model\ndef my_model(layer_names):\n    # Retrieve the output layers corresponding to the content and style layers\n    vgg_model = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n    vgg_model.trainable = False\n    outputs = [vgg_model.get_layer(name).output for name in layer_names]\n    model=tf.keras.Model([vgg_model.input], outputs)\n    return model","metadata":{"id":"9NUkwQi9j7Gd","execution":{"iopub.status.busy":"2023-11-19T12:01:45.997134Z","iopub.execute_input":"2023-11-19T12:01:45.997505Z","iopub.status.idle":"2023-11-19T12:01:46.003274Z","shell.execute_reply.started":"2023-11-19T12:01:45.997475Z","shell.execute_reply":"2023-11-19T12:01:46.002521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"style_extractor = my_model(style_layers)\nstyle_outputs = style_extractor(style_img*255)","metadata":{"id":"NZWP6435XbGa","execution":{"iopub.status.busy":"2023-11-19T12:02:06.677166Z","iopub.execute_input":"2023-11-19T12:02:06.677541Z","iopub.status.idle":"2023-11-19T12:02:07.079423Z","shell.execute_reply.started":"2023-11-19T12:02:06.677504Z","shell.execute_reply":"2023-11-19T12:02:07.078610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute the gram matrix\n# Einsum allows defining Tensors by defining their element-wise computation.\n# This computation is defined by equation, a shorthand form based on Einstein summation.\ndef gram_matrix(input_tensor): # input_tensor is of shape ch, n_H, n_W\n    result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)\n    input_shape = tf.shape(input_tensor)\n    num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32) # Unrolls n_H and n_W\n    return result/(num_locations)","metadata":{"id":"mTlc1rO7lljq","execution":{"iopub.status.busy":"2023-11-19T12:02:15.901891Z","iopub.execute_input":"2023-11-19T12:02:15.902243Z","iopub.status.idle":"2023-11-19T12:02:15.907743Z","shell.execute_reply.started":"2023-11-19T12:02:15.902215Z","shell.execute_reply":"2023-11-19T12:02:15.906865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class entire_model(tf.keras.models.Model):\n    def __init__(self, style_layers, content_layers):\n        super(entire_model, self).__init__()\n        self.vgg=my_model(style_layers + content_layers)\n        self.style_layers=style_layers\n        self.content_layers=content_layers\n        self.num_style_layers=len(style_layers)\n        self.vgg.trainable=False\n\n    def call(self, inputs):\n        inputs=inputs*255.0 # Scale back the pixel values\n        preprocessed_input=tf.keras.applications.vgg19.preprocess_input(inputs)\n        outputs=self.vgg(preprocessed_input)# Pass the preprocessed input to my_model\n\n        # Separate the representations of style and content\n        style_outputs, content_outputs=(outputs[:self.num_style_layers], outputs[self.num_style_layers:])\n        # Calculate the gram matrix for each layer in the style output. This will be the final style representation\n        style_outputs=[gram_matrix(layer) for layer in style_outputs]\n\n        # Store the content and style representation in dictionaries in a layer by layer manner\n        content_dict = {content_name:value\n                    for content_name, value\n                    in zip(self.content_layers, content_outputs)}\n\n        style_dict = {style_name:value\n                  for style_name, value\n                  in zip(self.style_layers, style_outputs)}\n\n        return {'content': content_dict, 'style': style_dict}\n        # Returns a dict of dicts with content and style representations, i.e., gram matrix of the style_layers and\n        # the content of the content_layers","metadata":{"id":"PAMfMhUK5qjs","execution":{"iopub.status.busy":"2023-11-19T12:02:46.216715Z","iopub.execute_input":"2023-11-19T12:02:46.217056Z","iopub.status.idle":"2023-11-19T12:02:46.229976Z","shell.execute_reply.started":"2023-11-19T12:02:46.217019Z","shell.execute_reply":"2023-11-19T12:02:46.228918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now we extract the style and content features by calling the above class\nextractor=entire_model(style_layers, content_layers)\nstyle_targets = extractor(style_img)['style']\ncontent_targets = extractor(content_img)['content']\n\nresults = extractor(tf.constant(content_img))","metadata":{"id":"zTZDU5Y0Dy43","execution":{"iopub.status.busy":"2023-11-19T12:02:50.926341Z","iopub.execute_input":"2023-11-19T12:02:50.926723Z","iopub.status.idle":"2023-11-19T12:02:51.543757Z","shell.execute_reply.started":"2023-11-19T12:02:50.926692Z","shell.execute_reply":"2023-11-19T12:02:51.542865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"style_weight=40\ncontent_weight=10\n\n# Custom weights for different style layers\nstyle_weights = {'block1_conv1': 0.7,\n                 'block2_conv1': 0.19,\n                 'block3_conv1': 0.24,\n                 'block4_conv1': 0.11,\n                 'block5_conv1': 0.26}\n# style_weights = {'block1_conv1': 0.3,\n#                  'block2_conv1': 0.45,\n#                  'block3_conv1': 0.15,\n#                  'block4_conv1': 0.05,\n#                  'block5_conv1': 0.05}","metadata":{"id":"HUwwlgzvJxS6","execution":{"iopub.status.busy":"2023-11-19T12:02:55.636280Z","iopub.execute_input":"2023-11-19T12:02:55.636632Z","iopub.status.idle":"2023-11-19T12:02:55.641663Z","shell.execute_reply.started":"2023-11-19T12:02:55.636602Z","shell.execute_reply":"2023-11-19T12:02:55.640857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def total_cost(outputs):\n    style_outputs=outputs['style']\n    content_outputs=outputs['content']\n    style_loss=tf.add_n([style_weights[name]*tf.reduce_mean((style_outputs[name]-style_targets[name])**2)\n                        for name in style_outputs.keys()])\n    style_loss*=style_weight/len(style_layers)# Normalize\n\n    content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2)\n                             for name in content_outputs.keys()])\n    content_loss*=content_weight/len(content_layers)\n    loss=style_loss+content_loss\n    return loss","metadata":{"id":"gojU17ArLXMt","execution":{"iopub.status.busy":"2023-11-19T12:03:00.026319Z","iopub.execute_input":"2023-11-19T12:03:00.026673Z","iopub.status.idle":"2023-11-19T12:03:00.034999Z","shell.execute_reply.started":"2023-11-19T12:03:00.026644Z","shell.execute_reply":"2023-11-19T12:03:00.034152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a tf.Variable to contain the image to optimize\ngenerate_image = tf.Variable(content_img)\n# Since this is a float image, define a function to keep the pixel values between 0 and 1\ndef clip_0_1(image):\n    return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)","metadata":{"id":"ir076-JMOMpr","execution":{"iopub.status.busy":"2023-11-19T12:03:04.956294Z","iopub.execute_input":"2023-11-19T12:03:04.956659Z","iopub.status.idle":"2023-11-19T12:03:04.962807Z","shell.execute_reply.started":"2023-11-19T12:03:04.956629Z","shell.execute_reply":"2023-11-19T12:03:04.961921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)","metadata":{"id":"ki8m2cLQOjee","execution":{"iopub.status.busy":"2023-11-19T12:03:24.201563Z","iopub.execute_input":"2023-11-19T12:03:24.201902Z","iopub.status.idle":"2023-11-19T12:03:24.206718Z","shell.execute_reply.started":"2023-11-19T12:03:24.201874Z","shell.execute_reply":"2023-11-19T12:03:24.205679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function()\ndef train_step(image):\n    with tf.GradientTape() as tape:\n        outputs = extractor(image)\n        loss = total_cost(outputs)\n\n    grad = tape.gradient(loss, image)\n    opt.apply_gradients([(grad, image)])\n    image.assign(clip_0_1(image))","metadata":{"id":"b8p6gBM9Nbg0","execution":{"iopub.status.busy":"2023-11-19T12:03:25.986340Z","iopub.execute_input":"2023-11-19T12:03:25.986746Z","iopub.status.idle":"2023-11-19T12:03:25.993030Z","shell.execute_reply.started":"2023-11-19T12:03:25.986711Z","shell.execute_reply":"2023-11-19T12:03:25.992153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_iterations=2500\nfor i in range(num_iterations):\n    train_step(generate_image)\n    if(i%500==0):\n        plt.figure(figsize=(12,12))\n        plt.subplot(1, 3, 1)\n        imshow(content_img, 'Original Image')\n        plt.subplot(1, 3, 2)\n        imshow(style_img, 'Style Image')\n        plt.subplot(1, 3, 3)\n        imshow(np.squeeze(generate_image.read_value(), 0), 'New Image - Step'+str(i))\n        ","metadata":{"id":"q6i1G-QmyI_f","outputId":"6275fa2d-927c-4862-f717-324d91cc01c7","execution":{"iopub.status.busy":"2023-11-19T12:03:31.256491Z","iopub.execute_input":"2023-11-19T12:03:31.256961Z","iopub.status.idle":"2023-11-19T12:05:24.995081Z","shell.execute_reply.started":"2023-11-19T12:03:31.256917Z","shell.execute_reply":"2023-11-19T12:05:24.994147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_step(generate_image)\nplt.imshow(np.squeeze(generate_image.read_value(), 0))\nplt.axis('off')\nfig1 = plt.gcf()\nfig1.savefig('new_image.png', bbox_inches='tight')","metadata":{"id":"-Y96xR8uybmk","outputId":"eec15caa-b4f1-477e-c66b-bd67c6c31529","execution":{"iopub.status.busy":"2023-11-19T12:05:45.567375Z","iopub.execute_input":"2023-11-19T12:05:45.567714Z","iopub.status.idle":"2023-11-19T12:05:45.786643Z","shell.execute_reply.started":"2023-11-19T12:05:45.567684Z","shell.execute_reply":"2023-11-19T12:05:45.785534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next steps:\n\n - Generate 10000 images\n - zip and submit","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}