{"cells":[{"metadata":{},"cell_type":"markdown","source":"
\n

\n

Cassava Leaf Disease - CycleGAN data augmentation

\n\n\nTODO: Explain the goal of the noteobok and give some description\n\n\n#### This work is based on my previous work [Improving CycleGAN - Monet paintings](https://www.kaggle.com/dimitreoliveira/improving-cyclegan-monet-paintings) from the [I’m Something of a Painter Myself](https://www.kaggle.com/c/gan-getting-started) competition.\n- Dataset source [here](https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tfrecords-classes-512x512)\n- Dataset source [discussion thread](https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/198744)"},{"metadata":{},"cell_type":"markdown","source":"## Dependencies"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import os, random, json, PIL, shutil, re, imageio, glob\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom PIL import ImageDraw\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import Model, losses, optimizers, applications\nfrom tensorflow.keras.callbacks import Callback\n\n\ndef seed_everything(seed=0):\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n os.environ['TF_DETERMINISTIC_OPS'] = '1'\n \nSEED = 0\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hardware configuration"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n tpu = None\n\nif tpu:\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n strategy = tf.distribute.get_strategy()\n\n\nREPLICAS = strategy.num_replicas_in_sync\nAUTO = tf.data.experimental.AUTOTUNE\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model parameters"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"HEIGHT_DS = 512\nWIDTH_DS = 512\nHEIGHT = 256\nWIDTH = 256\nHEIGHT_RESIZE = 256\nWIDTH_RESIZE = 256\nCHANNELS = 3\nBATCH_SIZE = 16\nEPOCHS = 30\nTRANSFORMER_BLOCKS = 4\nGENERATOR_LR = 2e-4\nDISCRIMINATOR_LR = 2e-4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-classes-{HEIGHT_DS}x{WIDTH_DS}')\n\nCBB_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/CBB*.tfrec')\nCBSD_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/CBSD*.tfrec')\nCGM_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/CGM*.tfrec')\nCMD_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/CMD*.tfrec')\nHEALTHY_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/Healthy*.tfrec')\n\n\ndef count_data_items(filenames):\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n return np.sum(n)\n\nn_healthy_samples = count_data_items(HEALTHY_FILENAMES)\n\n\nprint(f'CBB {len(CBB_FILENAMES)} TFRecord files with {count_data_items(CBB_FILENAMES)} total samples')\nprint(f'CBSD {len(CBSD_FILENAMES)} TFRecord files with {count_data_items(CBSD_FILENAMES)} total samples')\nprint(f'CGM {len(CGM_FILENAMES)} TFRecord files with {count_data_items(CGM_FILENAMES)} total samples')\nprint(f'CMD {len(CMD_FILENAMES)} TFRecord files with {count_data_items(CMD_FILENAMES)} total samples')\nprint(f'Healthy {len(HEALTHY_FILENAMES)} TFRecord files with {n_healthy_samples} total samples')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations\n\nData augmentation for GANs should be done very carefully, especially for tasks similar to style transfer, if we apply transformations that can change too much the style of the data (e.g. brightness, contrast, saturation) it can cause the generator to do not efficiently learn the base style, so in this case, we are using only spatial transformations like, flips, rotates and crops."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def data_augment(image):\n p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n# p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n\n \n# # Random jitter\n# image = tf.image.resize(image, [560, 560], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n\n # Flips\n image = tf.image.random_flip_left_right(image)\n image = tf.image.random_flip_up_down(image)\n if p_spatial > .75:\n image = tf.image.transpose(image)\n # Rotates\n if p_rotate > .75:\n image = tf.image.rot90(image, k=3) # rotate 270º\n elif p_rotate > .5:\n image = tf.image.rot90(image, k=2) # rotate 180º\n elif p_rotate > .25:\n image = tf.image.rot90(image, k=1) # rotate 90º\n # Pixel-level transforms\n if p_pixel_1 >= .4:\n image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n if p_pixel_2 >= .4:\n image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n if p_pixel_3 >= .4:\n image = tf.image.random_brightness(image, max_delta=.1)\n \n# # Crops\n# if p_crop > .6: # random crop\n# crop_size = tf.random.uniform([], int(HEIGHT*.7), HEIGHT, dtype=tf.int32)\n# image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n# elif p_crop > .2: # central crop\n# if p_crop > .5:\n# image = tf.image.central_crop(image, central_fraction=.7)\n# elif p_crop > .35:\n# image = tf.image.central_crop(image, central_fraction=.8)\n# else:\n# image = tf.image.central_crop(image, central_fraction=.9)\n \n# # Train on crops\n# image = tf.image.random_crop(image, size=[HEIGHT_RESIZE, WIDTH_RESIZE, CHANNELS])\n \n image = tf.image.resize(image, [HEIGHT_RESIZE, WIDTH_RESIZE])\n \n return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Auxiliar functions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def normalize_img(img):\n img = tf.cast(img, dtype=tf.float32)\n # Map values in the range [-1, 1]\n return (img / 127.5) - 1.0\n\ndef decode_image(image):\n image = tf.image.decode_jpeg(image, channels=CHANNELS)\n image = tf.image.resize(image, [HEIGHT, WIDTH])\n image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])\n return image\n\ndef read_tfrecord(example):\n tfrecord_format = {\n 'image': tf.io.FixedLenFeature([], tf.string), \n 'target': tf.io.FixedLenFeature([], tf.int64), \n 'image_name': tf.io.FixedLenFeature([], tf.string)\n }\n example = tf.io.parse_single_example(example, tfrecord_format)\n image = decode_image(example['image'])\n return image\n\ndef load_dataset(filenames):\n dataset = tf.data.TFRecordDataset(filenames)\n dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n return dataset\n\ndef get_dataset(filenames, augment=None, repeat=True, shuffle=True, batch_size=1):\n dataset = load_dataset(filenames)\n\n if augment:\n dataset = dataset.map(augment, num_parallel_calls=AUTO)\n dataset = dataset.map(normalize_img, num_parallel_calls=AUTO)\n if repeat:\n dataset = dataset.repeat()\n if shuffle:\n dataset = dataset.shuffle(512)\n \n dataset = dataset.batch(batch_size)\n dataset = dataset.cache()\n dataset = dataset.prefetch(AUTO)\n \n return dataset\n\ndef display_samples(ds, n_samples):\n ds_iter = iter(ds)\n for n_sample in range(n_samples):\n example_sample = next(ds_iter)\n plt.subplot(121)\n plt.imshow(example_sample[0] * 0.5 + 0.5)\n plt.axis('off')\n plt.show()\n \ndef display_generated_samples(ds, model, n_samples):\n ds_iter = iter(ds)\n for n_sample in range(n_samples):\n example_sample = next(ds_iter)\n generated_sample = model.predict(example_sample)\n \n f = plt.figure(figsize=(12, 12))\n \n plt.subplot(121)\n plt.title('Input image')\n plt.imshow(example_sample[0] * 0.5 + 0.5)\n plt.axis('off')\n \n plt.subplot(122)\n plt.title('Generated image')\n plt.imshow(generated_sample[0] * 0.5 + 0.5)\n plt.axis('off')\n plt.show()\n \ndef evaluate_cycle(ds, generator_a, generator_b, n_samples=1):\n fig, axes = plt.subplots(n_samples, 3, figsize=(22, (n_samples*6)))\n axes = axes.flatten()\n \n ds_iter = iter(ds)\n for n_sample in range(n_samples):\n idx = n_sample*3\n example_sample = next(ds_iter)\n generated_a_sample = generator_a.predict(example_sample)\n generated_b_sample = generator_b.predict(generated_a_sample)\n \n axes[idx].set_title('Input image', fontsize=18)\n axes[idx].imshow(example_sample[0] * 0.5 + 0.5)\n axes[idx].axis('off')\n \n axes[idx+1].set_title('Generated image', fontsize=18)\n axes[idx+1].imshow(generated_a_sample[0] * 0.5 + 0.5)\n axes[idx+1].axis('off')\n \n axes[idx+2].set_title('Cycled image', fontsize=18)\n axes[idx+2].imshow(generated_b_sample[0] * 0.5 + 0.5)\n axes[idx+2].axis('off')\n \n plt.show()\n\ndef create_gif(images_path, gif_path):\n images = []\n filenames = glob.glob(images_path)\n filenames.sort(key=lambda x: int(''.join(filter(str.isdigit, x))))\n for epoch, filename in enumerate(filenames):\n img = PIL.ImageDraw.Image.open(filename)\n ImageDraw.Draw(img).text((0, 0), # Coordinates\n f'Epoch {epoch+1}')\n images.append(img)\n imageio.mimsave(gif_path, images, fps=2) # Save gif\n \ndef predict_and_save(input_ds, generator_model, output_path):\n i = 1\n for img in input_ds:\n prediction = generator_model(img, training=False)[0].numpy() # make predition\n prediction = (prediction * 127.5 + 127.5).astype(np.uint8) # re-scale\n im = PIL.Image.fromarray(prediction)\n im.save(f'{output_path}{str(i)}.jpg')\n i += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Auxiliar functions (model)\n\nHere we the building blocks of our models:\n- Encoder block: Apply convolutional filters while also reducing data resolution and increasing features.\n- Decoder block: Apply convolutional filters while also increasing data resolution and decreasing features.\n- Transformer block: Apply convolutional filters to find relevant data patterns and keeps features constant."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"conv_initializer = tf.random_normal_initializer(mean=0.0, stddev=0.02)\ngamma_initializer = tf.keras.initializers.RandomNormal(mean=0.0, stddev=0.02)\n \ndef encoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, activation=L.ReLU(), name='block_x'):\n block = L.Conv2D(filters, size, \n strides=strides, \n padding='same', \n use_bias=False, \n kernel_initializer=conv_initializer, \n name=f'encoder_{name}')(input_layer)\n\n if apply_instancenorm:\n block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer)(block)\n \n block = activation(block)\n\n return block\n\ndef transformer_block(input_layer, size=3, strides=1, name='block_x'):\n filters = input_layer.shape[-1]\n \n block = L.Conv2D(filters, size, strides=strides, padding='same', use_bias=False, \n kernel_initializer=conv_initializer, name=f'transformer_{name}_1')(input_layer)\n# block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer)(block)\n block = L.ReLU()(block)\n \n block = L.Conv2D(filters, size, strides=strides, padding='same', use_bias=False, \n kernel_initializer=conv_initializer, name=f'transformer_{name}_2')(block)\n# block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer)(block)\n \n block = L.Add()([block, input_layer])\n\n return block\n\ndef decoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, name='block_x'):\n block = L.Conv2DTranspose(filters, size, \n strides=strides, \n padding='same', \n use_bias=False, \n kernel_initializer=conv_initializer, \n name=f'decoder_{name}')(input_layer)\n\n if apply_instancenorm:\n block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer)(block)\n\n block = L.ReLU()(block)\n \n return block\n\n# Resized convolution\ndef decoder_rc_block(input_layer, filters, size=3, strides=1, apply_instancenorm=True, name='block_x'):\n block = tf.image.resize(images=input_layer, method='bilinear', \n size=(input_layer.shape[1]*2, input_layer.shape[2]*2))\n \n# block = tf.pad(block, [[0, 0], [1, 1], [1, 1], [0, 0]], \"SYMMETRIC\") # Works only with GPU\n# block = L.Conv2D(filters, size, strides=strides, padding='valid', use_bias=False, # Works only with GPU\n block = L.Conv2D(filters, size, \n strides=strides, \n padding='same', \n use_bias=False, \n kernel_initializer=conv_initializer, \n name=f'decoder_{name}')(block)\n\n if apply_instancenorm:\n block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer)(block)\n\n block = L.ReLU()(block)\n \n return block","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generator model\n\nThe `generator` is responsible for generating images from a specific domain. `CycleGAN` architecture has two generators, in this context for example we can take one `generator` that will take `Healthy` images and generate `CBB` images, and the other `generator` will take `CBB` images and generate `Healthy` images.\n\nBellow, we have the architecture of the original `CycleGAN` `generator`, ours have some changes to improve performance on this task.\n\n
"},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"def generator_fn(height=HEIGHT, width=WIDTH, channels=CHANNELS, transformer_blocks=TRANSFORMER_BLOCKS):\n OUTPUT_CHANNELS = 3\n inputs = L.Input(shape=[height, width, channels], name='input_image')\n\n # Encoder\n enc_1 = encoder_block(inputs, 64, 7, 1, apply_instancenorm=False, activation=L.ReLU(), name='block_1') # (bs, 256, 256, 64)\n enc_2 = encoder_block(enc_1, 128, 3, 2, apply_instancenorm=True, activation=L.ReLU(), name='block_2') # (bs, 128, 128, 128)\n enc_3 = encoder_block(enc_2, 256, 3, 2, apply_instancenorm=True, activation=L.ReLU(), name='block_3') # (bs, 64, 64, 256)\n \n # Transformer\n x = enc_3\n for n in range(transformer_blocks):\n x = transformer_block(x, 3, 1, name=f'block_{n+1}') # (bs, 64, 64, 256)\n\n # Decoder\n x_skip = L.Concatenate(name='enc_dec_skip_1')([x, enc_3]) # encoder - decoder skip connection\n \n dec_1 = decoder_block(x_skip, 128, 3, 2, apply_instancenorm=True, name='block_1') # (bs, 128, 128, 128)\n x_skip = L.Concatenate(name='enc_dec_skip_2')([dec_1, enc_2]) # encoder - decoder skip connection\n \n dec_2 = decoder_block(x_skip, 64, 3, 2, apply_instancenorm=True, name='block_2') # (bs, 256, 256, 64)\n x_skip = L.Concatenate(name='enc_dec_skip_3')([dec_2, enc_1]) # encoder - decoder skip connection\n\n outputs = last = L.Conv2D(OUTPUT_CHANNELS, 7, \n strides=1, padding='same', \n kernel_initializer=conv_initializer, \n use_bias=False, \n activation='tanh', \n name='decoder_output_block')(x_skip) # (bs, 256, 256, 3)\n\n generator = Model(inputs, outputs)\n \n return generator\n\nsample_generator = generator_fn()\nsample_generator.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Discriminator model\n\n\nThe `discriminator` is responsible for differentiating real images from images that have been generated by a `generator` model.\n\nBellow, we have the architecture of the original `CycleGAN` `discriminator`, again, ours have some changes to improve performance on this task.\n\n
"},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"def discriminator_fn(height=HEIGHT, width=WIDTH, channels=CHANNELS):\n inputs = L.Input(shape=[height, width, channels], name='input_image')\n #inputs_patch = L.experimental.preprocessing.RandomCrop(height=70, width=70, name='input_image_patch')(inputs) # Works only with GPU\n\n# # Encoder \n# x = encoder_block(inputs, 64, 4, 2, apply_instancenorm=False, activation=L.LeakyReLU(0.2), name='block_1') # (bs, 128, 128, 64)\n# x = encoder_block(x, 128, 4, 2, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_2') # (bs, 64, 64, 128)\n# x = encoder_block(x, 256, 4, 2, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_3') # (bs, 32, 32, 256)\n# x = encoder_block(x, 512, 4, 1, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_4') # (bs, 32, 32, 512)\n \n # Using pre-trained model\n base_model = applications.MobileNetV2(weights='imagenet', include_top=False)\n x = base_model(inputs)\n \n outputs = L.Conv2D(1, 4, strides=1, padding='valid', kernel_initializer=conv_initializer)(x) # (bs, 29, 29, 1)\n \n discriminator = Model(inputs, outputs)\n \n return discriminator\n\n\nsample_discriminator = discriminator_fn()\nsample_discriminator.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build model (CycleGAN)"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class CycleGan(Model):\n def __init__(\n self,\n first_domain_generator,\n second_domain_generator,\n first_domain_discriminator,\n second_domain_discriminator,\n first_domain_name='first_domain',\n second_domain_name='second_domain',\n lambda_cycle=10,\n ):\n super(CycleGan, self).__init__()\n self.f_gen = first_domain_generator\n self.s_gen = second_domain_generator\n self.f_disc = first_domain_discriminator\n self.s_disc = second_domain_discriminator\n self.first_domain_name = first_domain_name\n self.second_domain_name = second_domain_name\n self.lambda_cycle = lambda_cycle\n \n def compile(\n self,\n f_gen_optimizer,\n s_gen_optimizer,\n f_disc_optimizer,\n s_disc_optimizer,\n gen_loss_fn,\n disc_loss_fn,\n cycle_loss_fn,\n identity_loss_fn\n ):\n super(CycleGan, self).compile()\n self.f_gen_optimizer = f_gen_optimizer\n self.s_gen_optimizer = s_gen_optimizer\n self.f_disc_optimizer = f_disc_optimizer\n self.s_disc_optimizer = s_disc_optimizer\n self.gen_loss_fn = gen_loss_fn\n self.disc_loss_fn = disc_loss_fn\n self.cycle_loss_fn = cycle_loss_fn\n self.identity_loss_fn = identity_loss_fn\n \n def train_step(self, batch_data):\n real_first_domain, real_second_domain = batch_data\n \n with tf.GradientTape(persistent=True) as tape:\n # second_domain to first_domain back to second_domain\n fake_first_domain = self.f_gen(real_second_domain, training=True)\n cycled_second_domain = self.s_gen(fake_first_domain, training=True)\n\n # first_domain to second_domain back to first_domain\n fake_second_domain = self.s_gen(real_first_domain, training=True)\n cycled_first_domain = self.f_gen(fake_second_domain, training=True)\n\n # generating itself\n same_first_domain = self.f_gen(real_first_domain, training=True)\n same_second_domain = self.s_gen(real_second_domain, training=True)\n\n # discriminator used to check, inputing real images\n disc_real_first_domain = self.f_disc(real_first_domain, training=True)\n disc_real_second_domain = self.s_disc(real_second_domain, training=True)\n\n # discriminator used to check, inputing fake images\n disc_fake_first_domain = self.f_disc(fake_first_domain, training=True)\n disc_fake_second_domain = self.s_disc(fake_second_domain, training=True)\n\n # evaluates generator loss\n first_domain_gen_loss = self.gen_loss_fn(disc_fake_first_domain)\n second_domain_gen_loss = self.gen_loss_fn(disc_fake_second_domain)\n\n # evaluates total cycle consistency loss\n total_cycle_loss = self.cycle_loss_fn(real_first_domain, cycled_first_domain, self.lambda_cycle) + self.cycle_loss_fn(real_second_domain, cycled_second_domain, self.lambda_cycle)\n\n # evaluates total generator loss\n total_first_domain_gen_loss = first_domain_gen_loss + total_cycle_loss + self.identity_loss_fn(real_first_domain, same_first_domain, self.lambda_cycle)\n total_second_domain_gen_loss = second_domain_gen_loss + total_cycle_loss + self.identity_loss_fn(real_second_domain, same_second_domain, self.lambda_cycle)\n\n # evaluates discriminator loss\n first_domain_disc_loss = self.disc_loss_fn(disc_real_first_domain, disc_fake_first_domain)\n second_domain_disc_loss = self.disc_loss_fn(disc_real_second_domain, disc_fake_second_domain)\n\n # Calculate the gradients for generator and discriminator\n first_domain_generator_gradients = tape.gradient(total_first_domain_gen_loss,\n self.f_gen.trainable_variables)\n second_domain_generator_gradients = tape.gradient(total_second_domain_gen_loss,\n self.s_gen.trainable_variables)\n\n first_domain_discriminator_gradients = tape.gradient(first_domain_disc_loss,\n self.f_disc.trainable_variables)\n second_domain_discriminator_gradients = tape.gradient(second_domain_disc_loss,\n self.s_disc.trainable_variables)\n\n # Apply the gradients to the optimizer\n self.f_gen_optimizer.apply_gradients(zip(first_domain_generator_gradients,\n self.f_gen.trainable_variables))\n\n self.s_gen_optimizer.apply_gradients(zip(second_domain_generator_gradients,\n self.s_gen.trainable_variables))\n\n self.f_disc_optimizer.apply_gradients(zip(first_domain_discriminator_gradients,\n self.f_disc.trainable_variables))\n\n self.s_disc_optimizer.apply_gradients(zip(second_domain_discriminator_gradients,\n self.s_disc.trainable_variables))\n \n return {f'{self.first_domain_name}_gen_loss': total_first_domain_gen_loss,\n f'{self.second_domain_name}_gen_loss': total_second_domain_gen_loss,\n f'{self.first_domain_name}_disc_loss': first_domain_disc_loss,\n f'{self.second_domain_name}_disc_loss': second_domain_disc_loss\n }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss functions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"with strategy.scope():\n # Discriminator loss {0: fake, 1: real} (The discriminator loss outputs the average of the real and generated loss)\n def discriminator_loss(real, generated):\n real_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.ones_like(real), real)\n generated_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.zeros_like(generated), generated)\n total_disc_loss = real_loss + generated_loss\n return total_disc_loss * 0.5\n \n # Generator loss\n def generator_loss(generated):\n return losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE)(tf.ones_like(generated), generated)\n \n # Cycle consistency loss (measures if original image and the twice transformed image to be similar to one another)\n with strategy.scope():\n def calc_cycle_loss(real_image, cycled_image, LAMBDA):\n loss = tf.reduce_mean(tf.abs(real_image - cycled_image))\n return loss\n# return LAMBDA * loss\n\n # Identity loss (compares the image with its generator (i.e. Healthy with CBB generator))\n with strategy.scope():\n def identity_loss(real_image, same_image, LAMBDA):\n loss = tf.reduce_mean(tf.abs(real_image - same_image))\n return loss\n# return LAMBDA * 0.5 * loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create datasets"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Create dataset\n## Single class datsets\nhealthy_ds = get_dataset(HEALTHY_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)\ncbb_ds = get_dataset(CBB_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)\ncbsd_ds = get_dataset(CBSD_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)\ncgm_ds = get_dataset(CGM_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)\ncmd_ds = get_dataset(CMD_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)\n\n## Joint datasets\ncbb_healthy_ds = tf.data.Dataset.zip((cbb_ds, healthy_ds))\ncbsd_healthy_ds = tf.data.Dataset.zip((cbsd_ds, healthy_ds))\ncgm_healthy_ds = tf.data.Dataset.zip((cgm_ds, healthy_ds))\ncmd_healthy_ds = tf.data.Dataset.zip((cmd_ds, healthy_ds))\n\n## Eval datasets\nhealthy_ds_eval = get_dataset(HEALTHY_FILENAMES, repeat=False, shuffle=False, batch_size=1)\ncbb_ds_eval = get_dataset(CBB_FILENAMES, repeat=False, shuffle=False, batch_size=1)\ncbsd_ds_eval = get_dataset(CBSD_FILENAMES, repeat=False, shuffle=False, batch_size=1)\ncgm_ds_eval = get_dataset(CGM_FILENAMES, repeat=False, shuffle=False, batch_size=1)\ncmd_ds_eval = get_dataset(CMD_FILENAMES, repeat=False, shuffle=False, batch_size=1)\n\n# Callbacks\nclass GANMonitor(Callback):\n \"\"\"A callback to generate and save images after each epoch\"\"\"\n\n def __init__(self, generator, output_path, input_ds=healthy_ds_eval, num_img=1):\n self.num_img = num_img\n self.input_ds = input_ds\n self.generator = generator\n self.output_path = output_path\n # Create directories to save the generate images\n if not os.path.exists(self.output_path):\n os.makedirs(self.output_path)\n\n def on_epoch_end(self, epoch, logs=None):\n for i, img in enumerate(self.input_ds.take(self.num_img)):\n prediction = self.generator(img, training=False)[0].numpy()\n prediction = (prediction * 127.5 + 127.5).astype(np.uint8)\n prediction = PIL.Image.fromarray(prediction)\n prediction.save(f'{self.output_path}/generated_{i}_{epoch+1}.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CBB generator training"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\nK.clear_session()\n\nwith strategy.scope():\n # Create generators\n healthy_cbb_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n cbb_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n\n healthy_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n cbb_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n\n # Create discriminators\n healthy_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n cbb_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n\n healthy_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n cbb_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n\n # Create GAN\n gan_model = CycleGan(cbb_generator, healthy_cbb_generator, \n cbb_discriminator, healthy_discriminator, \n 'cbb', 'healthy')\n \n\n gan_model.compile(f_gen_optimizer=cbb_generator_optimizer,\n s_gen_optimizer=healthy_generator_optimizer,\n f_disc_optimizer=cbb_discriminator_optimizer,\n s_disc_optimizer=healthy_discriminator_optimizer,\n gen_loss_fn=generator_loss,\n disc_loss_fn=discriminator_loss,\n cycle_loss_fn=calc_cycle_loss,\n identity_loss_fn=identity_loss)\n\n\nhistory = gan_model.fit(cbb_healthy_ds, \n epochs=EPOCHS, \n batch_size=BATCH_SIZE,\n callbacks=[GANMonitor(cbb_generator, 'cbb')], \n steps_per_epoch=(n_healthy_samples//BATCH_SIZE), \n verbose=2).history\n\n# Output models\nhealthy_cbb_generator.save('healthy_cbb_generator.h5')\ncbb_generator.save('cbb_generator.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CBSD generator training"},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\nK.clear_session()\n\nwith strategy.scope():\n # Create generators\n healthy_cbsd_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n cbsd_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n\n healthy_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n cbsd_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n\n # Create discriminators\n healthy_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n cbsd_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n\n healthy_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n cbsd_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n\n # Create GAN\n gan_model = CycleGan(cbsd_generator, healthy_cbsd_generator, \n cbsd_discriminator, healthy_discriminator, \n 'cbsd', 'healthy')\n \n\n gan_model.compile(f_gen_optimizer=cbsd_generator_optimizer,\n s_gen_optimizer=healthy_generator_optimizer,\n f_disc_optimizer=cbsd_discriminator_optimizer,\n s_disc_optimizer=healthy_discriminator_optimizer,\n gen_loss_fn=generator_loss,\n disc_loss_fn=discriminator_loss,\n cycle_loss_fn=calc_cycle_loss,\n identity_loss_fn=identity_loss)\n \nhistory = gan_model.fit(cbsd_healthy_ds, \n epochs=EPOCHS, \n batch_size=BATCH_SIZE,\n callbacks=[GANMonitor(cbsd_generator, 'cbsd')], \n steps_per_epoch=(n_healthy_samples//BATCH_SIZE), \n verbose=2).history\n\n# Output models\nhealthy_cbsd_generator.save('healthy_cbsd_generator.h5')\ncbsd_generator.save('cbsd_generator.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CGM generator training"},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\nK.clear_session()\n\nwith strategy.scope():\n # Create generators\n healthy_cgm_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n cgm_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n\n healthy_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n cgm_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n\n # Create discriminators\n healthy_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n cgm_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n\n healthy_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n cgm_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n\n # Create GAN\n gan_model = CycleGan(cgm_generator, healthy_cgm_generator, \n cgm_discriminator, healthy_discriminator, \n 'cgm', 'healthy')\n \n\n gan_model.compile(f_gen_optimizer=cgm_generator_optimizer,\n s_gen_optimizer=healthy_generator_optimizer,\n f_disc_optimizer=cgm_discriminator_optimizer,\n s_disc_optimizer=healthy_discriminator_optimizer,\n gen_loss_fn=generator_loss,\n disc_loss_fn=discriminator_loss,\n cycle_loss_fn=calc_cycle_loss,\n identity_loss_fn=identity_loss)\n \nhistory = gan_model.fit(cgm_healthy_ds, \n epochs=EPOCHS, \n batch_size=BATCH_SIZE,\n callbacks=[GANMonitor(cgm_generator, 'cgm')], \n steps_per_epoch=(n_healthy_samples//BATCH_SIZE), \n verbose=2).history\n\n# Output models\nhealthy_cgm_generator.save('healthy_cgm_generator.h5')\ncgm_generator.save('cgm_generator.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CMD generator training"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\nK.clear_session()\n\nwith strategy.scope():\n # Create generators\n healthy_cmd_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n cmd_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)\n\n healthy_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n cmd_generator_optimizer = optimizers.Adam(learning_rate=GENERATOR_LR, beta_1=0.5)\n\n # Create discriminators\n healthy_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n cmd_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)\n\n healthy_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n cmd_discriminator_optimizer = optimizers.Adam(learning_rate=DISCRIMINATOR_LR, beta_1=0.5)\n\n # Create GAN\n gan_model = CycleGan(cmd_generator, healthy_cmd_generator, \n cmd_discriminator, healthy_discriminator, \n 'cmd', 'healthy')\n \n\n gan_model.compile(f_gen_optimizer=cmd_generator_optimizer,\n s_gen_optimizer=healthy_generator_optimizer,\n f_disc_optimizer=cmd_discriminator_optimizer,\n s_disc_optimizer=healthy_discriminator_optimizer,\n gen_loss_fn=generator_loss,\n disc_loss_fn=discriminator_loss,\n cycle_loss_fn=calc_cycle_loss,\n identity_loss_fn=identity_loss)\n \nhistory = gan_model.fit(cmd_healthy_ds, \n epochs=EPOCHS, \n batch_size=BATCH_SIZE,\n callbacks=[GANMonitor(cmd_generator, 'cmd')], \n steps_per_epoch=(n_healthy_samples//BATCH_SIZE), \n verbose=2).history\n\n# Output models\nhealthy_cmd_generator.save('healthy_cmd_generator.h5')\ncmd_generator.save('cmd_generator.h5')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Load models\n## Discriminators\ncbb_generator.load_weights('cbb_generator.h5')\ncbsd_generator.load_weights('cbsd_generator.h5')\ncgm_generator.load_weights('cgm_generator.h5')\ncmd_generator.load_weights('cmd_generator.h5')\n# Generators\nhealthy_cbb_generator.load_weights('healthy_cbb_generator.h5')\nhealthy_cbsd_generator.load_weights('healthy_cbsd_generator.h5')\nhealthy_cgm_generator.load_weights('healthy_cgm_generator.h5')\nhealthy_cmd_generator.load_weights('healthy_cmd_generator.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see the generators progress at each epoch by creating a `gif` that is a generated image at each epoch.\n\n## Generated image GIFs"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Create GIFs\n# create_gif('/kaggle/working/healthy/*.png', 'healthy.gif') # Create healthy images gif\ncreate_gif('/kaggle/working/cbb/*.png', 'cbb.gif') # Create cbb images gif\ncreate_gif('/kaggle/working/cbsd/*.png', 'cbsd.gif') # Create cbsd images gif\ncreate_gif('/kaggle/working/cgm/*.png', 'cgm.gif') # Create cgm images gif\ncreate_gif('/kaggle/working/cmd/*.png', 'cmd.gif') # Create cmd images gif","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"
\n
Left \"Healthy\" to \"CBB\" right \"Healthy\" to \"CBSD\" .
\n \n \n \n \n \n
\n \n \n \n
\n
\n\n
\n
Left \"Healthy\" to \"CGM\" right \"Healthy\" to \"CMD\" .
\n \n \n \n \n \n
\n \n \n \n
\n
"},{"metadata":{},"cell_type":"markdown","source":"# Evaluating generator models\n\nHere we are going to evaluate the generator models including how good is the generator cycle, this means that we will get an image from a domain to generate an image to another domain, then use the generated image to generate back the original image domain.\n\n## Healthy (input) -> CBB (generated) -> Healthy (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"evaluate_cycle(healthy_ds_eval.take(2), cbb_generator, healthy_cbb_generator, n_samples=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Healthy (input) -> CBSD (generated) -> Healthy (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"evaluate_cycle(healthy_ds_eval.take(2), cbsd_generator, healthy_cbsd_generator, n_samples=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Healthy (input) -> CGM (generated) -> Healthy (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"evaluate_cycle(healthy_ds_eval.take(2), cgm_generator, healthy_cgm_generator, n_samples=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Healthy (input) -> CMD (generated) -> Healthy (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"evaluate_cycle(healthy_ds_eval.take(2), cmd_generator, healthy_cmd_generator, n_samples=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize predictions\n\nA common issue with images generated by GANs is that the often show some undisered artifacts, a very common on is known as \"[checkerboard artifacts](https://distill.pub/2016/deconv-checkerboard/)\", a good practice is to inspect some of the images to see its quality and if some of these undisered artifacts are present.\n\n## CBB (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(healthy_ds_eval.take(4), cbb_generator, 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CBSD (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(healthy_ds_eval.take(4), cbsd_generator, 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CGM (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(healthy_ds_eval.take(4), cgm_generator, 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CMD (generated)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(healthy_ds_eval.take(4), cmd_generator, 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Healthy (generated)\n\n### From CBB"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(cbb_ds_eval.take(3), healthy_cbb_generator, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### From CBSD"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(cbsd_ds_eval.take(3), healthy_cbsd_generator, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### From CGM"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(cgm_ds_eval.take(3), healthy_cgm_generator, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### From CMD"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"display_generated_samples(cmd_ds_eval.take(3), healthy_cmd_generator, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generating images predictions\n\n#### Not duing here because takes too much time using TPU\n\n### CBB"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# %%time\n\n# # Create folders\n# os.makedirs('../cbb_generated/') # Create folder to save generated images\n# # Generate images\n# predict_and_save(healthy_ds_eval, cbb_generator, '../cbb_generated/')\n# # Zip folders\n# shutil.make_archive('/kaggle/working/cbb_generated/', 'zip', '../cbb_generated')\n# # Count images\n# print(f\"Generated CBB samples: {len([name for name in os.listdir('../cbb_generated/') if os.path.isfile(os.path.join('../cbb_generated/', name))])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CBSD"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# %%time\n\n# Create folders\n# os.makedirs('../cbsd_generated/') # Create folder to save generated images\n# Generate images\n# predict_and_save(healthy_ds_eval, cbsd_generator, '../cbsd_generated/')\n# Zip folders\n# shutil.make_archive('/kaggle/working/cbsd_generated/', 'zip', '../cbsd_generated')\n# Count images\n# print(f\"Generated CBSD samples: {len([name for name in os.listdir('../cbsd_generated/') if os.path.isfile(os.path.join('../cbsd_generated/', name))])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CGM"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# %%time\n\n# Create folders\n# os.makedirs('../cgm_generated/') # Create folder to save generated images\n# Generate images\n# predict_and_save(healthy_ds_eval, cgm_generator, '../cgm_generated/')\n# Zip folders\n# shutil.make_archive('/kaggle/working/cgm_generated/', 'zip', '../cgm_generated')\n# Count images\n# print(f\"Generated CGM samples: {len([name for name in os.listdir('../cgm_generated/') if os.path.isfile(os.path.join('../cgm_generated/', name))])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CMD"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# %%time\n\n# Create folders\n# os.makedirs('../cmd_generated/') # Create folder to save generated images\n# Generate images\n# predict_and_save(healthy_ds_eval, cmd_generator, '../cmd_generated/')\n# Zip folders\n# shutil.make_archive('/kaggle/working/cmd_generated/', 'zip', '../cmd_generated')\n# Count images\n# print(f\"Generated CMD samples: {len([name for name in os.listdir('../cmd_generated/') if os.path.isfile(os.path.join('../cmd_generated/', name))])}\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}