{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport time\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image as img\nimport numpy as np\nimport pandas as pd\nimport glob\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.patches as patches\nfrom tqdm import tqdm\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom keras import layers\nfrom keras.models import Model,Sequential\nfrom tensorflow import keras\nfrom keras.optimizers import SGD, Adam\nfrom keras.layers import ZeroPadding2D, Cropping2D\nfrom IPython import display\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-19T13:36:09.201824Z","iopub.execute_input":"2023-03-19T13:36:09.202304Z","iopub.status.idle":"2023-03-19T13:36:18.861462Z","shell.execute_reply.started":"2023-03-19T13:36:09.202240Z","shell.execute_reply":"2023-03-19T13:36:18.860343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# That a cycle GAN","metadata":{}},{"cell_type":"markdown","source":"![](http://iust-projects.ir/assets/images/projects/dip/14/wiki/3_5.jpg)","metadata":{}},{"cell_type":"code","source":"PREFIX = '/kaggle/input/vesuvius-challenge-ink-detection/train/1/'\nBUFFER = 30  # Buffer size in x and y direction\nZ_START = 27 # First slice in the z direction to use\nZ_DIM = 10   # Number of slices in the z direction\nRATIO = 10\nplt.imshow(img.open(PREFIX+\"ir.png\"), cmap=\"gray\")","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:18.863844Z","iopub.execute_input":"2023-03-19T13:36:18.864652Z","iopub.status.idle":"2023-03-19T13:36:22.176115Z","shell.execute_reply.started":"2023-03-19T13:36:18.864604Z","shell.execute_reply":"2023-03-19T13:36:22.174677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"mask = np.array(img.open(PREFIX+\"mask.png\").convert('1'))\nfig, (ax1, ax2) = plt.subplots(1, 2)\nax1.set_title(\"mask.png\")\nax1.imshow(mask, cmap='gray')\ninklabels = np.array(img.open(PREFIX+\"inklabels.png\"))\nax2.set_title(\"inklabels.png\")\nax2.imshow(inklabels, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:22.177172Z","iopub.execute_input":"2023-03-19T13:36:22.177529Z","iopub.status.idle":"2023-03-19T13:36:25.397212Z","shell.execute_reply.started":"2023-03-19T13:36:22.177493Z","shell.execute_reply":"2023-03-19T13:36:25.396258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Load the 3d x-ray scan, one slice at a time\nimages = [np.array(img.open(filename), dtype=np.float32)/65535.0 for filename in sorted(glob.glob(PREFIX+\"surface_volume/*.tif\"))[Z_START:Z_START+Z_DIM]]\nfig, axes = plt.subplots(1, len(images), figsize=(15, 3))\nfor image, ax in zip(images, axes):\n  ax.imshow(np.array(img.fromarray(image).resize((image.shape[1]//20, image.shape[0]//20)), dtype=np.float32), cmap='gray')\n  ax.set_xticks([]); ax.set_yticks([])\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:25.401458Z","iopub.execute_input":"2023-03-19T13:36:25.403707Z","iopub.status.idle":"2023-03-19T13:36:43.621969Z","shell.execute_reply.started":"2023-03-19T13:36:25.403670Z","shell.execute_reply":"2023-03-19T13:36:43.621029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rect = (1100, 3500, 700, 950)\nfig, ax = plt.subplots()\nax.imshow(inklabels, cmap = 'gray')\npatch = patches.Rectangle((rect[0], rect[1]), rect[2], rect[3], linewidth=2, edgecolor='r', facecolor='none')\nax.add_patch(patch)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:43.625267Z","iopub.execute_input":"2023-03-19T13:36:43.626091Z","iopub.status.idle":"2023-03-19T13:36:45.542324Z","shell.execute_reply.started":"2023-03-19T13:36:43.626049Z","shell.execute_reply":"2023-03-19T13:36:45.541261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some preprocess before start the Cycle Gan\n1. Split the .tif in 10K images\n1. Add padding at the end of the image because, like we use a model with two reducition layer, we need a size that can fit with that\n","metadata":{}},{"cell_type":"code","source":"def prepare_data (images_, labels_, size_pooling = 10, ratio = 10):\n    images_ = np.transpose(images_, [1,2,0])\n    images = cv2.resize(images_.astype(np.float64), (images_.shape[1]//size_pooling, images_.shape[0]//size_pooling))\n    labels = cv2.resize(labels_.astype(np.float64), (labels_.shape[1]//size_pooling, labels_.shape[0]//size_pooling))\n    width = images.shape[1]//ratio\n    height = images.shape[0]//ratio\n    X = np.zeros((ratio**2, height, width, images.shape[-1] ))\n    y = np.zeros((ratio**2, height, width ))\n    k = 0\n    for i in range(0, images.shape[0]-height,height ):\n        for j in range(0, images.shape[1]-width,width ):\n            #pretraitement\n            X[k] = images[i:i+height, j:j+width]\n            y[k] = labels[i:i+height, j:j+width]\n            k+=1\n    return X, y.reshape((y.shape[0], y.shape[1], y.shape[2], 1))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:45.543778Z","iopub.execute_input":"2023-03-19T13:36:45.544864Z","iopub.status.idle":"2023-03-19T13:36:45.554965Z","shell.execute_reply.started":"2023-03-19T13:36:45.544823Z","shell.execute_reply":"2023-03-19T13:36:45.553945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = prepare_data(np.array(images), inklabels)\nX = np.pad(X, ((0,0), (0,3), (0,1), (0,0)), mode='constant')\ny = np.pad(y, ((0,0), (0,3), (0,1), (0,0)), mode='constant')\nprint(X.shape, y.shape)\nWIDTH = X.shape[2]\nHEIGHT = X.shape[1]\nDIM = X.shape[3]\n","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:45.556455Z","iopub.execute_input":"2023-03-19T13:36:45.557056Z","iopub.status.idle":"2023-03-19T13:36:53.129776Z","shell.execute_reply.started":"2023-03-19T13:36:45.557015Z","shell.execute_reply":"2023-03-19T13:36:53.127712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a basic model of a cycle gan, we need to specify that : \n1. the first model take (dim = 10) and return (dim = 1) \n1. while the second model take (dim = 1) and return (dim = 10)\n* Add dropout to prevent an overfitting\n* The pool will be replace  by a conv2d with stride = 2","metadata":{}},{"cell_type":"markdown","source":"![](https://www.augustehoangduc.fr/wp-content/uploads/2021/04/u-net-architecture.png)","metadata":{}},{"cell_type":"code","source":"def downSample (num_filer,stride,dropout = False):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    model = Sequential()\n    model.add(layers.Conv2D(num_filer,(4,4),stride,padding=\"same\",use_bias=False,kernel_initializer=initializer))  \n    model.add(layers.BatchNormalization())\n    if dropout :\n        model.add(layers.Dropout(0.2))\n    model.add(layers.LeakyReLU(0.3))   \n    return model\ndef upSample(num_filter,stride,dropout = False):\n    initializer = tf.random_normal_initializer(0., 0.02)\n    model = Sequential()\n    model.add(layers.Conv2DTranspose(num_filter,(4,4),stride,padding=\"same\",use_bias=False,kernel_initializer=initializer))\n    model.add(layers.BatchNormalization())\n    if dropout :\n        model.add(layers.Dropout(0.5))\n    model.add(layers.Activation(layers.ReLU()))\n    return model\n\ndef maxPool(num_filter):\n    return downSample(num_filter,2)\n\ndef generator (forward = True):\n    if forward:\n        input = layers.Input(shape = (HEIGHT,WIDTH,DIM))\n    else :\n        input = layers.Input(shape = (HEIGHT,WIDTH,1))\n\n    first_conv1 = downSample(64,1)(input)\n    first_conv2 = downSample(64,1)(first_conv1)\n\n\n    first_conv3 = maxPool(64)(first_conv2)\n\n    # Model size = (40,31,64)\n    \n    second_conv1 = downSample(128,1)(first_conv3)\n    second_conv2 = downSample(128,1)(second_conv1)\n    second_conv3 = maxPool(128)(second_conv2)\n\n    # Model size = (20,15,128)\n    \n    third_conv1 = downSample(256,1)(second_conv3)\n    third_conv2 = downSample(256,1)(third_conv1)\n\n    up_0 = second_conv2\n#     up_0 = ZeroPadding2D(padding=((1,0),(0,0)))(up_0)\n    \n    zero_convtranspose1 = layers.concatenate([up_0,upSample(128,2,dropout=True)(third_conv2)])\n    zero_convtranspose2 = upSample(128,1)(zero_convtranspose1)\n    zero_convtranspose3 = upSample(128,1)(zero_convtranspose2)\n    \n    up_1 = first_conv2\n#     up_1 = ZeroPadding2D(padding=((2,1),(1,0)))(up_1)\n\n    first_convtranspose1 = layers.concatenate([up_1,upSample(64,2,dropout=True)(zero_convtranspose3)])\n    first_convtranspose2 = upSample(64,1)(first_convtranspose1)\n    first_convtranspose3 = upSample(64,1)(first_convtranspose2)    \n    if forward :\n        last = layers.Conv2DTranspose(1,4,1,\"same\",activation=\"sigmoid\")(first_convtranspose3)\n    else:\n        last = layers.Conv2DTranspose(DIM,4,1,\"same\",activation=\"sigmoid\")(first_convtranspose3)\n\n    return Model(input,last)\n\ndef discrim(dim):\n    input = layers.Input((HEIGHT,WIDTH,dim))\n\n    output = layers.Conv2D(64,(3,3),2,\"same\")(input)\n    output = layers.Dropout(0.2)(output)\n    output = layers.BatchNormalization()(output)\n    output = layers.Activation(layers.LeakyReLU())(output)\n    #  (40,31,64)\n    output = layers.Conv2D(128,(3,3),2,\"same\")(output)\n    output = layers.Dropout(0.2)(output)\n    output = layers.BatchNormalization()(output)\n    output = layers.Activation(layers.LeakyReLU())(output)\n    #  (20,15,128)\n    output = layers.Conv2D(256,(3,3),2,\"same\")(output)\n    output = layers.Dropout(0.2)(output)\n    output = layers.BatchNormalization()(output)\n    output = layers.Activation(layers.LeakyReLU())(output)\n    #  (10,7,256)    \n    output = layers.Flatten()(output)\n    output = layers.Dense(200,activation=layers.LeakyReLU())(output)\n    output = layers.Dense(100,activation=layers.LeakyReLU())(output)\n    output = layers.Dense(1,\"sigmoid\")(output)\n\n    return Model(input,output)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:53.131328Z","iopub.execute_input":"2023-03-19T13:36:53.131935Z","iopub.status.idle":"2023-03-19T13:36:53.151682Z","shell.execute_reply.started":"2023-03-19T13:36:53.131894Z","shell.execute_reply":"2023-03-19T13:36:53.150580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"gen_model_forward = generator()\ndiscrim_model_forward = discrim(1)\ngen_model_backward = generator(forward = False)\ndiscrim_model_backward = discrim(DIM)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:53.153180Z","iopub.execute_input":"2023-03-19T13:36:53.153759Z","iopub.status.idle":"2023-03-19T13:36:58.074808Z","shell.execute_reply.started":"2023-03-19T13:36:53.153685Z","shell.execute_reply":"2023-03-19T13:36:58.073795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discrim_model_backward.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:58.076116Z","iopub.execute_input":"2023-03-19T13:36:58.076462Z","iopub.status.idle":"2023-03-19T13:36:58.117452Z","shell.execute_reply.started":"2023-03-19T13:36:58.076425Z","shell.execute_reply":"2023-03-19T13:36:58.116597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create the loss function","metadata":{}},{"cell_type":"code","source":"def generator_loss(generated):\n        return tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(generated), generated)\ndef calc_cycle_loss(real_image, cycled_image, LAMBDA):\n        loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))\n        return LAMBDA * loss1\ndef identity_loss(real_image, same_image, LAMBDA):\n        loss = tf.reduce_mean(tf.abs(real_image - same_image))\n        return LAMBDA * 0.5 * loss\ndef discriminator_loss(real, generated):\n        real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.ones_like(real), real)\n        generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(tf.zeros_like(generated), generated)\n        total_disc_loss = real_loss + generated_loss\n        return total_disc_loss * 0.5\ngenerator_forward_optimizer = tf.keras.optimizers.Adam(1e-4)\ndiscriminator_forward_optimizer = tf.keras.optimizers.Adam(1e-5)\n\ngenerator_backward_optimizer = tf.keras.optimizers.Adam(1e-4)\ndiscriminator_backward_optimizer = tf.keras.optimizers.Adam(1e-5)\ncross_entropy = tf.keras.losses.BinaryCrossentropy()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:58.118527Z","iopub.execute_input":"2023-03-19T13:36:58.118975Z","iopub.status.idle":"2023-03-19T13:36:58.151413Z","shell.execute_reply.started":"2023-03-19T13:36:58.118936Z","shell.execute_reply":"2023-03-19T13:36:58.150370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAMBDA = 10\ndef train_step(images, label):\n    with  tf.GradientTape(persistent=True) as tape:\n        generated_forward_first = gen_model_forward(images, training=True)\n        generated_forward_second = gen_model_backward(generated_forward_first, training=True)\n        \n        generated_backward_first = gen_model_backward(label, training = True)\n        generated_backward_second = gen_model_forward(generated_backward_first, training=True)\n        \n        discrim_backward_first = discrim_model_backward(images, training = True)\n        discrim_forward_first = discrim_model_forward(label, training = True)\n        \n        discrim_backward_second = discrim_model_forward(generated_forward_first, training = True)\n        discrim_forward_second = discrim_model_backward(generated_backward_first, training = True)\n        \n        loss_cycled = calc_cycle_loss(images, generated_forward_second, LAMBDA) +  calc_cycle_loss(label, generated_backward_second, LAMBDA)\n        \n        forward_gen_loss = generator_loss(discrim_forward_second)\n        backward_gen_loss = generator_loss(discrim_backward_second)\n        \n        total_gen_forward = forward_gen_loss + loss_cycled\n        total_gen_backward = backward_gen_loss + loss_cycled\n        \n        # evaluates discriminator loss\n        forward_disc_loss = discriminator_loss(discrim_forward_first, discrim_forward_second)\n        backward_disc_loss = discriminator_loss(discrim_backward_first, discrim_backward_second)\n\n    forward_generator_gradients = tape.gradient(total_gen_forward,\n                                              gen_model_forward.trainable_variables)\n    backward_generator_gradients = tape.gradient(total_gen_backward,\n                                              gen_model_backward.trainable_variables)\n\n    forward_discriminator_gradients = tape.gradient(forward_disc_loss,\n                                                  discrim_model_forward.trainable_variables)\n    backward_discriminator_gradients = tape.gradient(backward_disc_loss,\n                                                  discrim_model_backward.trainable_variables)\n\n    # Apply the gradients to the optimizer\n    generator_forward_optimizer.apply_gradients(zip(forward_generator_gradients,\n                                             gen_model_forward.trainable_variables))\n\n    generator_backward_optimizer.apply_gradients(zip(backward_generator_gradients,\n                                             gen_model_backward.trainable_variables))\n\n    discriminator_forward_optimizer.apply_gradients(zip(forward_discriminator_gradients,\n                                              discrim_model_forward.trainable_variables))\n\n    discriminator_backward_optimizer.apply_gradients(zip(backward_discriminator_gradients,\n                                              discrim_model_backward.trainable_variables))\n    return forward_disc_loss, backward_disc_loss, forward_gen_loss, backward_gen_loss\n","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:58.153170Z","iopub.execute_input":"2023-03-19T13:36:58.153807Z","iopub.status.idle":"2023-03-19T13:36:58.167205Z","shell.execute_reply.started":"2023-03-19T13:36:58.153769Z","shell.execute_reply":"2023-03-19T13:36:58.166140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image(image, label):\n    prediction_image = np.array(gen_model_forward(image, training=True))\n    prediction_label = np.array(gen_model_backward(label, training=True))\n    print(\"needed\")\n    plt.imshow(prediction_image[0], cmap=\"gray\")\n    plt.show()\n    print(\"cycled useless\")\n    plt.imshow(prediction_label[0,:,:,0:1], cmap=\"gray\")\n    plt.show()\ndef build_image(images, labels, batch = 50):\n    res = np.zeros((labels.shape[1]*RATIO, labels.shape[2]*RATIO))\n    k = 0\n    for i in range(images.shape[0]):\n        if i%batch == 0:\n            prediction = gen_model_forward(images[i:i+batch])\n        x = i%RATIO * images.shape[2]\n        y = i//RATIO * images.shape[1]\n        res[y:y+images.shape[1], x:x+images.shape[2]] = prediction[i%batch,:,:,0]\n    return res","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:58.168529Z","iopub.execute_input":"2023-03-19T13:36:58.169274Z","iopub.status.idle":"2023-03-19T13:36:58.188818Z","shell.execute_reply.started":"2023-03-19T13:36:58.169230Z","shell.execute_reply":"2023-03-19T13:36:58.187576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = build_image(X, y, 100)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:36:58.193442Z","iopub.execute_input":"2023-03-19T13:36:58.194155Z","iopub.status.idle":"2023-03-19T13:37:05.021890Z","shell.execute_reply.started":"2023-03-19T13:36:58.194117Z","shell.execute_reply":"2023-03-19T13:37:05.020850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(a)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:37:05.023564Z","iopub.execute_input":"2023-03-19T13:37:05.023935Z","iopub.status.idle":"2023-03-19T13:37:05.290398Z","shell.execute_reply.started":"2023-03-19T13:37:05.023897Z","shell.execute_reply":"2023-03-19T13:37:05.289377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(batch,epochs):\n    for epoch in range(epochs):\n        start = time.time()\n        for i in range(0, X.shape[0]-batch, batch):\n            image_batch = X[i:i+batch]\n            label_batch = y[i:i+batch]\n            forward_disc_loss, backward_disc_loss, forward_gen_loss, backward_gen_loss = train_step(image_batch, label_batch)\n        if epoch % 10 == 0:\n            a = build_image(X, y, 500)\n            plt.imshow(a, cmap='gray')\n            plt.show()\n        \n        \n            \n  ","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:37:05.291714Z","iopub.execute_input":"2023-03-19T13:37:05.292309Z","iopub.status.idle":"2023-03-19T13:37:05.300823Z","shell.execute_reply.started":"2023-03-19T13:37:05.292258Z","shell.execute_reply":"2023-03-19T13:37:05.299569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the models","metadata":{}},{"cell_type":"code","source":"train(50, 2)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T13:37:05.302422Z","iopub.execute_input":"2023-03-19T13:37:05.303190Z","iopub.status.idle":"2023-03-19T13:37:25.261111Z","shell.execute_reply.started":"2023-03-19T13:37:05.303152Z","shell.execute_reply":"2023-03-19T13:37:25.260097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}