{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"library(keras)\nlibrary(tfdatasets)\nlibrary(tidyverse)\nlibrary(rsample)\nlibrary(reticulate)\nlibrary(tensorflow)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conv2d_block <- function(inputs, use_batch_norm = TRUE, dropout = 0.3,\n                         filters = 16, kernel_size = c(3, 3), activation = \"relu\",\n                         kernel_initializer = \"he_normal\", padding = \"same\") {\n\n  x <- keras::layer_conv_2d(\n    inputs,\n    filters = filters,\n    kernel_size = kernel_size,\n    activation = activation,\n    kernel_initializer = kernel_initializer,\n    padding = padding\n  )\n\n  if (use_batch_norm) {\n    x <- keras::layer_batch_normalization(x)\n  }\n\n  if (dropout > 0) {\n    x <- keras::layer_dropout(x, rate = dropout)\n  }\n\n  x <- keras::layer_conv_2d(\n    x,\n    filters = filters,\n    kernel_size = kernel_size,\n    activation = activation,\n    kernel_initializer = kernel_initializer,\n    padding = padding\n  )\n\n  if (use_batch_norm) {\n    x <- keras::layer_batch_normalization(x)\n  }\n\n  x\n}\n\n#' U-Net: Convolutional Networks for Biomedical Image Segmentation\n#'\n#' @param input_shape Dimensionality of the input (integer) not including the\n#'   samples axis. Must be length 3 numeric vector.\n#' @param num_classes Number of classes.\n#' @param dropout Dropout rate applied between downsampling and upsampling phases.\n#' @param filters Number of filters of the first convolution.\n#' @param num_layers Number of downsizing blocks in the encoder.\n#' @param  output_activation Activation in the output layer.\n#'\n#' @export\nunet <- function(input_shape, num_classes = 1, dropout = 0.5, filters = 64,\n                 num_layers = 4, output_activation = \"sigmoid\") {\n\n\n  input <- keras::layer_input(shape = input_shape)\n\n  x <- input\n  down_layers <- list()\n\n  for (i in seq_len(num_layers)) {\n\n    x <- conv2d_block(\n      inputs = x,\n      filters = filters,\n      use_batch_norm = FALSE,\n      dropout = 0,\n      padding = \"same\"\n    )\n\n    down_layers[[i]] <- x\n\n    x <- keras::layer_max_pooling_2d(x, pool_size = c(2,2), strides = c(2,2))\n\n    filters <- filters * 2\n\n  }\n\n  if (dropout > 0) {\n    x <- keras::layer_dropout(x, rate = dropout)\n  }\n\n  x <- conv2d_block(\n    inputs = x,\n    filters = filters,\n    use_batch_norm = FALSE,\n    dropout = 0.0,\n    padding = 'same'\n  )\n\n  for (conv in rev(down_layers)) {\n\n    filters <- filters / 2L\n\n    x <- keras::layer_conv_2d_transpose(\n      x,\n      filters = filters,\n      kernel_size = c(2,2),\n      padding = \"same\",\n      strides = c(2,2)\n    )\n\n    x <- keras::layer_concatenate(list(conv, x))\n    x <- conv2d_block(\n      inputs = x,\n      filters = filters,\n      use_batch_norm = FALSE,\n      dropout = 0.0,\n      padding = 'same'\n    )\n\n  }\n\n  output <- keras::layer_conv_2d(\n    x,\n    filters = num_classes,\n    kernel_size = c(1,1),\n    activation = output_activation\n  )\n\n  model <- keras::keras_model(input, output)\n\n  model\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" zipt<- file.path(\"../input/carvana-image-masking-challenge/train.zip\")\n zipm<- file.path(\"../input/carvana-image-masking-challenge/train_masks.zip\")\n unzip(zipfile = zipt, exdir = getwd())\n unzip(zipfile = zipm, exdir = getwd())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true},"cell_type":"code","source":"\n\n\n\n\n\n\ndata.train<- file.path(\"./train\")\ndata.mask<- file.path(\"./train_masks\")\n\n\n\n\n\n# takes additional parameters, including number of downsizing blocks, \n# number of filters to start with, and number of classes to identify\n# see ?unet for more info\nmodel <- unet(input_shape = c(128, 128, 3))\n\n# libraries we're going to need later\n\nimages <- tibble(\n  img = list.files((data.train), full.names = TRUE),\n  mask = list.files((data.mask), full.names = TRUE)\n  ) %>% \n  sample_n(2) %>% \n  map(. %>% magick::image_read() %>% magick::image_resize(\"128x128\"))\n\nout <- magick::image_append(c(\n  magick::image_append(images$img, stack = TRUE), \n  magick::image_append(images$mask, stack = TRUE)\n  )\n)\n\n\n\ndata <- tibble(\n  img = list.files((data.train), full.names = TRUE),\n  mask = list.files((data.mask), full.names = TRUE)\n)\n\ndata <- initial_split(data, prop = 0.8)\n\n\n\n\n\n\n\ntraining_dataset <- training(data) %>%  \n  tensor_slices_dataset() %>% \n  dataset_map(~.x %>% list_modify(\n    # decode_jpeg yields a 3d tensor of shape (1280, 1918, 3)\n    img = tf$image$decode_jpeg(tf$io$read_file(.x$img)),\n    # decode_gif yields a 4d tensor of shape (1, 1280, 1918, 3),\n    # so we remove the unneeded batch dimension and all but one \n    # of the 3 (identical) channels\n    mask = tf$image$decode_gif(tf$io$read_file(.x$mask))[1,,,][,,1,drop=FALSE]\n  ))\n\n\n\n\nexample <- training_dataset %>% as_iterator() %>% iter_next()\nexample\n\n\ntraining_dataset <- training_dataset %>% \n  dataset_map(~.x %>% list_modify(\n    img = tf$image$convert_image_dtype(.x$img, dtype = tf$float32),\n    mask = tf$image$convert_image_dtype(.x$mask, dtype = tf$float32)\n  ))\n\n\nrandom_bsh <- function(img) {\n  img %>% \n    tf$image$random_brightness(max_delta = 0.3) %>% \n    tf$image$random_contrast(lower = 0.5, upper = 0.7) %>% \n    tf$image$random_saturation(lower = 0.5, upper = 0.7) %>% \n    # make sure we still are between 0 and 1\n    tf$clip_by_value(0, 1) \n}\n\ntraining_dataset <- training_dataset %>% \n  dataset_map(~.x %>% list_modify(\n    img = random_bsh(.x$img)\n  ))\n\n\n\n\nexample <- training_dataset %>% as_iterator() %>% iter_next()\nexample$img %>% as.array() %>% as.raster() %>% plot()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncreate_dataset <- function(data, train, batch_size = 32L) {\n  \n  dataset <- data %>% \n    tensor_slices_dataset() %>% \n    dataset_map(~.x %>% list_modify(\n      img = tf$image$decode_jpeg(tf$io$read_file(.x$img)),\n      mask = tf$image$decode_gif(tf$io$read_file(.x$mask))[1,,,][,,1,drop=FALSE]\n    )) %>% \n    dataset_map(~.x %>% list_modify(\n      img = tf$image$convert_image_dtype(.x$img, dtype = tf$float32),\n      mask = tf$image$convert_image_dtype(.x$mask, dtype = tf$float32)\n    )) %>% \n    dataset_map(~.x %>% list_modify(\n      img = tf$image$resize(.x$img, size = shape(128, 128)),\n      mask = tf$image$resize(.x$mask, size = shape(128, 128))\n    ))\n  \n  # data augmentation performed on training set only\n  if (train) {\n    dataset <- dataset %>% \n      dataset_map(~.x %>% list_modify(\n        img = random_bsh(.x$img)\n      )) \n  }\n  \n  # shuffling on training set only\n  if (train) {\n    dataset <- dataset %>% \n      dataset_shuffle(buffer_size = batch_size*128)\n  }\n  \n  # train in batches; batch size might need to be adapted depending on\n  # available memory\n  dataset <- dataset %>% \n    dataset_batch(batch_size)\n  \n  dataset %>% \n    # output needs to be unnamed\n    dataset_map(unname) \n}\n\ntraining_dataset <- create_dataset(training(data), train = TRUE)\nvalidation_dataset <- create_dataset(testing(data), train = FALSE)\n\nmodel <- unet(input_shape = c(128, 128, 3))\nsummary(model)\n\ndice <- custom_metric(\"dice\", function(y_true, y_pred, smooth = 1.0) {\n  y_true_f <- k_flatten(y_true)\n  y_pred_f <- k_flatten(y_pred)\n  intersection <- k_sum(y_true_f * y_pred_f)\n  (2 * intersection + smooth) / (k_sum(y_true_f) + k_sum(y_pred_f) + smooth)\n})\n\nmodel %>% compile(\n  optimizer = optimizer_rmsprop(lr = 1e-5),\n  loss = \"binary_crossentropy\",\n  metrics = list(dice, metric_binary_accuracy)\n)\n\n\nmodel %>% fit(\n  training_dataset,\n  epochs = 5, \n  validation_data = validation_dataset\n)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch <- validation_dataset %>% as_iterator() %>% iter_next()\npredictions <- predict(model, batch)\n\nimages <- tibble(\n  image = batch[[1]] %>% array_branch(1),\n  predicted_mask = predictions[,,,1] %>% array_branch(1),\n  mask = batch[[2]][,,,1]  %>% array_branch(1)\n) %>% \n  sample_n(2) %>% \n  map_depth(2, function(x) {\n    as.raster(x) %>% magick::image_read()\n  }) %>% \n  map(~do.call(c, .x))\n\n\nout <- magick::image_append(c(\n  magick::image_append(images$mask, stack = TRUE),\n  magick::image_append(images$image, stack = TRUE), \n  magick::image_append(images$predicted_mask, stack = TRUE)\n  )\n)\n\nplot(out)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"3.6.3"}},"nbformat":4,"nbformat_minor":4}