{"cells":[{"metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true},"cell_type":"code","source":"library(keras)\nlibrary(data.table)\nlibrary(reticulate)\nlibrary(tensorflow)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submi=fread('../input/siim-isic-melanoma-classification/sample_submission.csv')\ntrainTab=fread('../input/siim-isic-melanoma-classification/train.csv')\n\ntrain40=fread(\"../input/landscape/train40Features.csv\")\ntest40=fread(\"../input/landscape/test40Features.csv\")\n\ntarget=trainTab$target\n\n# trainM=fread(\"../input/landscape/trainMetrics.csv\")\n# testM=fread(\"../input/landscape/testMetrics.csv\")\n\n\n# train40=cbind(train40, trainM)\n# test40=cbind(test40, testM)\n\n\ntrain40 = array_reshape(data.matrix(train40), \n              c(nrow(train40), 40, 1), order=\"F\")\n\ntest40 = array_reshape(data.matrix(test40), \n                        c(nrow(test40), 40, 1), order=\"F\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"py_run_string(\"import numpy as np\")\npy_run_string(\"train = np.load(\\\"../input/siimisic-melanoma-resized-images/x_train_128.npy\\\", mmap_mode=\\\"r\\\")\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim = 128\ndatagen = image_data_generator(brightness_range = c(0.8,1.2),\n                               rescale=1/255, width_shift_range=0.3,\n                               height_shift_range=0.3, shear_range=0.3, \n                               zoom_range=0.3, validation_split=0.3,\n                               horizontal_flip = TRUE, rotation_range = 40)\n\nVdatagen = image_data_generator(rescale=1/255)\n\n\ngenerator <- function(rows, batch_size) {\n  function() {\n    gRows=sample(rows, size=batch_size)\n    py$grows=as.integer(gRows-1)\n    py_run_string(\"xs=train[grows,:,:,:]\")\n    genout = flow_images_from_data(x=py$xs, y=gRows, \n                                   batch_size=batch_size, generator = datagen)\n    out=generator_next(genout)\n    xx=array_reshape(train40[out[[2]],,], c(batch_size, 40,1), order=\"F\")\n    return(list(list(xx, out[[1]]), \n                target[out[[2]]]))\n  }\n}\n\nVgenerator <- function(rows, batch_size) {\n  function() {\n    gRows=sample(rows, size=batch_size)\n    py$grows=as.integer(gRows-1)\n    py_run_string(\"xs=train[grows,:,:,:]\")\n    genout = flow_images_from_data(x=py$xs, y=gRows,\n             batch_size=batch_size, generator = Vdatagen)\n    out=generator_next(genout)\n    xx=array_reshape(train40[out[[2]],,], c(batch_size, 40,1), order=\"F\")\n    return(list(list(xx, out[[1]]), \n                target[out[[2]]]))\n  }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_inp <-  layer_input(shape = c(dim, dim, 3))\n\nconv7x7 <-  img_inp %>%\n            layer_conv_2d(filters = 56, kernel_size = c(7, 7), strides = 2 , padding = \"same\",activation = \"relu\")\n\nPEPX11 <- conv7x7 %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 2 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\")\n\n\nconv1x1A <- conv7x7 %>%\n            layer_conv_2d(filters = 56, kernel_size = c(1, 1), strides = 2 , padding = \"same\", activation = \"relu\")\n\nPEPX12 <- layer_concatenate(c(conv1x1A, PEPX11)) %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX13 <- layer_concatenate(c(conv1x1A, PEPX11, PEPX12)) %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX21 <- layer_concatenate(c(conv1x1A, PEPX11, PEPX12, PEPX13)) %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 2 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\")\n\nconv1x1B <- layer_concatenate(c(conv1x1A, PEPX11, PEPX12, PEPX13)) %>%\n            layer_conv_2d(filters = 112, kernel_size = c(1, 1), strides = 2 , padding = \"same\", activation = \"relu\")\n\nPEPX22 <- layer_concatenate(c(conv1x1B, PEPX21)) %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX23 <- layer_concatenate(c(conv1x1B ,PEPX21, PEPX22)) %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 56, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX24 <- layer_concatenate(c(conv1x1B ,PEPX21, PEPX22, PEPX23)) %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 112, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX31 <- layer_concatenate(c(conv1x1B, PEPX21, PEPX22, PEPX23 ,PEPX24)) %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 2 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\")\n\nconv1x1C <- layer_concatenate(c(conv1x1B, PEPX21, PEPX22, PEPX23, PEPX24)) %>%\n            layer_conv_2d(filters = 224, kernel_size = c(1, 1), strides = 2, padding = \"same\", activation = \"relu\")\n\nPEPX32 <- layer_concatenate(c(conv1x1C, PEPX31)) %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX33 <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32)) %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX34 <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32, PEPX33)) %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX35 <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32, PEPX33, PEPX34)) %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 216, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX36 <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32, PEPX33, PEPX34, PEPX35)) %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 224, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX41 <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32, PEPX33, PEPX34, PEPX35, PEPX36)) %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 2 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\")\n\nconv1x1D <- layer_concatenate(c(conv1x1C, PEPX31, PEPX32, PEPX33, PEPX34, PEPX35, PEPX36)) %>%\n            layer_conv_2d(filters = 424, kernel_size = c(1, 1), strides = 2, padding = \"same\", activation = \"relu\")\n\nPEPX42 <- layer_concatenate(c(conv1x1D, PEPX41)) %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 424, kernel_size = c(1, 1), activation = \"relu\")\n\nPEPX43 <- layer_concatenate(c(conv1x1D, PEPX41, PEPX42)) %>%\n          layer_conv_2d(filters = 400, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 400, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_depthwise_conv_2d(kernel_size = c(3, 3), strides = 1 , padding = \"same\", activation = \"relu\") %>%\n          layer_conv_2d(filters = 400, kernel_size = c(1, 1), activation = \"relu\") %>%\n          layer_conv_2d(filters = 400, kernel_size = c(1, 1), activation = \"relu\")\n\nflatten <- layer_concatenate(c(PEPX41, PEPX42, PEPX43, conv1x1D)) %>%\n           layer_flatten()\n\nFC3 <- flatten %>%\n       layer_dense(units = 1, activation = \"sigmoid\")\n\ny <- keras_model(img_inp, FC3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ks = 3\n\n# only tabular data\n\nx <- keras_model_sequential() %>%\n  layer_conv_1d(filters = 32, kernel_size = ks, activation = \"relu\", \n                input_shape = c(40,1)) %>%\n  layer_max_pooling_1d() %>%\n  layer_conv_1d(filters = 64, kernel_size = ks, activation = \"relu\") %>%\n  layer_max_pooling_1d() %>%\n  layer_conv_1d(filters = 64, kernel_size = ks, activation = \"relu\") %>%\n  layer_flatten() %>%\n  layer_dropout(rate=0.3) %>%\n  layer_dense(units = 256, activation = \"relu\") %>%\n  layer_dense(units = 1, activation=\"sigmoid\")\n\navg = layer_average(c(x$output, y$output))\n# create and compile model\nmodel <- keras_model(inputs = c(x$inputs, y$inputs), outputs = avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"py_run_string(\"import tensorflow as tf\")\npy_run_string(\"from tensorflow.keras import backend as K\")\npy_run_string(\"def binary_focal_loss(gamma=2., alpha=.75):\n                def binary_focal_loss_fixed(y_true, y_pred):\n                  pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n                  pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n                  epsilon = K.epsilon()\n\n                  pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n                  pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n                  return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\ -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n                return binary_focal_loss_fixed\n\")\n\npy_run_string(\n\"\n# CUSTOM LEARNING SCHEUDLE\nLR_START = 1e-5\nLR_MAX = 0.000421875\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_STEP_DECAY = 0.75\n\ndef lrfn(epoch):\n  if epoch < LR_RAMPUP_EPOCHS:\n    lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n  elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n    lr = LR_MAX\n  else:\n    lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//10)\n  return lr\n\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model %>% compile(optimizer= 'adam', loss= py$binary_focal_loss(), metrics='AUC')\n\nnewFilepath=\"modelMixed.h5\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs=64\n\ntrainRows=sample(1:nrow(train40), size=nrow(train40)*0.7)\nvalRows=setdiff(1:nrow(train40), trainRows)\ntrain.gen=generator(trainRows, bs)\nval.gen=Vgenerator(valRows, bs)\n\nREduceLR = callback_learning_rate_scheduler(py$lrfn)\n\n\nESMonitor = callback_early_stopping(monitor='val_loss', \n                                    min_delta=0, patience=30,\n                                    verbose=0, mode='min',\n                                    restore_best_weights=TRUE)\n\nMCheckPoint = callback_model_checkpoint(newFilepath, save_weights_only=TRUE, \n                  monitor = \"val_loss\", mode=\"min\", verbose = 1,\n                          save_best_only = TRUE)\n\n\nNepochs=20\n\nhistory <- model %>% fit_generator(generator=train.gen, epochs = Nepochs,\n                        validation_data=val.gen,\n                        callbacks=c(REduceLR, ESMonitor, MCheckPoint),\n                        verbose = 1, steps_per_epoch = length(trainRows)/bs,\n                        validation_steps = length(valRows)/bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PredictTest <- function(rows) {\n  function() {\n    py$grows=as.integer(rows-1)\n    py_run_string(\"xs=test[grows,:,:,:]\")\n    xx=array_reshape(train40[rows,,],c(length(rows), 40,1))\n    return(list(xx, py$xs/255))\n  }\n}\n\npy_run_string(\"del train\")\npy_run_string(\"test = np.load(\\\"../input/siimisic-melanoma-resized-images/x_test_128.npy\\\", mmap_mode=\\\"r\\\")\")\n\nload_model_weights_hdf5(object=model, filepath = newFilepath)\n\nbatchsize=100\nocPred=rep(0, dim(submi)[1])\n\nnbatches= round(dim(submi)[1]/batchsize)\n\nfor(batch in 1:nbatches){\n  if(batch == nbatches) rows = 10901:10982\n  else rows=(batchsize*(batch-1)+1):(batchsize*(batch))\n  \n  print(paste('Predicting batch ', batch, 'with rows ', min(rows), 'to ', max(rows)))\n  xu=PredictTest(rows)()\n  ocPred[rows]=predict(model,xu)\n}\n\nsubmi$target=ocPred\nfwrite(submi, 'submiMixed.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"3.6.3"}},"nbformat":4,"nbformat_minor":4}