{"cells":[{"metadata":{},"cell_type":"markdown","source":"We use Keras-Tuner to tune EfficientNet model. Only image data is used. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q tensorflow==2.3.0 # Use 2.3.0 for built-in EfficientNet\n!pip install -q git+https://github.com/keras-team/keras-tuner@master # Use github head for newly added TPU support\n!pip install -q cloud-tpu-client # Needed for sync TPU version","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import random, re, math\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nprint('Tensorflow version ' + tf.__version__)\nimport kerastuner as kt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Initiate TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # Sync TPU version\n    from cloud_tpu_client import Client\n    c = Client()\n    c.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n    \n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n    \n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.data.experimental import AUTOTUNE\n\n# Configuration\nIMAGE_SIZE = [384, 384]\nEPOCHS = 15\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\n# Data access\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\n\n# training filenames directory\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\n# test filenames directory\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')\n# submission file\nsubmission = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Augmentation for TPU as KPL not supporting yet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        rot = 15. * tf.random.normal([1],dtype='float32')\n    else:\n        rot = 180. * tf.random.normal([1],dtype='float32')\n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        shr = 5. * tf.random.normal([1],dtype='float32') \n    else:\n        shr = 2. * tf.random.normal([1],dtype='float32')\n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10. \n    else:\n        h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10. \n    else:\n        w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        h_shift = 16. * tf.random.normal([1],dtype='float32') \n    else:\n        h_shift = 8. * tf.random.normal([1],dtype='float32')\n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        w_shift = 16. * tf.random.normal([1],dtype='float32') \n    else:\n        w_shift = 8. * tf.random.normal([1],dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return image, label\n\n# function to decode our images (normalize and reshape)\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    # convert image to floats in [0, 255] range\n    image = tf.cast(image, tf.float32)\n    # explicit size needed for TPU\n    image = tf.ensure_shape(image, [*IMAGE_SIZE, 3])\n    return image\n\n# this function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        # meta features\n        \"age_approx\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex\": tf.io.FixedLenFeature([], tf.int64),\n        \"anatom_site_general_challenge\": tf.io.FixedLenFeature([], tf.int64)\n        \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    # returns a dataset of (image, label)\n    return image, label\n\n# this function parse our image and also get our image_name (id) to perform predictions\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        # meta features\n        \"age_approx\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex\": tf.io.FixedLenFeature([], tf.int64),\n        \"anatom_site_general_challenge\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    # returns a dataset of (image, key)\n    return image, image_name\n    \ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTOTUNE)\n    # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order)\n    # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTOTUNE) \n    return dataset\n\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement \n    # in the next function (below), this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.7, 1.3)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    image = tf.image.random_brightness(image, 0.1)\n    \n    return image, label\n\ndef get_training_dataset(filenames, labeled = True, ordered = False):\n    dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTOTUNE)\n    dataset = dataset.map(transform, num_parallel_calls = AUTOTUNE)\n    # the training dataset must repeat for several epochs\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_validation_dataset(filenames, labeled = True, ordered = True):\n    dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    # using gpu, not enought memory to use cache\n    # dataset = dataset.cache()\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTOTUNE) \n    return dataset\n\ndef get_test_dataset(filenames, labeled = False, ordered = True):\n    dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTOTUNE) \n    return dataset\n\n# function to count how many photos we have in\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# this function parse our images and also get the target variable\ndef read_tfrecord_full(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"image_name\": tf.io.FixedLenFeature([], tf.string), \n        \"target\": tf.io.FixedLenFeature([], tf.int64), \n        # meta features\n        \"age_approx\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex\": tf.io.FixedLenFeature([], tf.int64),\n        \"anatom_site_general_challenge\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    target = tf.cast(example['target'], tf.float32)\n    # meta features\n    data = {}\n    data['age_approx'] = tf.cast(example['age_approx'], tf.int32)\n    data['sex'] = tf.cast(example['sex'], tf.int32)\n    data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)\n    return image, image_name, target, data\n\ndef load_dataset_full(filenames):        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTOTUNE)\n    # returns a dataset of (image_name, target)\n    dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTOTUNE) \n    return dataset\n\ndef get_data_full(filenames):\n    dataset = load_dataset_full(filenames)\n    dataset = dataset.map(setup_input3, num_parallel_calls = AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nTRAIN_FILES, VAL_FILES = sklearn.model_selection.train_test_split(TRAINING_FILENAMES, train_size=0.8)\ntrain_dataset = get_training_dataset(TRAIN_FILES, labeled = True, ordered = False)\nval_dataset = get_validation_dataset(VAL_FILES, labeled = True, ordered = False)\nall_dataset = get_training_dataset(TRAIN_FILES + VAL_FILES, labeled=True, ordered=False)\nK.clear_session()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x, y in train_dataset.take(1):\n    print(x.numpy().shape, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAIN_FILES)\n# use validation data for training\nNUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following is some not-yet-merged function. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras as keras\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import efficientnet\nfrom tensorflow.keras.layers.experimental import preprocessing\n\nfrom kerastuner.engine import hypermodel\n\nimport os\n\n\nEFFICIENTNET_MODELS = {'B0': efficientnet.EfficientNetB0,\n                       'B1': efficientnet.EfficientNetB1,\n                       'B2': efficientnet.EfficientNetB2,\n                       'B3': efficientnet.EfficientNetB3,\n                       'B4': efficientnet.EfficientNetB4,\n                       'B5': efficientnet.EfficientNetB5,\n                       'B6': efficientnet.EfficientNetB6,\n                       'B7': efficientnet.EfficientNetB7}\n\nEFFICIENTNET_IMG_SIZE = {'B0': 224,\n                         'B1': 240,\n                         'B2': 260,\n                         'B3': 300,\n                         'B4': 380,\n                         'B5': 456,\n                         'B6': 528,\n                         'B7': 600}\n\n\nclass HyperEfficientNet(hypermodel.HyperModel):\n    \"\"\"An EfficientNet HyperModel.\n    Models built by this HyperModel takes input image data in\n    ints [0, 255]. The output data should be one-hot encoded\n    with number of classes matching `classes`.\n      # Arguments:\n        include_top: whether to include the fully-connected\n            layer at the top of the network. Model is not\n            compiled if include_top is set to False.\n        input_shape: shape tuple, e.g. `(256, 256, 3)`.\n              Input images will be resized if different from\n              the default input size of the version of\n              efficientnet base model used.\n              One of `input_shape` or `input_tensor` must be\n              specified.\n        input_tensor: Keras tensor to use as image input for the model.\n              One of `input_shape` or `input_tensor` must be\n              specified.\n        classes: number of classes to classify images into.\n        weights: str or None. Default is 'imagenet', where the weights pre-trained\n              on imagenet will be downloaded. Otherwise the weights will be\n              loaded from the directory in 'weights', and are expected to be in\n              h5 format with naming convention '{weights}/b{n}_notop.h5' where n\n              is 0 to 7. If set to None, the weights will be initiated from scratch.\n        augmentation_model: optional Model or HyperModel for image augmentation.\n        **kwargs: Additional keyword arguments that apply to all\n            HyperModels. See `kerastuner.HyperModel`.\n    \"\"\"\n    def __init__(self,\n                 include_top=True,\n                 input_shape=None,\n                 input_tensor=None,\n                 classes=None,\n                 weights='imagenet',\n                 augmentation_model=None,\n                 **kwargs):\n        if not isinstance(augmentation_model, (hypermodel.HyperModel,\n                                               keras.Model,\n                                               type(None))):\n            raise ValueError('Keyword augmentation_model should be '\n                             'a HyperModel, a Keras Model or empty. '\n                             'Received {}.'.format(augmentation_model))\n\n        if include_top and classes is None:\n            raise ValueError('You must specify `classes` when '\n                             '`include_top=True`')\n\n        if input_shape is None and input_tensor is None:\n            raise ValueError('You must specify either `input_shape` '\n                             'or `input_tensor`.')\n\n        self.include_top = include_top\n        self.input_shape = input_shape\n        self.input_tensor = input_tensor\n        self.classes = classes\n        self.augmentation_model = augmentation_model\n        self.weights = weights\n\n        super(HyperEfficientNet, self).__init__(**kwargs)\n\n    def build(self, hp):\n\n        if self.input_tensor is not None:\n            inputs = tf.keras.utils.get_source_inputs(self.input_tensor)\n            x = self.input_tensor\n        else:\n            inputs = layers.Input(shape=self.input_shape)\n            x = inputs\n\n        if self.augmentation_model:\n            if isinstance(self.augmentation_model, hypermodel.HyperModel):\n                augmentation_model = self.augmentation_model.build(hp)\n            elif isinstance(self.augmentation_model, keras.models.Model):\n                augmentation_model = self.augmentation_model\n\n            x = augmentation_model(x)\n\n        # Select one of pre-trained EfficientNet as feature extractor\n        version = hp.Choice('version',\n                            ['B{}'.format(i) for i in range(8)],\n                            default='B0')\n        img_size = EFFICIENTNET_IMG_SIZE[version]\n\n        weights = self.weights\n        if weights and (weights != 'imagenet'):\n            weights = os.path.join(weights, version.lower())\n            weights += '_notop.h5'\n            if not os.path.isfile(weights):\n                raise ValueError('Expect path {} to include weight file; but '\n                                 'no file is found'.format(weights))\n\n        x = preprocessing.Resizing(img_size, img_size, interpolation='bilinear')(x)\n        efficientnet_model = EFFICIENTNET_MODELS[version](include_top=False,\n                                                          input_tensor=x,\n                                                          weights=weights)\n\n        # Rebuild top layers of the model.\n        x = efficientnet_model.output\n\n        pooling = hp.Choice('pooling', ['avg', 'max'], default='avg')\n        if pooling == 'avg':\n            x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n        elif pooling == 'max':\n            x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n        if self.include_top:\n            top_dropout_rate = hp.Float('top_dropout_rate',\n                                        min_value=0.2,\n                                        max_value=0.8,\n                                        default=0.2)\n            x = layers.Dropout(top_dropout_rate, name='top_dropout')(x)\n\n            x = layers.Dense(\n                self.classes, activation='sigmoid', name='probs')(x)\n\n            # compile\n            model = keras.Model(inputs, x, name='EfficientNet')\n            self._compile(model, hp)\n\n            return model\n        else:\n            return keras.Model(inputs, x, name='EfficientNet')\n\n    def _compile(self, model, hp):\n        \"\"\" Compile model using hyperparameters in hp.\n            When subclassing the hypermodel, this may\n            be overriden to change behavior of compiling.\n        \"\"\"\n        learning_rate = hp.Choice('learning_rate', [0.1, 0.01, 0.001], default=0.01)\n        optimizer = tf.keras.optimizers.SGD(\n                momentum=0.1,\n                learning_rate=learning_rate)\n\n        model.compile(\n            optimizer=optimizer,\n            loss='categorical_crossentropy',\n            metrics=['accuracy'],\n            experimental_steps_per_execution=4)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define HyperModel using built-in application\n# from kerastuner.applications.efficientnet import HyperEfficientNet\nhm = HyperEfficientNet(input_shape=[IMAGE_SIZE[0], IMAGE_SIZE[1], 3] , classes=1)\n\n\n# Optional: Restrict default hyperparameters.\n# To take effect, pass this `hp` instance when constructing tuner as `hyperparameters=hp`\nfrom kerastuner.engine.hyperparameters import HyperParameters\nhp = HyperParameters()\nhp.Choice('version', ['B0', 'B1', 'B2', 'B3', 'B4', 'B5', 'B6']) #restrict choice of EfficientNet version from B0-B7 to B0-B4\n\n# Initiate Tuner\ntuner = kt.tuners.randomsearch.RandomSearch(\n    hypermodel=hm,\n    objective=kt.Objective(\"val_auc\", direction=\"max\"),\n    max_trials=7,\n    hyperparameters=hp,\n    distribution_strategy=strategy, # This strategy's scope is used for building each model during the search.\n    directory='tuner_melanoma',\n    project_name='hyperband_efficientnet',\n    overwrite=True,\n    loss='binary_crossentropy',\n    metrics=['AUC']\n)\ntuner.search_space_summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tuner.search(train_dataset,\n             epochs=15,\n             validation_data=val_dataset,\n             steps_per_epoch=STEPS_PER_EPOCH,\n             verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tuner.get_best_models()[0]\nmodel.fit(all_dataset, epochs=40, steps_per_epoch=(NUM_VALIDATION_IMAGES + NUM_TRAINING_IMAGES)//BATCH_SIZE, callbacks=[tf.keras.callbacks.ReduceLROnPlateau(monitor='AUC')])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_test_dataset(TEST_FILENAMES)\ntargets = []\nimage_names = []\nnp_decode = np.vectorize(lambda x: x.decode('UTF-8'))\nfor image, image_name in test_dataset:\n    targets.append(model(image))\n    image_names.append(np_decode(image_name.numpy()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = np.concatenate(targets).squeeze()\nimage_names = np.concatenate(image_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(dict(\n    image_name = image_names,\n    target = targets\n))\n\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}