{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet\n!pip install tf-explain","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport os\nimport math\nimport random\nimport itertools\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport matplotlib.cm as cm\nfrom functools import partial\nimport matplotlib.pyplot as plt\nimport efficientnet.keras as efn\nfrom datetime import datetime, date\nfrom tensorflow.keras import backend as K\nfrom sklearn.metrics import roc_curve, auc\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_recall_curve, confusion_matrix, classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import StratifiedKFold, train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global Settings\nSEED = 1\nDIM = 512\nEPOCHS = 25\nBATCH_SIZE = 32\nNUM_CLASSES = 1\nVERBOSE_LEVEL = 1\nMETA_CONTRIBUTION = 0.1\n\nLR_MAX = 1e-6\nLR_MIN = 1e-8\nLR_START = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tpu = None\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n    \nif tpu:\n    BATCH_SIZE = 128  # increase the batch size if we have a tpu\n    USE_TENSORBOARD = False # Tensorboard does not work with tpu\n\n# seed everything\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\n# get the current timestamp. This timestamp is used to save the model data with a unique name\nnow = datetime.now()\ntoday = date.today()\ncurrent_time = now.strftime(\"%H:%M:%S\")\ntimestamp = str(today) + \"_\" + str(current_time)\n\n# environment settings\nprint(\"Tensorflow version \" + tf.__version__)\nAUTOTUNE = tf.data.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here we apply some manual augmentations that cannot be done with tf.image, \n# such as shearing, zooming and translation. Rotation can be done in tf.image but only in factors of 90 degrees, \n# so we do it manually instead.\n# Source: https://www.kaggle.com/teyang/melanoma-detection-using-effnet-and-meta-data#5.-Train-and-Evaluate-Model\nROT_ = 180.0\nSHR_ = 2\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=DIM):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def color(x):\n    \"\"\"Color augmentation\n    Args:\n        x: Image\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.random_hue(x, 0.05)\n    x = tf.image.random_brightness(x, 0.1, seed=SEED)\n    x = tf.image.random_contrast(x, 0.6, 1.4, seed=SEED)\n    x = tf.image.random_saturation(x, 0.6, 1.4, seed=SEED)\n    return x\n\n\ndef flip(x):\n    \"\"\"Flip augmentation\n    Args:\n        x: Image to flip\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.random_flip_left_right(x, seed=SEED)\n    x = tf.image.random_flip_up_down(x, seed=SEED)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augment_image(image, augment=True):  \n    augmentations = [color, flip, transform] \n    if augment:\n        # Data augmentation\n        for f in augmentations:\n            if random.randint(1, 10) <= 5:\n                image = f(image)     \n                \n    image = tf.image.central_crop(image, DIM*0.95 / DIM)\n    image = tf.image.resize(image, [DIM, DIM])\n    image = tf.reshape(image, [DIM, DIM, 3]) \n                \n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The malignant-v2-768x768 dataset includes all malginant images from 2020, 2019, 2018 and 2017 competition as well as data directly from ISIC.\n* Source: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/169139\n* The first 15 TFRecords contain the malignant images from this years 2020 comp. There are 584 malignant images.\n* The next 15 TFRecords contain 580 images downloaded from ISIC's online gallery and are not included in the competition.\n* The next 15 even numbered TFRecords contain the malignant images from 2018 2017 comp data.\n* The next 15 odd numbered TFRecords contain the malignant images from 2019 new portion comp data. \n* Note: The 2019 Data may decrease the model performance. More Infos here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/168028 "},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH_MALIGNANT = KaggleDatasets().get_gcs_path('malignant-v2-768x768')\nGCS_PATH_BENIGN = KaggleDatasets().get_gcs_path('benign-melanoma-tfrecords-full')\nGCS_PATH_TEST = KaggleDatasets().get_gcs_path('melanoma-test')\n\nMALIGNANT_FILES = tf.io.gfile.glob(GCS_PATH_MALIGNANT + '/*.tfrec')\nBENIGN_FILES = tf.io.gfile.glob(GCS_PATH_BENIGN + '/*.tfrec')\nTEST_FILES = tf.io.gfile.glob(GCS_PATH_TEST + '/*.tfrec')\n\nprint(\"MALIGNANT TF Records\", len(MALIGNANT_FILES))\nprint(\"BENIGN TF Records\", len(BENIGN_FILES))\nprint(\"TEST TF Records\", len(TEST_FILES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_FROM_2020 = MALIGNANT_FILES[0:15]\nDATA_ISIC = MALIGNANT_FILES[15:30]\nDATA_17_18_19 = MALIGNANT_FILES[30:60]\n\n# Because 2019 may potentially decrease the model performance, we exlude it\nDATA__17_18 = []\nfor i in range(0, len(DATA_17_18_19), 2):\n    DATA__17_18.append(DATA_17_18_19[i])\n\nprint(\"DATA_FROM_2020\", len(DATA_FROM_2020))  \nprint(\"DATA_ISIC\", len(DATA_ISIC))  \nprint(\"DATA__17_18\", len(DATA__17_18)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# There are always 200 images in a benign record, but only arround 40 in a malignant record\n# So for the validation datset, we use 5 malignant files from 2020 and 1 benign file => 395 validation images\n# We try to get the same distribution of 1:10 in both sets\nTRAINING_FILENAMES = sum([DATA_FROM_2020[0:10], DATA_ISIC, DATA__17_18], BENIGN_FILES[10:140])\nVALIDATION_FILENAMES = sum([DATA_FROM_2020[10:15], BENIGN_FILES[0:10]], [])\n\nrandom.shuffle(TRAINING_FILENAMES)\nrandom.shuffle(VALIDATION_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n\nprint(\"Training images\", TRAINING_IMAGES)\nprint(\"Validation images\", count_data_items(VALIDATION_FILENAMES))\nprint(\" \")\nprint(\"Benign images in train set\", count_data_items(BENIGN_FILES[10:140]))\nprint(\"Malignant images in train set\", count_data_items(sum([DATA_FROM_2020[0:10], DATA_ISIC, DATA__17_18], [])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the class weights and the inital bias\nbenign_cases = count_data_items(BENIGN_FILES[10:140])\nmalignant_cases = count_data_items(sum([DATA_FROM_2020[0:10], DATA_ISIC, DATA__17_18], []))\n\ninitial_bias = np.log([malignant_cases/benign_cases])\nweight_for_0 = (1 / benign_cases)*(TRAINING_IMAGES)/2.0 \nweight_for_1 = (1 / malignant_cases)*(TRAINING_IMAGES)/2.0\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nprint(\" \")\nprint(class_weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalization_layer = tf.keras.layers.experimental.preprocessing.Rescaling(1./255)\nresizing_layer = tf.keras.layers.experimental.preprocessing.Resizing(DIM, DIM)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.cache() # cache ds for performance gains\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    dataset = dataset.map(lambda x, y: (normalization_layer(x), y), num_parallel_calls=AUTOTUNE) # normalize the image so the values are between 0 and 255\n    dataset = dataset.map(lambda x, y: (resizing_layer(x), y), num_parallel_calls=AUTOTUNE) # resize the images to the same height and width\n\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(files=TRAINING_FILENAMES, augment=True):\n    dataset = load_dataset(files, labeled=True)\n    if augment:\n        dataset = dataset.map(lambda x, y: (augment_image(x, augment=augment), y), num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(TRAINING_IMAGES, reshuffle_each_iteration=True)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(files=VALIDATION_FILENAMES, ordered=False, repeat=False):\n    dataset = load_dataset(files, labeled=True, ordered=ordered)\n    if repeat:\n        dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    # if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n    #    numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\n\ndef title_from_label_and_target(label, correct_label):\n    CLASSES = [0, 1]\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\n\ndef display_one_image(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2),\n                  color='red' if red else 'black', fontdict={'verticalalignment': 'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\n\ndef display_batch_of_images(databatch, predictions=None, unbatched=False):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(\n        databatch) if not unbatched else databatch\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n\n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n\n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot = (rows, cols, 1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE, FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols, FIGSIZE))\n\n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = 'Benign' if label == 0 else 'Malignant'\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        # magic formula tested to work from 1x1 to 10x10 images\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows, cols)*40+3\n        subplot = display_one_image(\n            image, title, subplot, not correct, titlesize=dynamic_titlesize)\n\n    # layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_dataset = get_training_dataset(augment=False)\nexample_dataset = example_dataset.unbatch().batch(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_batch = iter(example_dataset) \nimage_batch, label_batch = next(example_batch)\ndisplay_batch_of_images((image_batch, label_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmented_images = [augment_image(x, augment=True) for x in image_batch]\naugmented_images = [np.clip(x, 0, 1) for x in augmented_images]\nlabels = [l.numpy() for l in label_batch]\ndisplay_batch_of_images((augmented_images, labels), unbatched=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# images are in float32 format with normalized values\nfor i in range(10):\n    image = image_batch[i]\n    print(\"min:\", np.min(image), \" -  max:\", np.max(image))\n\nprint(image.dtype)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reset_model():\n    model = tf.keras.models.load_model(\"stage_0.h5\")\n    model.load_weights(\"stage_0.hdf5\")\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_parameters(lr, epochs):\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05),\n    metrics = [\n        tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n        tf.keras.metrics.AUC(name='auc'),\n    ]\n\n    return loss, metrics, optimizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compile_model(model):\n    loss, metrics, optimizer = get_model_parameters(LR_START, EPOCHS)\n    if tpu:\n        model.compile(\n            loss=loss,\n            metrics=metrics,\n            optimizer=optimizer,\n            # Reduce python overhead, and maximize the performance of your TPU\n            # Anything between 2 and `steps_per_epoch` could help here.\n            steps_per_execution=5,\n        )\n    else:\n        model.compile(\n            loss=loss,\n            metrics=metrics,\n            optimizer=optimizer,\n        )\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    output_bias = None\n    if initial_bias is not None:\n        output_bias = tf.keras.initializers.Constant(initial_bias)\n        \n    base_model = efn.EfficientNetB6(\n        include_top=False, \n        weights=\"imagenet\",\n        input_shape=[DIM,DIM,3],\n    )\n    base_model.trainable = False\n\n    model = tf.keras.models.Sequential([\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(8, activation='relu'),\n        tf.keras.layers.Dense(NUM_CLASSES, activation='sigmoid', bias_initializer=output_bias)\n    ])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clear the session - this helps when we are creating multiple models\nK.clear_session()\n\n# Creating the model in the strategy scope places the model on the TPU\nwith strategy.scope():\n    model = build_model()\n    model = compile_model(model)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initial Model Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_dataset = get_training_dataset(augment=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    training_dataset,\n    epochs=15,\n    steps_per_epoch=30,\n    class_weight=class_weight,\n    verbose=VERBOSE_LEVEL\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Full Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.framework import ops\nfrom tensorflow.python.ops import math_ops\nfrom tensorflow.python.eager import context\n\n\ndef cyclic_learning_rate(global_step,\n                         learning_rate=0.01,\n                         max_lr=0.1,\n                         step_size=20.,\n                         gamma=0.99994,\n                         mode='triangular',\n                         name=None):\n    if global_step is None:\n        raise ValueError(\"global_step is required for cyclic_learning_rate.\")\n\n    learning_rate = ops.convert_to_tensor(\n        learning_rate, name=\"learning_rate\")\n\n    dtype = learning_rate.dtype\n    global_step = math_ops.cast(global_step, dtype)\n    step_size = math_ops.cast(step_size, dtype)\n\n    def cyclic_lr():\n        \"\"\"Helper to recompute learning rate; most helpful in eager-mode.\"\"\"\n        # computing: cycle = floor( 1 + global_step / ( 2 * step_size ) )\n        double_step = math_ops.multiply(2., step_size)\n        global_div_double_step = math_ops.divide(global_step, double_step)\n        cycle = math_ops.floor(math_ops.add(1., global_div_double_step))\n        # computing: x = abs( global_step / step_size – 2 * cycle + 1 )\n        double_cycle = math_ops.multiply(2., cycle)\n        global_div_step = math_ops.divide(global_step, step_size)\n        tmp = math_ops.subtract(global_div_step, double_cycle)\n        x = math_ops.abs(math_ops.add(1., tmp))\n        # computing: clr = learning_rate + ( max_lr – learning_rate ) * max( 0, 1 - x )\n        a1 = math_ops.maximum(0., math_ops.subtract(1., x))\n        a2 = math_ops.subtract(max_lr, learning_rate)\n        clr = math_ops.multiply(a1, a2)\n        if mode == 'triangular2':\n            clr = math_ops.divide(clr, math_ops.cast(math_ops.pow(2, math_ops.cast(\n                cycle-1, tf.int32)), tf.float32))\n        if mode == 'exp_range':\n            clr = math_ops.multiply(math_ops.pow(gamma, global_step), clr)\n        return math_ops.add(clr, learning_rate, name=name)\n\n    if not context.executing_eagerly():\n        cyclic_lr = cyclic_lr()\n\n    return cyclic_lr\n\n\ndef get_lr_callback(mode, learning_rate, max_lr, step_size):\n    def lrfn(epoch):\n        return float(\n            cyclic_learning_rate(\n                epoch,\n                mode=mode,\n                learning_rate=learning_rate,\n                max_lr=max_lr,\n                step_size=step_size,\n            )().numpy()\n        )\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=2)\n    return lr_callback\n\n\ndef plot_clr(mode, learning_rate, max_lr, step_size, epochs):\n    rates = []\n    for i in range(0, epochs):\n        x = cyclic_learning_rate(\n            i,\n            mode=mode,\n            learning_rate=learning_rate,\n            max_lr=max_lr,\n            step_size=step_size,\n        )().numpy()\n        rates.append(x)\n\n    plt.xlabel('Iterations (epochs)')\n    plt.ylabel('Learning rate')\n    plt.plot(range(epochs), rates)\n    \n\ndef display_training_curves(training, validation, title, subplot):\n  ax = plt.subplot(subplot)\n  ax.plot(training)\n  ax.plot(validation)\n  ax.set_title('model '+ title)\n  ax.set_ylabel(title)\n  ax.set_xlabel('epoch')\n  ax.legend(['training', 'validation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mode='triangular2'\nstep_size=2.\nclr_callback = get_lr_callback(mode, LR_MIN, LR_MAX, step_size)\nplot_clr(mode, LR_MIN, LR_MAX, step_size, EPOCHS)\n\ncallbacks = [clr_callback]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nwith strategy.scope():\n    model.trainable = True\n    model = compile_model(model)    \n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = 200\nvalidation_steps_per_epoch = 40\n\ntraining_dataset = get_training_dataset(augment=True)\nvalidation_dataset = get_validation_dataset(repeat=True)\n    \nhistory = model.fit(\n    training_dataset,\n    epochs=EPOCHS,\n    callbacks=callbacks,\n    class_weight=class_weight,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_dataset,\n    validation_steps=validation_steps_per_epoch,\n    verbose=VERBOSE_LEVEL\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(\"model_weights.hdf5\")\nmodel.save(\"model.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_auc(t_y, p_y, timestamp):\n    \"\"\" Helper function to plot the auc curve\n\n    Parameters:\n        t_y (array): True binary labels\n        p_y (array): Target scores\n\n    Returns:\n        Null\n    \"\"\"\n    fpr, tpr, thresholds = roc_curve(t_y, p_y, pos_label=1)\n    fig, c_ax = plt.subplots(1, 1, figsize=(8, 8))\n    c_ax.plot(fpr, tpr, label='%s (AUC:%0.2f)' % ('Target', auc(fpr, tpr)))\n    c_ax.plot([0, 1], [0, 1], color='navy', lw=1, linestyle='--')\n    c_ax.legend()\n    c_ax.set_xlabel('False Positive Rate')\n    c_ax.set_ylabel('True Positive Rate')\n        \n        \ndef plot_confusion_matrix(cm, labels, timestamp):\n    \"\"\" Helper function to plot a confusion matrix\n\n        Parameters:\n            cm (confusion matrix)\n\n        Returns:\n            Null\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest')\n    plt.title('Confusion Matrix')\n    plt.colorbar()\n    tick_marks = np.arange(len(labels))\n    plt.xticks(tick_marks, labels, rotation=55)\n    plt.yticks(tick_marks, labels)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] < thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    \n\ndef plot_precision_recall_curve(labels, predictions):\n    \"\"\" Helper function to plot the precision recall curve\n\n    Parameters:\n        precision\n        recall\n\n    Returns:\n        Null\n    \"\"\"   \n    precision, recall, thresholds = precision_recall_curve(labels, predictions)\n    fig, ax = plt.subplots(figsize=(8,8))\n    ax.plot(precision, recall)\n    ax.set_xlabel('Recall')\n    ax.set_ylabel('Precision')\n        \n\ndef plot_history(history, timestamp):\n    \"\"\" Helper function to plot the history of a tensorflow model\n\n        Parameters:\n            history (history object): The history from a tf model\n            timestamp (string): The timestamp of the function execution\n\n        Returns:\n            Null\n    \"\"\"\n    f = plt.figure()\n    f.set_figwidth(15)\n\n    f.add_subplot(1, 2, 1)\n    plt.plot(history['val_loss'], label='val loss')\n    plt.plot(history['loss'], label='train loss')\n    plt.legend()\n    plt.title(\"Modell Loss\")\n\n    f.add_subplot(1, 2, 2)\n    plt.plot(history['val_accuracy'], label='val accuracy')\n    plt.plot(history['accuracy'], label='train accuracy')\n    plt.legend()\n    plt.title(\"Modell Accuracy\")\n        \n        \ndef plot_metrics(history, timestamp):\n    metrics = ['loss', 'accuracy', 'auc']\n    plt.figure(figsize=(10, 8))\n    for n, metric in enumerate(metrics):\n        name = metric.replace(\"_\", \" \").capitalize()\n        plt.subplot(3, 1, n+1)\n        plt.plot(history.epoch, history.history[metric], label='Train')\n        plt.plot(history.epoch,\n                 history.history['val_'+metric], linestyle=\"--\", label='Val')\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        plt.legend()\n\n        \ndef calc_f1(prec, recall):\n    \"\"\" Helper function to calculate the F1 Score\n\n        Parameters:\n            prec (int): precision\n            recall (int): recall\n\n        Returns:\n            f1 score (int)\n    \"\"\"\n    return 2*(prec*recall)/(prec+recall) if recall and prec else 0\n\n\ndef pred_to_binary(pred, threshold):\n    \"\"\" Helper function turn the model predictions into a binary (0,1) format\n\n    Parameters:\n        pred (float): Model prediction\n\n    Returns:\n        binary prediction (int)\n    \"\"\"\n    if pred < threshold:\n        return 0\n    else:\n        return 1\n\n    \ndef predict_on_dataset(model, dataset):\n    print(\"start predicting ...\")\n    labels = []\n    predictions = []\n\n    for image_batch, label_batch in iter(dataset):\n        labels.append(label_batch.numpy())\n        batch_predictions = model.predict(image_batch)\n        predictions.append(batch_predictions)\n\n    # flatten the lists\n    labels = [item for sublist in labels for item in sublist]\n    predictions = [item[0] for sublist in predictions for item in sublist]\n    return predictions, labels\n\n\ndef evaluate_model(model, dataset, history, timestamp):\n    predictions, labels = predict_on_dataset(model, dataset)\n\n    # calculate the precision, recall and the thresholds\n    precision, recall, thresholds = precision_recall_curve(labels, predictions)\n\n    # calculate the f1 score\n    f1score = [calc_f1(precision[i], recall[i])\n               for i in range(len(thresholds))]\n\n    # get the index from the highest f1 score\n    idx = np.argmax(f1score)\n\n    # get the precision, recall, threshold and the f1score\n    precision = round(precision[idx], 4)\n    recall = round(recall[idx], 4)\n    threshold = round(thresholds[idx], 4)\n    f1score = round(f1score[idx], 4)\n\n    print('Precision:', precision)\n    print('Recall:', recall)\n    print('Threshold:', threshold)\n    print('F1 Score:', f1score)\n\n    # create a confusion matrix\n    y_pred_binary = [pred_to_binary(x, threshold) for x in predictions]\n    cm = confusion_matrix(labels, y_pred_binary)\n\n    cm_plot_label = ['benign', 'malignant']\n    plot_confusion_matrix(cm, cm_plot_label, timestamp)\n    \n    print(\" \")\n\n    # plot model history\n    plot_metrics(history, timestamp)\n    \n    print(\" \")\n\n    # plot auc curve\n    plot_auc(labels, predictions, timestamp)\n    \n    print(\" \")\n    \n    # plot precision recall curve\n    plot_precision_recall_curve(labels, predictions)\n\n    return predictions, labels, threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_validation_dataset = get_validation_dataset(repeat=False)\npredictions, labels, threshold = evaluate_model(\n    model=model, \n    dataset=example_validation_dataset, \n    history=history,\n    timestamp=timestamp\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_names = ['Benign', 'Malignant']\ny_pred_binary = [pred_to_binary(x, threshold) for x in predictions]\nprint(classification_report(labels, y_pred_binary, target_names=target_names))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Interpretation with Grad CAM\nVisualize how parts of the image affects neural network's output by looking into the activation maps.\nSource: https://keras.io/examples/vision/grad_cam & https://arxiv.org/abs/1610.02391"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the CNN\nefficientnet_model = False\nfor layer in model.layers:\n    if layer.name == \"efficientnet-b6\":\n        efficientnet_model = layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_PATH = \"../input/malignant-v2-768x768/jpeg768/ISIC_0000296.jpg\"\nimg = tf.keras.preprocessing.image.load_img(IMAGE_PATH, target_size=(512, 512))\nimg = tf.keras.preprocessing.image.img_to_array(img) / 255\nplt.imshow(img)\norigin_img = img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the prediction for the image from the model\nprediction = model.predict(np.expand_dims(img, axis=0))\nbinary_prediction = [0 if x < 0.5 else 1 for x in prediction]\nprint(\"Prediction: \" + (\"Benign\" if binary_prediction == 0 else \"Malignant\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First, we create a model that maps the input image to the activations\n# of the last conv layer as well as the output predictions\ngrad_model = tf.keras.models.Model(\n    [efficientnet_model.inputs], [efficientnet_model.get_layer('top_conv').output, efficientnet_model.output]\n)\n\n# Then, we compute the gradient of the top predicted class for our input image\n# with respect to the activations of the last conv layer\nwith tf.GradientTape() as tape:\n    last_conv_layer_output, preds = grad_model(np.expand_dims(img, axis=0))\n    class_channel = preds[:, round(np.mean(tf.argmax(preds[0]).numpy()))]\n\n# This is the gradient of the output neuron (top predicted or chosen)\n# with regard to the output feature map of the last conv layer\ngrads = tape.gradient(class_channel, last_conv_layer_output)\n\n# This is a vector where each entry is the mean intensity of the gradient\n# over a specific feature map channel\npooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n\n# We multiply each channel in the feature map array\n# by \"how important this channel is\" with regard to the top predicted class\n# then sum all the channels to obtain the heatmap class activation\nlast_conv_layer_output = last_conv_layer_output[0]\nheatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]\nheatmap = tf.squeeze(heatmap)\n\n# For visualization purpose, we will also normalize the heatmap between 0 & 1\nheatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\nheatmap = heatmap.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display heatmap\nplt.matshow(heatmap)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the original image\no_img = tf.keras.preprocessing.image.load_img(IMAGE_PATH, target_size=(512, 512))\no_img = tf.keras.preprocessing.image.img_to_array(img) * 255\n\n# Rescale heatmap to a range 0-255\nheatmap = np.uint8(255 * heatmap)\n\n# Use jet colormap to colorize heatmap\njet = cm.get_cmap(\"jet\")\n\n# Use RGB values of the colormap\njet_colors = jet(np.arange(256))[:, :3]\njet_heatmap = jet_colors[heatmap]\n\n# Create an image with RGB colorized heatmap\njet_heatmap = tf.keras.preprocessing.image.array_to_img(jet_heatmap)\njet_heatmap = jet_heatmap.resize((512,512))\njet_heatmap = tf.keras.preprocessing.image.img_to_array(jet_heatmap)\n\n# Superimpose the heatmap on original image\nsuperimposed_img = jet_heatmap * 0.4 + o_img\nsuperimposed_img = tf.keras.preprocessing.image.array_to_img(superimposed_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize = (12, 8))\n\nax1 = fig.add_subplot(1, 2, 1)\nax1.imshow(superimposed_img)\n\nax2 = fig.add_subplot(1, 2, 2)\nax2.imshow(origin_img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Re-Training on misclassified images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Look at a handful of images and analyze them\nNUM_IMAGES = count_data_items(TRAINING_FILENAMES)\nprint(\"NUM_IMAGES\", NUM_IMAGES)\n\nmisclassified_analysis_ds = get_validation_dataset(files=TRAINING_FILENAMES, repeat=False, ordered=True)\nmisclassified_analysis_ds = misclassified_analysis_ds.unbatch().take(NUM_IMAGES).batch(BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"misclassified_images = []\nmisclassified_images_true_labels = []\n\nfor i, (img_batch, label_batch) in enumerate(tqdm(misclassified_analysis_ds, total=round(NUM_IMAGES / BATCH_SIZE))):\n    predictions = model.predict(img_batch, verbose=0, use_multiprocessing=True)\n    binary_predictions = [0 if x < 0.5 else 1 for x in predictions]\n\n    labels = label_batch.numpy()\n\n    for i, label in enumerate(labels):\n        if label != binary_predictions[i]:\n            misclassified_images.append(img_batch[i].numpy())\n            misclassified_images_true_labels.append(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total misclassified\", len(misclassified_images_true_labels) )\nprint(\"Misclassified malignant images\", sum(misclassified_images_true_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = misclassified_images[0]\ntrue_label = misclassified_images_true_labels[0]\nprint(\"True Label: \" + (\"Benign\" if true_label == 0 else \"Malignant\"))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_label = model.predict(tf.expand_dims(img, axis=0))\nbinary_prediction = 0 if predicted_label[0][0] < 0.5 else 1\nprint(\"Predicted Label: \" + (\"Benign\" if binary_prediction == 0 else \"Malignant\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a new augmented ds with 10 times the amount of miclassified images\nROT_ = 180.0\nSHR_ = 4\nHZOOM_ = 6.0\nWZOOM_ = 6.0\nHSHIFT_ = 6.0\nWSHIFT_ = 6.0\n\n\ndef augment_image(image, augment=True):  \n    augmentations = [color, flip, transform] \n    if augment:\n        # Data augmentation\n        for f in augmentations:\n            image = f(image) \n                        \n    # Central crop the image\n    image = tf.image.central_crop(image, DIM*0.95 / DIM)\n    image = tf.image.resize(image, [DIM, DIM])\n    image = tf.reshape(image, [DIM, DIM, 3]) \n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a new augmented ds with 10 times the amount of miclassified images\nmisclassified_images_augmented = []\nmisclassified_images_true_labels_augmented = []\n\ntotal = len(misclassified_images_true_labels) * 6\npbar = tqdm(total=total)\n\ncounter = 0\nwhile len(misclassified_images_augmented) < total:\n    if counter >= len(misclassified_images):\n        counter = 0\n\n    img = misclassified_images[counter]\n    lbl = misclassified_images_true_labels[counter]\n    img_aug = augment_image(img)\n\n    misclassified_images_augmented.append(img_aug)\n    misclassified_images_true_labels_augmented.append(lbl)\n\n    counter = counter + 1\n    pbar.update(1)\n\npbar.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_img = misclassified_images_augmented[0]\nprint(\"Label:\", misclassified_images_true_labels_augmented[0])\nplt.imshow(aug_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"misclassified_ds = tf.data.Dataset.from_tensor_slices((misclassified_images_augmented, misclassified_images_true_labels_augmented))\n\ntraining_dataset = get_training_dataset(augment=True)\ntraining_dataset = training_dataset.unbatch().take(20000)\n\nvalidation_dataset = get_validation_dataset(repeat=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"combined_ds = misclassified_ds.concatenate(training_dataset)\ncombined_ds = combined_ds.cache()\ncombined_ds = combined_ds.repeat()\ncombined_ds = combined_ds.shuffle(2048, seed=SEED, reshuffle_each_iteration=True)\ncombined_ds = combined_ds.batch(BATCH_SIZE)\ncombined_ds = combined_ds.prefetch(AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_callback = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, min_lr=1e-8)\ncallbacks = [lr_callback]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = 200\nvalidation_steps_per_epoch = 40\n    \nhistory = model.fit(\n    combined_ds,\n    epochs=EPOCHS,\n    callbacks=callbacks,\n    class_weight=class_weight,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_dataset,\n    validation_steps=validation_steps_per_epoch,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_validation_dataset = get_validation_dataset(repeat=False)\npredictions, labels, threshold = evaluate_model(\n    model=model, \n    dataset=example_validation_dataset, \n    history=history,\n    timestamp=timestamp\n)\n\ny_pred_binary = [pred_to_binary(x, threshold) for x in predictions]\nprint(classification_report(labels, y_pred_binary, target_names=target_names))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Meta Learner"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ndf_test = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# getting dummy variables for gender\nsex_dummies = pd.get_dummies(df['sex'], prefix='sex', dtype=\"int\")\ndf = pd.concat([df, sex_dummies], axis=1)\n\n# getting dummy variables for anatom_site_general_challenge\nanatom_dummies = pd.get_dummies(df['anatom_site_general_challenge'], prefix='anatom', dtype=\"int\")\ndf = pd.concat([df, anatom_dummies], axis=1)\n\n# dropping not useful columns\ndf.drop(['sex','diagnosis','benign_malignant','anatom_site_general_challenge', 'image_name', 'patient_id'], axis=1, inplace=True)\n\n# replace missing age values wiht the mean age\ndf['age_approx'] = df['age_approx'].fillna(int(np.mean(df['age_approx'])))\n\n# convert age to int\ndf['age_approx'] = df['age_approx'].astype('int')\n\n# Scale age column\nscaler = StandardScaler()\ndf[['age_approx']] = scaler.fit_transform(df[['age_approx']])\n\ndf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_columns = ['age_approx', 'sex_female', 'sex_male', 'anatom_head/neck',\n       'anatom_lower extremity', 'anatom_oral/genital', 'anatom_palms/soles',\n       'anatom_torso', 'anatom_upper extremity']\n\ntarget_columns = ['target']\n\ndf_train, df_test = train_test_split(df, test_size=0.20, random_state=SEED)\n\nx_train = df_train[feature_columns]\ny_train = df_train[target_columns]\n\nx_test = df_test[feature_columns]\ny_test = df_test[target_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Source: https://www.kaggle.com/teyang/melanoma-detection-using-effnet-and-meta-data\nmeta_model = RandomForestClassifier(\n    n_estimators=5000, \n    max_depth=5, \n    class_weight='balanced',\n    n_jobs=-1, \n    random_state=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold_no = 1\nkf = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nfor train_indexes, test_index in kf.split(x_train, y_train):    \n    \n    x_train_fold = x_train.iloc[train_indexes]\n    y_train_fold = list(y_train.iloc[train_indexes].loc[:,'target'])\n    y_train_fold = [int(x) for x in y_train_fold]\n\n    x_test_fold = x_train.iloc[test_index]\n    y_test_fold = list(y_train.iloc[test_index].loc[:,'target'])\n    y_test_fold = [int(x) for x in y_test_fold]\n\n\n    meta_model.fit(x_train_fold, y_train_fold)\n    predictions = meta_model.predict(x_test_fold)\n\n    print('Fold',str(fold_no), 'roc_auc_score:', roc_auc_score(y_test_fold, predictions))\n    fold_no += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare test df\ndf = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\n# getting dummy variables for gender\nsex_dummies = pd.get_dummies(df['sex'], prefix='sex', dtype=\"int\")\ndf = pd.concat([df, sex_dummies], axis=1)\n\n# getting dummy variables for anatom_site_general_challenge\nanatom_dummies = pd.get_dummies(df['anatom_site_general_challenge'], prefix='anatom', dtype=\"int\")\ndf = pd.concat([df, anatom_dummies], axis=1)\n\n# dropping not useful columns\ndf.drop(['sex','anatom_site_general_challenge', 'image_name', 'patient_id'], axis=1, inplace=True)\n\n# replace missing age values wiht the mean age\ndf['age_approx'] = df['age_approx'].fillna(int(np.mean(df['age_approx'])))\n\n# convert age to int\ndf['age_approx'] = df['age_approx'].astype('int')\n\n# Scale age column\nscaler = StandardScaler()\ndf[['age_approx']] = scaler.fit_transform(df[['age_approx']])\n\n\ndf_test = df[feature_columns]\nmeta_model_predictions = meta_model.predict_proba(df_test)\nmeta_model_predictions[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\nNUM_TEST_IMAGES = count_data_items(TEST_FILES)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds, steps=math.ceil(len(test_ids) / BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\npred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(pred_df.target,bins=25)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"combined_predictions = []\ndf = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\n\nfor index, row in pred_df.iterrows():\n    image_name = row['image_name']\n    target = row['target']\n    idx_meta_prediction = df.index[df['image_name'] == image_name].values[0]    \n    meta_prediction = meta_model_predictions[idx_meta_prediction][1]\n\n    combined_prediction = float(target) + (meta_prediction * META_CONTRIBUTION)\n    combined_predictions.append(combined_prediction)\n\npred_df_combined = pd.DataFrame({'image_name': test_ids, 'target': combined_predictions})\npred_df_combined.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(pred_df_combined.target,bins=25)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_combined.to_csv(\"./submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}