{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **PACKAGES**\nImport necessary packages.","execution_count":null},{"metadata":{"_uuid":"dc4c6a6c-4fb6-4251-b908-71b6a86a206c","_cell_guid":"5425c7ac-178c-4dd6-835d-e8802bfe75e5","trusted":true},"cell_type":"code","source":"!pip install tensorflow-addons==0.9.1\nimport numpy as np\nimport pandas as pd\nimport os, math\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow_addons as tfa\nfrom matplotlib import pyplot as plt\nimport matplotlib as mpl\n\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **TPU / GPU / CPU**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_strategy():\n    \n    \"\"\"Detect hardware, return appropriate distribution strategy.\"\"\"\n    \n    gpu = \"\"\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU', tpu.cluster_spec().as_dict()['worker'])\n        \n    except ValueError:\n        tpu = None\n        os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n        gpu = tf.config.list_physical_devices(\"GPU\")\n        if len(gpu) == 1:\n            print('Running on GPU', gpu)\n    \n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        tf.config.optimizer.set_jit(True)\n        print('Accelerated Linear Algebra enabled')\n        GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n        !gsutil ls $GCS_PATH\n    \n    elif len(gpu) == 1:\n        strategy = tf.distribute.OneDeviceStrategy(device = \"/gpu:0\")\n        tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n        GCS_PATH = '/kaggle/input/siim-isic-melanoma-classification/'\n    \n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n        GCS_PATH = '/kaggle/input/siim-isic-melanoma-classification/'\n    \n    print(\"REPLICAS:\", strategy.num_replicas_in_sync)\n    base_dir = '/kaggle/input/siim-isic-melanoma-classification/'\n    \n    return strategy, GCS_PATH, base_dir\n\nstrategy, GCS_PATH, base_dir = get_strategy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DATA PIPELINE\nUse TFRecords.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"training_files = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')\ntest_files = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')\n\nIMAGE_SIZE = [1024,1024]\nBATCH_SIZE = 12 * strategy.num_replicas_in_sync\n\nseed = 42\n\ndef parse_rec_train(data):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    file = tf.io.parse_single_example(data, LABELED_TFREC_FORMAT)\n    image = tf.image.decode_image(file[\"image\"])\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    image = tf.cast(image, tf.float32)\n    image = tf.keras.applications.inception_v3.preprocess_input(image)\n    target = tf.cast(file[\"target\"], tf.float32)\n    return image, target\n\nROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 3.0\nWZOOM_ = 3.0\nHSHIFT_ = 5.0\nWSHIFT_ = 5.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transform matrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\ndef transforms(image, DIM=1024):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])\n\ndef transform(image, target):\n    image = transforms(image)\n    image = tf.image.rot90(image, k = np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed = seed)\n    image = tf.image.random_flip_up_down(image, seed = seed)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image, target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# OVERSAMPLING","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nsplit = int(len(training_files) * 0.8)\ntraining_filenames = training_files[:split]\nvalidation_filenames = training_files[split:]\n\ndef get_num_of_repetition_for_example(training_example):\n    \n    _, label = training_example\n    \n    if label == 1.0:\n        num_to_repeat = 15\n    else:\n        num_to_repeat = 1\n        \n    return tf.cast(num_to_repeat, tf.int64)\n\ntrain_dataset = tf.data.TFRecordDataset(training_filenames, num_parallel_reads = AUTO).map(parse_rec_train, num_parallel_calls = AUTO)\ntrain_dataset = train_dataset.flat_map(lambda image, label: tf.data.Dataset.from_tensors((image, label)).repeat(get_num_of_repetition_for_example((image, label))))\n\nlabel_counter = Counter()\n\nfor images, labels in train_dataset:\n    label_counter.update([labels.numpy()])\n\nTRAIN_SIZE = sum([label_counter[x] for x in label_counter])\nprint(\"Number of examples in the oversampled training dataset: {}\".format(TRAIN_SIZE))\n\nprint(\"Number of positive train examples: {}\".format(label_counter[1.0])) \nprint(\"Number of negative train examples: {}\".format(label_counter[0.0]))\n\n# val_counter = Counter()\n\n# val_ds = tf.data.TFRecordDataset(validation_filenames, num_parallel_reads = AUTO).map(parse_rec_train, num_parallel_calls = AUTO)\n\n# for images, labels in val_ds:\n#     val_counter.update([labels.numpy()])\n\n# print(\"Number of positive val examples: {}\".format(val_counter[1.0])) # 187\n# print(\"Number of negative val examples: {}\".format(val_counter[0.0])) # 10158\n\n# del val_ds\n\nSTEPS_PER_EPOCH = int(np.ceil(TRAIN_SIZE / BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Define a learning rate function for the LearningRateScheduler. Also include an EarlyStopping to stop training when a monitored metric has stopped improving.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# LR_START = 0.00001\n# LR_MAX = 0.000005 * strategy.num_replicas_in_sync\n# LR_MIN = 0.00001\n# LR_RAMPUP_EPOCHS = 8\n# LR_SUSTAIN_EPOCHS = 0\n# LR_EXP_DECAY = 0.8\n\ndef scheduler(epoch):\n    if epoch < 5:\n        return 0.001\n    else:\n        return 0.001 * tf.math.exp(0.1 * (5 - epoch))\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 3e-4 * (0.70 ** (epoch/3)), verbose=True)\nes_callback = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', patience = 15, verbose = 1, mode = 'max', restore_best_weights = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # **MODEL 1 & TRAINING**\nInstantiating the model in the strategy scope creates the model on the TPU. Train the model with the initial bias.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = train_dataset.map(transform, num_parallel_calls = AUTO)\ntrain_dataset = train_dataset.repeat().shuffle(10000)\ntrain_dataset = train_dataset.batch(BATCH_SIZE).prefetch(AUTO)\n    \nval_dataset = tf.data.TFRecordDataset(validation_filenames, num_parallel_reads = AUTO).map(parse_rec_train, num_parallel_calls = AUTO)\nval_dataset = val_dataset.batch(BATCH_SIZE)\nval_dataset = val_dataset.prefetch(AUTO)\n\n\nwith strategy.scope():\n    base_model1 = tf.keras.applications.InceptionV3(weights = 'imagenet', include_top = False, pooling = 'avg', input_shape = (1024, 1024, 3))\n    \n    \n#     for layer in base_model1.layers:\n#         layer.trainable = False\n        \n    for layer in base_model1.layers:\n        layer.trainable = True\n        if isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.momentum = 0.9\n    \n    for layer in base_model1.layers[:10]:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = False\n    \n    x = base_model1.output\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     x = tf.keras.layers.Dense(1000, activation = 'relu')(x)\n    x = tf.keras.layers.Dense(3000, activation = 'relu')(x)\n#     x = tf.keras.layers.Dropout(0.25)(x)\n    predictions = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    model1 = tf.keras.Model(inputs = base_model1.input, outputs = predictions)\n    \n    model1.compile(\n        optimizer = tf.keras.optimizers.Nadam(),\n        loss = tfa.losses.SigmoidFocalCrossEntropy(alpha = 0.9, gamma = 2.0),\n#         loss = tf.keras.losses.BinaryCrossentropy(),\n        metrics = [tf.keras.metrics.AUC(), tf.keras.metrics.Precision(), tf.keras.metrics.BinaryAccuracy()]\n    )\n\n    model1.summary()\n    \n    history1 = model1.fit(train_dataset,\n                        epochs = 15,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        callbacks = [lr_callback, es_callback],\n                        validation_data = val_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **MODEL EVALUATION**\nPlot the metrics to visualize the model's performance.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mpl.rcParams['figure.figsize'] = (12, 10)\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n\nmetrics1 = ['loss', 'auc', 'precision', 'binary_accuracy']\n\ndef plot_metrics(history, metrics):\n    history.history['loss'] = [np.mean(i) for i in history.history['loss']]\n    history.history['val_loss'] = [np.mean(i) for i in history.history['val_loss']]\n    for n, metric in enumerate(metrics):\n        name = metric.replace(\"_\", \" \").upper()\n        plt.subplot(2, 2, n + 1)\n        plt.plot(history.epoch, history.history[metric], color = colors[0], label = 'Train')\n        plt.plot(history.epoch, history.history['val_' + metric], color = colors[0], linestyle = \"--\", label = 'Val')\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        plt.legend()\n\nplot_metrics(history1, metrics1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **PREDICTIONS 1**\nWe now get the prediction probabilities for our test images. Since we are splitting the dataset and iterating separately on images and ids, order matters.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_rec_test(data):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n    }\n    file = tf.io.parse_single_example(data, UNLABELED_TFREC_FORMAT)\n    image = tf.image.decode_image(file[\"image\"])\n    image = tf.cast(image, tf.float32)\n    image = tf.keras.applications.inception_v3.preprocess_input(image)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    idnum = file[\"image_name\"]\n    return image, idnum\n\nprint('Computing probabilities...')\ntest_ds = tf.data.TFRecordDataset(test_files, num_parallel_reads = AUTO)\ntest_ds = test_ds.with_options(tf.data.Options()).map(parse_rec_test, num_parallel_calls = AUTO)\ntest_ds = test_ds.batch(BATCH_SIZE).prefetch(AUTO)\ntest_images = test_ds.map(lambda image, idnum: image)\n\nprobabilities1 = model1.predict(test_images)\nprobabilities1 = probabilities1[:,0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL 2 & TRAINING","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    base_model2 = tf.keras.applications.Xception(weights = 'imagenet', include_top = False, pooling = 'max', input_shape = (1024, 1024, 3))\n    \n    for layer in base_model2.layers:\n        layer.trainable = False\n        \n    x = base_model2.output\n    x = tf.keras.layers.Dense(500, activation = 'relu')(x)\n    predictions = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    model2 = tf.keras.Model(inputs = base_model2.input, outputs = predictions)\n        \n    model2.compile(\n        optimizer = tf.keras.optimizers.RMSprop(),\n        loss = tfa.losses.SigmoidFocalCrossEntropy(alpha = 0.9, gamma = 2.0),\n        metrics = [tf.keras.metrics.AUC(), tf.keras.metrics.Precision(), tf.keras.metrics.BinaryAccuracy()]\n    )\n\n    history2 = model2.fit(train_dataset,\n                        epochs = 15,\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        callbacks = [lr_callback, es_callback],\n                        validation_data = val_dataset)\n\nmetrics2 = ['loss', 'auc_1', 'precision_1', 'binary_accuracy']\n\nplot_metrics(history2, metrics2)\n\nprobabilities2 = model2.predict(test_images)\nprobabilities2 = probabilities2[:,0]\n\n# Calculate mean of probabilities.\nprobabilities = (probabilities1 + probabilities2) / 2\nthreshold = 0.5\npredictions = (probabilities>threshold)*1\n\n# Calculate softmax and get the maximum.\n# from scipy.special import softmax\n# prob = np.column_stack((probabilities1, probabilities2))\n# prob = softmax(prob, axis = 1)\n# prob = np.mean(prob, axis = 1)\n# prob = probabilities2\n# predictions = (prob > 0.2)*1\nprint(\"Threshold: \" + threshold)\nprint(\"Number of positive predictions: \" + sum(predictions))\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(10982))).numpy().astype('U') # all in one batch\n# np.savetxt('submissionprob.csv', np.rec.fromarrays([test_ids, probabilities]), fmt=['%s', '%d'], delimiter=',', header='image_name,target', comments='')\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='image_name,target', comments='')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}