{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# pepi\nimport re\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\nimport random\nimport math\nimport PIL\nimport tensorflow as tf, tensorflow.keras.backend as K\nimport tensorflow as tf\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, BatchNormalization, LeakyReLU, Dropout, Input\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import SGD, RMSprop, Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.data.experimental import AUTOTUNE\nfrom tensorflow.keras.metrics import AUC, Accuracy\nfrom tensorflow.keras.losses import BinaryCrossentropy\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport pydicom \nfrom tqdm import tqdm\nimport missingno as msno \nfrom sklearn.metrics import roc_curve, roc_auc_score, auc\nimport shutil\nfrom sklearn.model_selection import train_test_split\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\n\nimport tensorflow_hub as hub\n# Load model from TFHub into KerasLayer\nmodel_url = \"https://tfhub.dev/google/bit/m-r50x1/1\"\nmodule = hub.KerasLayer(model_url)\n\nbase_dir = '/kaggle/input/siim-isic-melanoma-classification/'\nexternal_base_dir = '/kaggle/input/melanoma-external-malignant-256/'\nexternal_tfrec_base_dir = '/kaggle/input/melanoma-256x256'\n\nsns.set(style=\"darkgrid\")\nrandom.seed(42)\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Initialize notebook parameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'GPU'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_HEIGHT = 256\nIMG_WIDTH = 256\nN_CHANNELS = 3\nepochs = 20\nBATCH_SIZE = 8 * REPLICAS\nIMAGE_SIZE = [IMG_HEIGHT, IMG_WIDTH]\nIMAGE_RESIZE = [IMG_HEIGHT, IMG_WIDTH]\ninput_shape = (IMG_HEIGHT, IMG_WIDTH, N_CHANNELS)\nBALANCE_DATA = False # Will calculate class weigths from sklearn before training","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Getting images ready for the model\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == 'TPU':\n    files_dir = KaggleDatasets().get_gcs_path('melanoma-256x256') # TPUs are fed from google cloud storage\nelse:\n    files_dir = external_tfrec_base_dir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(files_dir + '/train*.tfrec'),\n    test_size=0.1, random_state=42\n)\nTEST_FILENAMES = tf.io.gfile.glob(files_dir + '/test*.tfrec')\nprint('Train TFRecord Files:', len(TRAINING_FILENAMES))\nprint('Validation TFRecord Files:', len(VALID_FILENAMES))\nprint('Test TFRecord Files:', len(TEST_FILENAMES))\n\nsubmission_example = pd.read_csv(base_dir + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following code is used for data augmentation, it applies a transformation matrix to the original images. The matrix encodes zoom, shift and rotations.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 90. * tf.random.normal([1],dtype='float32')\n    shr = 2. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    h_shift = 8. * tf.random.normal([1],dtype='float32') \n    w_shift = 8. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bellow is the input pipeline. It reads the TFRecords and applies the augmentation transformations.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_data = True\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # normalize data to be between 0-1. Increases gradient descent speed      \n    image = tf.reshape(image, [IMAGE_SIZE[0],IMAGE_SIZE[1], 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef augmentation_pipeline(image, label):\n    \"\"\"\n    add augmentation functions here\n    \"\"\"\n    if aug_data:\n        image,_ = transform(image,label)\n        image = tf.image.random_flip_left_right(image)\n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE \nprint(\"Num of steps per epoch:\", STEPS_PER_EPOCH)\nprint(\"Num of steps per validation\", VALID_STEPS)\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nds = train_dataset.unbatch().take(12*5)  \nds_val = valid_dataset.unbatch().take(12*5)  \nshow_dataset(64, 12, 5, ds)\nshow_dataset(64, 12, 5, ds_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_roc(y_true, y_score):\n    \"\"\"\n    \"\"\"\n    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_true.ravel(), y_score.ravel())\n    roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n    \n    plt.figure()\n    lw = 2\n    plt.plot(fpr[2], tpr[2], color='darkorange',\n             lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \ndef plot_train_hist(hist):\n    # Plot training & validation accuracy values\n    plt.plot(hist.history['auc'])\n    plt.plot(hist.history['val_auc'])\n    plt.title('Model accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()\n\n    # Plot training & validation loss values\n    plt.plot(hist.history['loss'])\n    plt.plot(hist.history['val_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## BiT training transfer learning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BiTModelTransfer(tf.keras.Model):\n\n  def __init__(self, num_classes, module):\n    super().__init__()\n\n    self.num_classes = num_classes\n    self.head = tf.keras.layers.Dense(num_classes, kernel_initializer='zeros', activation='sigmoid')\n    self.bit_model = module\n\n  def call(self, images):\n    bit_embedding = self.bit_model(images)\n    return self.head(bit_embedding)\n\nmodel = BiTModelTransfer(num_classes=1, module=module)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(base_dir + 'train.csv')\ny_train = train_df['target']\n\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 classes=np.unique(y_train),\n                                                 y=y_train)\n\n\nclass_weights = {0: class_weights[0],1: class_weights[1]}\nif not BALANCE_DATA:\n    class_weights = {0: 1,1: 2}\nprint(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define optimiser and loss\nlr = 0.0003\nSCHEDULE_BOUNDARIES = [200, 300, 400, 500]\nlr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay(boundaries=SCHEDULE_BOUNDARIES,\n                                                                  values=[lr, lr*0.1, lr*0.001, lr*0.0001, lr*0.00001])\noptimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9)\n\nloss_fn = tf.keras.losses.BinaryCrossentropy(from_logits=True, label_smoothing=0.05)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=optimizer,\n             loss=loss_fn,\n             metrics=['accuracy', 'AUC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_train = model.fit(train_dataset,\n                 validation_data=valid_dataset,\n                 steps_per_epoch=STEPS_PER_EPOCH,\n                 validation_steps=VALID_STEPS,\n                 epochs=epochs, class_weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_train_hist(hist_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Create predictions csv¶","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model.predict(test_images_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del submission_example['target']\nsubmission_example = submission_example.merge(pred_df, on='image_name')\n#sub.to_csv('submission_label_smoothing.csv', index=False)\nsubmission_example.to_csv('submission_b5.csv', index=False)\nsubmission_example.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}