{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nDataset    : 2019 and 2020\nMode       : BiT \nImage size : 384\n\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport re\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport math\nimport tensorflow_hub as hub\nfrom matplotlib import pyplot as plt\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow.keras.layers as L\n\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\n#import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"img_2019_dir = '../input/isic2019-384x384/train'\nimg_2019_csv_loc = '../input/isic2019-384x384/train.csv'\nimg_2020_dir = '.../input/melanoma-384x384'\nimg_2020_train_csv = '../input/siim-isic-melanoma-classification/train.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_2019_csv = pd.read_csv(img_2019_csv_loc)\nimg_2019_csv.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_2020_csv = pd.read_csv(img_2020_train_csv)\nimg_2020_csv.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'TPU'\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 545454\n\n# Configuration\nEPOCH = 11\nBATCH_SIZE = 4 * strategy.num_replicas_in_sync\n\nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768\nIMAGE_SIZE = 384\nIMG_SIZES = [IMAGE_SIZE,IMAGE_SIZE,IMAGE_SIZE,IMAGE_SIZE,IMAGE_SIZE]\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [1,1,1,1,1]\n#INC2018 = [1,1,1,1,1]\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [BATCH_SIZE]*FOLDS\nEPOCHS = [EPOCH]*FOLDS\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(BATCH_SIZES)\nprint(EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataSets GCS PATH","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS\n\nfor i,k in enumerate(IMG_SIZES):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n    GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\n    \nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(GCS_PATH)\nprint(GCS_PATH2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#KaggleDatasets().get_gcs_path('jpeg-isic2019-384x384')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fold Creation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 0\nDISPLAY_PLOT = True\n\nTRAINING_FILENAMES   = []\nVALIDATION_FILENAMES = []\n\n# Data access\nMAIN_TEST_GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\nTEST_FILENAMES = tf.io.gfile.glob(MAIN_TEST_GCS_PATH + '/tfrecords/test*.tfrec')\n\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\n\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n            \n    print('#'*25); print('#### FOLD',fold+1)\n    print('#### Image Size %i and batch_size %i'%(IMG_SIZES[fold],BATCH_SIZES[fold]*REPLICAS))\n    \n    # CREATE TRAIN SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n    \n    if INC2019[fold]:\n        files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n        print('#### Using 2019 external data')\n        \n    np.random.shuffle(files_train); print('#'*25)\n    \n    TRAINING_FILENAMES.append(files_train)\n    \n    # CREATE VALIDATION SUBSETS\n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n    VALIDATION_FILENAMES.append(files_valid)\n    \n    # CREATE TEST SUBSETS\n    #files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print((TRAINING_FILENAMES[5]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading TFrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.image.resize(image, (IMAGE_SIZE,IMAGE_SIZE))\n    return image\n\ndef read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    \n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    \n    return image, label\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n\n    # To add if using Bit\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n\n    return image, idnum if return_image_name else 0\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparation of images to input for training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order) \n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.transpose(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.7, 1.3)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    image = tf.image.random_brightness(image, 0.1)    \n    # used in Christ's notebook\n    #image = tf.image.random_saturation(image, 0, 2)\n    #imgage = tf.image.random_contrast(img, 0.8, 1.2)\n    #imgage = tf.image.random_brightness(img, 0.1)\n\n    return image, label\n\ndef get_training_dataset(fold_index):\n    dataset = load_dataset(TRAINING_FILENAMES[fold_index], labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(fold_index):\n    dataset = load_dataset(VALIDATION_FILENAMES[fold_index], labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES[1])\nNUM_VALID_IMAGES = count_data_items(VALIDATION_FILENAMES[1])\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images,{} vaid images, {} unlabeled test images'.\n       format(NUM_TRAINING_IMAGES,NUM_VALID_IMAGES, NUM_TEST_IMAGES))\nprint('Steps per epoch :{}'.format(STEPS_PER_EPOCH))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LR Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.00001, lr_max=0.0001, \n               lr_min=0.000001, lr_rampup_epochs=1, \n               lr_sustain_epochs=0, lr_exp_decay=.78):\n    lr_max = lr_max #* strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 0.003 * BATCH_SIZE / 512 \n\nlr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay(boundaries=[5,10,15], \n                                                                   values=[lr, lr*0.1, lr*0.001, lr*0.0001])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BiT Model Loading","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELPATH = KaggleDatasets().get_gcs_path('big-transfer-models-without-top')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r101x1_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r101x3_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r152x4_1/')\n# module = hub.KerasLayer(f'{MODELPATH}/bit_m-r50x1_1/')\n#module = hub.KerasLayer(f'{MODELPATH}/bit_m-r50x3_1/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'bit-fold1.h5', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    inputs = tf.keras.layers.Input(shape=(IMAGE_SIZE,IMAGE_SIZE,3))\n    \n    MODELPATH = KaggleDatasets().get_gcs_path('big-transfer-models-without-top')\n    module = hub.KerasLayer(f'{MODELPATH}/bit_m-r152x4_1/')\n    back_bone = module\n    back_bone.trainable = True\n    \n    logits = back_bone(inputs)\n    #logits = tf.keras.layers.Dense(256, activation='relu',kernel_regularizer=tf.keras.regularizers.l2()\n    #                               , dtype='float32')(logits)\n    \n    outputs = tf.keras.layers.Dense(1, activation='sigmoid', dtype='float32')(logits)\n    \n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    \n    #focal_loss = tfa.losses.sigmoid_focal_crossentropy(gamma = 2.0, alpha = 0.80)\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss = tfa.losses.SigmoidFocalCrossEntropy(gamma = 2.0, alpha = 0.80),\n        #tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.01),\n        metrics=[tf.keras.metrics.AUC()]\n        #['binary_crossentropy',tf.keras.metrics.AUC()]\n    )\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrfn = build_lrfn()\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nclass_weight = {0: 1, 1: 2}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rng = [i for i in range(EPOCH)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    get_training_dataset(0), \n    epochs=EPOCH, \n    callbacks=[lr_schedule],\n    steps_per_epoch=STEPS_PER_EPOCH,\n    class_weight=class_weight,\n    validation_data=get_validation_dataset(0)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#bce = history.history['binary_crossentropy']\n#val_bce = history.history['val_binary_crossentropy']\n\nauc = history.history['auc']\nval_auc = history.history['val_auc']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(bce))\n\nplt.plot(epochs, auc, 'b', label='Training AUC')\nplt.plot(epochs, val_auc, 'r', label='Validation AUC')\nplt.title('Training and validation AUC')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation Loss')\nplt.legend()\n\n#plt.figure()\n \n#plt.plot(epochs, bce, 'b', label='Training BCE')\n#plt.plot(epochs, val_bce, 'r', label='Validation BCE')\n#plt.title('Training and validation BCE')\n#plt.legend()\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/siim-model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test')\n#/kaggle/temp/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ['KAGGLE_USERNAME'] ='rajnishe'\nos.environ['KAGGLE_KEY'] = '35ac284ce25621f8c0caf11d4974879b'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('/tmp/siim-model/Bit-f1-focal-loss-152-epoch9.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -ltr /tmp/siim-model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = '''{\n  \"title\": \"siim-bit-model-v4\",\n  \"id\": \"rajnishe/siim-Bit-Model-v4\",\n  \"licenses\": [\n    {\n      \"name\": \"CC0-1.0\"\n    }\n  ]\n}\n'''\ntext_file = open(\"/tmp/siim-model/dataset-metadata.json\", 'w+')\nn = text_file.write(data)\ntext_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!kaggle datasets create -p /tmp/siim-model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}