{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Tensorflow HuBMAP - Hacking the Kidney competition starter kit:\n\n\n\n# Versions\n\nV1: Base kernal copied From [this](https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-train) kernel.\n","metadata":{"papermill":{"duration":0.019852,"end_time":"2021-04-01T19:07:42.806299","exception":false,"start_time":"2021-04-01T19:07:42.786447","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Refferences:\n* @marcosnovaes  https://www.kaggle.com/marcosnovaes/hubmap-looking-at-tfrecords and https://www.kaggle.com/marcosnovaes/hubmap-unet-keras-model-fit-with-tpu\n* @mgornergoogle https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n* @qubvel https://github.com/qubvel/segmentation_models  !! 25 available backbones for each of 4 architectures\n* @kool777, @joshi98kishan https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n* @cdeotte https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n","metadata":{"papermill":{"duration":0.018458,"end_time":"2021-04-01T19:07:42.842832","exception":false,"start_time":"2021-04-01T19:07:42.824374","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Init - parameters, packages, gcs_paths, tpu","metadata":{"papermill":{"duration":0.017497,"end_time":"2021-04-01T19:07:42.87812","exception":false,"start_time":"2021-04-01T19:07:42.860623","status":"completed"},"tags":[]}},{"cell_type":"code","source":"P = {}\nP['EPOCHS'] = 60\nP['BACKBONE'] = 'efficientnetb0' \nP['NFOLDS'] = 5\nP['SEED'] = 2021\nP['VERBOSE'] = 1\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 24 # BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nP['TILING'] = [1024, 512] # 1024,512 1024,256 1024,128 1536,512 768,384\nP['DIM'] = P['TILING'][1] \nP['DIM_FROM'] = P['TILING'][0]\n\nP['LR'] = 3e-4\nP['OVERLAPP'] = False\nP['STEPS_COE'] = 1\nP['FOLDS'] = [0,1,2,3,4]\nP['smoothing'] = 1e-7\nP['WANDB'] = False\nP['SOFT_PROB'] = 0.7\n\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)","metadata":{"papermill":{"duration":0.054694,"end_time":"2021-04-01T19:07:42.950632","exception":false,"start_time":"2021-04-01T19:07:42.895938","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install segmentation_models -q\n# ! pip install -q tensorflow-io\n# import tensorflow_io as tfio\n%matplotlib inline\n\nimport os\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport math\nimport random\nimport segmentation_models as sm\nfrom segmentation_models.losses import bce_jaccard_loss\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\n\n# from tensorflow.keras import mixed_precision\n# mixed_precision.set_global_policy('mixed_float16')\n\n# tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":18.05982,"end_time":"2021-04-01T19:08:01.028368","exception":false,"start_time":"2021-04-01T19:07:42.968548","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_all(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_all(P['SEED'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","metadata":{"papermill":{"duration":5.713531,"end_time":"2021-04-01T19:08:06.761999","exception":false,"start_time":"2021-04-01T19:08:01.048468","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GCS_PATHS","metadata":{"papermill":{"duration":0.018959,"end_time":"2021-04-01T19:08:06.800006","exception":false,"start_time":"2021-04-01T19:08:06.781047","status":"completed"},"tags":[]}},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('hubmap-data-1024-512-tfrecord-soft-mask')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\n\nALL_TRAINING_FILENAMES","metadata":{"papermill":{"duration":0.493869,"end_time":"2021-04-01T19:08:07.312862","exception":false,"start_time":"2021-04-01T19:08:06.818993","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint('NUM_TRAINING_IMAGES:' )\nif P['OVERLAPP']:\n    print(count_data_items(ALL_TRAINING_FILENAMES2)+count_data_items(ALL_TRAINING_FILENAMES))\nelse:\n    print(count_data_items(ALL_TRAINING_FILENAMES))","metadata":{"papermill":{"duration":0.033264,"end_time":"2021-04-01T19:08:07.910897","exception":false,"start_time":"2021-04-01T19:08:07.877633","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets pipeline","metadata":{"papermill":{"duration":0.019558,"end_time":"2021-04-01T19:08:07.950242","exception":false,"start_time":"2021-04-01T19:08:07.930684","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu?scriptVersionId=54288693&cellId=16\ndef make_mask(num_holes,side_length,rows, cols, num_channels):\n    \n    \"\"\"Builds the mask for all sprinkles.\"\"\"\n    \n    row_range = tf.tile(tf.range(rows)[..., tf.newaxis], [1, num_holes])\n    col_range = tf.tile(tf.range(cols)[..., tf.newaxis], [1, num_holes])\n    r_idx = tf.random.uniform([num_holes], minval=0, maxval=rows-1,\n                              dtype=tf.int32)\n    c_idx = tf.random.uniform([num_holes], minval=0, maxval=cols-1,\n                              dtype=tf.int32)\n    r1 = tf.clip_by_value(r_idx - side_length // 2, 0, rows)\n    r2 = tf.clip_by_value(r_idx + side_length // 2, 0, rows)\n    c1 = tf.clip_by_value(c_idx - side_length // 2, 0, cols)\n    c2 = tf.clip_by_value(c_idx + side_length // 2, 0, cols)\n    row_mask = (row_range > r1) & (row_range < r2)\n    col_mask = (col_range > c1) & (col_range < c2)\n\n    # Combine masks into one layer and duplicate over channels.\n    mask = row_mask[:, tf.newaxis] & col_mask\n    mask = tf.reduce_any(mask, axis=-1)\n    mask = mask[..., tf.newaxis]\n    mask = tf.tile(mask, [1, 1, num_channels])\n    return mask\n    \ndef sprinkles(image): \n    num_holes = 20\n    side_length = 15\n    mode = 'normal'\n    PROBABILITY = 1\n    \n    RandProb = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n    if (RandProb == 0)|(num_holes == 0): return image\n    \n    img_shape = tf.shape(image)\n    if mode is 'normal':\n        rejected = tf.zeros_like(image)\n    elif mode is 'salt_pepper':\n        num_holes = num_holes // 2\n        rejected_high = tf.ones_like(image)\n        rejected_low = tf.zeros_like(image)\n    elif mode is 'gaussian':\n        rejected = tf.random.normal(img_shape, dtype=tf.float32)\n    else:\n        raise ValueError(f'Unknown mode \"{mode}\" given.')\n        \n    rows = img_shape[0]\n    cols = img_shape[1]\n    num_channels = img_shape[-1]\n    if mode is 'salt_pepper':\n        mask1 = make_mask(num_holes,side_length,rows, cols, num_channels)\n        mask2 = make_mask(num_holes,side_length,rows, cols, num_channels)\n        filtered_image = tf.where(mask1, rejected_high, image)\n        filtered_image = tf.where(mask2, rejected_low, filtered_image)\n    else:\n        mask = make_mask(num_holes,side_length,rows, cols, num_channels)\n        filtered_image = tf.where(mask, rejected, image)\n    return filtered_image\n\ndef transform_shear(image, height, shear, mask=False):\n    \n    '''\n    shear augmentation on image\n    and mask.\n    --------------------------------\n    \n    Arguments:\n    image -- input image\n    mask -- input mask\n    \n    Return:\n    image -- augmented image \n    mask -- augmented mask\n    '''\n    \n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    if mask:\n        return tf.reshape(d, [DIM,DIM,1])\n    \n    return tf.reshape(d, [DIM,DIM,3])\n\ndef transform_shift(image, height, h_shift, w_shift, mask=False):\n    \n    '''\n    shift augmentation on image\n    and mask.\n    --------------------------------\n    \n    Arguments:\n    image -- input image\n    mask -- input mask\n    \n    Return:\n    image -- augmented image \n    mask -- augmented mask\n    '''\n    \n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = h_shift * tf.random.uniform([1],dtype='float32') \n    width_shift = w_shift * tf.random.uniform([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    if mask:\n        return tf.reshape(d, [DIM,DIM,1])\n    \n    return tf.reshape(d, [DIM,DIM,3])\n\ndef augmentations(image, mask):\n    \n    '''\n    Apply different augmentations on \n    image and mask.\n    --------------------------------\n    \n    Arguments:\n    image -- input image\n    mask -- input mask\n    \n    Return:\n    image -- augmented image \n    mask -- augmented mask\n    '''\n    \n    spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    shift = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    pixel = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    drop_coarse = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # SPATIAL-LEVEL TRANSFORMATIONS\n    ## FLIP LEFT-RIGHT\n    if spatial >= .4:\n        image = tf.image.flip_left_right(image)\n        mask = tf.image.flip_left_right(mask)\n    \n    ## FLIP UP-DOWN\n    if spatial >= .5:   \n        image = tf.image.flip_up_down(image)\n        mask = tf.image.flip_up_down(mask)\n        \n    ## ROTATIONS\n    if rotate > .8:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n        mask = tf.image.rot90(mask, k=3) # rotate 270º\n    elif rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n        mask = tf.image.rot90(mask, k=2) # rotate 180º\n    elif rotate > .4:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        mask = tf.image.rot90(mask, k=1) # rotate 90º\n    \n#     ## SHEAR \n#     if shear >= .5:\n#         image = transform_shear(image, height=P['DIM'], shear=20.)\n#         mask = transform_shear(mask, height=P['DIM'], shear=20., mask=True)\n    \n#     ## SHIFT\n#     if shift >= .5:\n#         image = transform_shift(image, height=P['DIM'], h_shift=15., w_shift=15.)\n#         mask = transform_shift(mask, height=P['DIM'], h_shift=15., w_shift=15., mask=True)\n\n    ## COARSE-DROPOUT\n    if drop_coarse >= .5:\n        image = sprinkles(image)\n        mask = sprinkles(mask)\n    \n    # PIXEL-LEVEL TRANSFORMATION\n    if pixel >= .2:\n        \n        if pixel >= .7:\n            image = tf.image.random_brightness(image, .2)\n        elif pixel >= .6:\n            image = tf.image.random_hue(image, .2)\n        elif pixel >= .5:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n        elif pixel >= .4:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n        \n    return image, mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIM = P['DIM']\ndef _parse_image_function(example_proto,augment = True, soft_label=True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'soft_mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    soft_mask =  tf.reshape(tf.io.decode_raw(single_example['soft_mask'],out_type=np.dtype('float32')),(DIM,DIM,1))\n    \n#     mini_size = 640\n#     image = tf.image.resize(image,(mini_size,mini_size))\n#     mask = tf.image.resize(tf.cast(mask,'uint8'),(mini_size,mini_size))\n\n#     image = tfio.experimental.color.rgb_to_bgr(image)\n\n    if soft_label:    \n        new_mask = P['SOFT_PROB'] * tf.cast(mask, tf.float32) + (1 - P['SOFT_PROB']) * tf.cast(soft_mask,tf.float32)\n    else:\n        new_mask = mask\n        \n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        image, new_mask = augmentations(image, new_mask)\n        \n    \n    return tf.cast(image, tf.float32)/255.0, tf.cast(new_mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = True, soft_label=True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment, soft_label = soft_label), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, augment = True, soft_label = True)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(1024, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=False)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered, augment = False, soft_label = True)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"papermill":{"duration":0.042071,"end_time":"2021-04-01T19:08:08.01285","exception":false,"start_time":"2021-04-01T19:08:07.970779","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"EDA","metadata":{"papermill":{"duration":0.019447,"end_time":"2021-04-01T19:08:08.052328","exception":false,"start_time":"2021-04-01T19:08:08.032881","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom skimage.segmentation import mark_boundaries\n\nTRAINING_FILENAMES = ALL_TRAINING_FILENAMES[0]\nprint(TRAINING_FILENAMES)\n# BATCH_SIZE = 64\ntrain_data = get_training_dataset()\n\n# print(len(train_data))\nfor i, (imgs, masks) in enumerate(train_data):\n    if i ==9:\n        break\n\n\nprint(imgs.shape)\n\nM = 20\nN=4\n\nplt.figure(figsize = (M,M))\ngs1 = gridspec.GridSpec(N*2,N*2)\n\nfor i in range(N*N):\n   # i = i + 1 # grid spec indexes from 0\n    ax1 = plt.subplot(gs1[i*2])\n    plt.axis('on')\n    ax1.set_xticklabels([])\n    ax1.set_yticklabels([])\n    ax1.set_aspect('equal')\n    img = imgs[i]\n    mask = masks[i]\n    ax1.imshow(img)\n    \n    ax2 = plt.subplot(gs1[i*2+1])\n    plt.axis('on')\n    ax2.set_xticklabels([])\n    ax2.set_yticklabels([])\n    ax2.set_aspect('equal')\n    \n    ax2.imshow(mask)\n\nplt.show()","metadata":{"papermill":{"duration":1.118687,"end_time":"2021-04-01T19:08:09.190836","exception":false,"start_time":"2021-04-01T19:08:08.072149","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom skimage.segmentation import mark_boundaries\n\nVALIDATION_FILENAMES = ALL_TRAINING_FILENAMES[0]\nprint(VALIDATION_FILENAMES)\n# BATCH_SIZE = 64\nvalid_data = get_validation_dataset()\n\n# print(len(train_data))\nfor i ,(imgs, masks) in enumerate(valid_data):\n    if i == 9:\n        break\n    \n\nprint(imgs.shape)\n\nM = 20\nN=4\n\nplt.figure(figsize = (M,M))\ngs1 = gridspec.GridSpec(N*2,N*2)\n\nfor i in range(N*N):\n   # i = i + 1 # grid spec indexes from 0\n    ax1 = plt.subplot(gs1[i*2])\n    plt.axis('on')\n    ax1.set_xticklabels([])\n    ax1.set_yticklabels([])\n    ax1.set_aspect('equal')\n    img = imgs[i]\n    mask = masks[i]\n    ax1.imshow(img)\n    \n    ax2 = plt.subplot(gs1[i*2+1])\n    plt.axis('on')\n    ax2.set_xticklabels([])\n    ax2.set_yticklabels([])\n    ax2.set_aspect('equal')\n    \n    ax2.imshow(mask)\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.019842,"end_time":"2021-04-01T19:08:09.231015","exception":false,"start_time":"2021-04-01T19:08:09.211173","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(target,output, axis = None, smooth=1e-10):\n    output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n    target = tf.dtypes.cast(target, tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = K.flatten(y_true)\n    y_pred_pos = K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})","metadata":{"papermill":{"duration":0.03665,"end_time":"2021-04-01T19:08:09.288331","exception":false,"start_time":"2021-04-01T19:08:09.251681","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://github.com/vgarshin/kaggle_kidney/blob/master/kidney_train.ipynb\nfrom tensorflow.keras.losses import binary_crossentropy\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2 * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred, smooth=1):\n    return (1 - dice_coef(y_true, y_pred, smooth))\n\ndef bce_dice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n\n# https://www.kaggle.com/elcaiseri/hubmap-pytorch-starter-vit-segmentation-train\nfrom tensorflow.keras.losses import Loss\nfrom segmentation_models.losses import DiceLoss\nclass DiceBCELoss(Loss):\n    # Formula Given above.\n    def __init__(self):\n        super(DiceBCELoss, self).__init__()\n\n    def call(self,y_true, y_pred):\n        \n        return bce_dice_loss(y_true, y_pred)\n#         return bce_jaccard_loss(y_true, y_pred)","metadata":{"papermill":{"duration":0.033176,"end_time":"2021-04-01T19:08:09.341938","exception":false,"start_time":"2021-04-01T19:08:09.308762","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model fit","metadata":{"papermill":{"duration":0.019867,"end_time":"2021-04-01T19:08:09.382547","exception":false,"start_time":"2021-04-01T19:08:09.36268","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if P['WANDB']:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    secret_value_0 = user_secrets.get_secret(\"WANDB_KEY\")\n\n    !pip install wandb==0.10.10\n    !wandb login $secret_value_0\n    \n    \n    import wandb\n    from wandb.keras import WandbCallback\n    wandb.init(config=P, project='hubmap-hacking-the-kidney')\n#     wandb.init(config=P, project='hubmap-hacking-the-kidney')\n    ","metadata":{"papermill":{"duration":16.27548,"end_time":"2021-04-01T19:08:25.678433","exception":false,"start_time":"2021-04-01T19:08:09.402953","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\n# for fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n#     print(*tr_idx, sep=', ')\n#     print(*val_idx, sep=', ')","metadata":{"papermill":{"duration":0.040016,"end_time":"2021-04-01T19:08:25.74834","exception":false,"start_time":"2021-04-01T19:08:25.708324","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trs_idx = np.array([[0, 2, 3, 5, 7, 8, 9, 10, 11, 12, 13, 14],\n                   [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14],\n                   [0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14],\n                   [0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],\n                   [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14]])\n\nvals_idx = np.array([[1, 6, 4],\n                    [7, 9, 11],\n                    [2, 3, 13],\n                    [5, 8, 14],\n                    [0, 10, 12]])\n\n\n\n# for fold,(tr_idx, val_idx) in zip(P['FOLDS'],zip(trs_idx[P['FOLDS']],vals_idx[P['FOLDS']])):\n#     print(fold)\n#     print(*tr_idx, sep=', ')\n#     print(*val_idx, sep=', ')","metadata":{"papermill":{"duration":0.049315,"end_time":"2021-04-01T19:08:25.828139","exception":false,"start_time":"2021-04-01T19:08:25.778824","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M = {}\nmetrics = ['loss','dice_coef','accuracy']\nfor fm in metrics:\n    M['val_'+fm] = []\n\nfold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\n# for fold,(tr_idx, val_idx) in zip(P['FOLDS'],zip(trs_idx[P['FOLDS']],vals_idx[P['FOLDS']])):\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)): \n    if fold not in P['FOLDS']:\n        continue\n        \n    print('#'*35); print('############ FOLD ',fold,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    print(f'Valid File Number: {val_idx}' )\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    \n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    \n    print(f'Valid File Name: {VALIDATION_FILENAMES}')\n    \n#     if P['OVERLAPP']:\n#         VALIDATION_FILENAMES += [ALL_TRAINING_FILENAMES2[fi] for fi in val_idx]\n        \n    \n    STEPS_PER_EPOCH = P['STEPS_COE'] * count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():   \n#         loss_obj = DiceBCELoss()\n#         input_shape=(P['DIM'],P['DIM'],3)\n        model = sm.Unet(P['BACKBONE'], encoder_weights='imagenet', classes=1, activation='sigmoid')\n        model.compile(optimizer = tf.keras.optimizers.Adam(lr = P['LR']),\n                      loss = DiceBCELoss(),\n                      metrics=[dice_coef,'accuracy'])\n        \n    # CALLBACKS\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(f\"/kaggle/working/{P['BACKBONE']}_Unet_model_fold_{fold}_sm_ex_sl.h5\",\n                                                    verbose=P['VERBOSE'],monitor='val_dice_coef',\n                                                    mode='max',save_best_only=True)\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coef',mode = 'max',\n                                                  patience=13, restore_best_weights=True)\n    \n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_dice_coef', mode = 'max', factor=0.2,\n                                                  patience=4, min_lr=1e-8,min_delta=0.0001,\n                                                 verbose=1)\n    \n    if P['WANDB']:\n        wandb.run.name= f\"{P['BACKBONE']}_fold_{fold}\"\n        wandb.watch_called = False\n        call_back = [WandbCallback(), checkpoint, reduce, early_stop]\n    else:\n        call_back = [checkpoint, reduce, early_stop]\n        \n    preprocess_input = sm.get_preprocessing(P['BACKBONE'])\n    \n    print(f'Training Model Fold {fold}...')\n    history = model.fit(\n        get_training_dataset(),\n        epochs = P['EPOCHS'],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        callbacks = call_back,\n        validation_data = get_validation_dataset(),\n        verbose=1\n    )   \n    \n    #with strategy.scope():\n    #    model = tf.keras.models.load_model('/kaggle/working/model-fold-%i.h5'%fold, custom_objects = {\"dice_coe\": dice_coe})\n    \n    # SAVE METRICS\n    m = model.evaluate(get_validation_dataset(),return_dict=True)\n    for fm in metrics:\n        M['val_'+fm].append(m[fm])\n    \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if P['DISPLAY_PLOT']:        \n        plt.figure(figsize=(15,5))\n        n_e = np.arange(len(history.history['dice_coef']))\n        plt.plot(n_e,history.history['dice_coef'],'-o',label='Train dice_coef',color='#ff7f0e')\n        plt.plot(n_e,history.history['val_dice_coef'],'-o',label='Val dice_coef',color='#1f77b4')\n        x = np.argmax( history.history['val_dice_coef'] ); y = np.max( history.history['val_dice_coef'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max dice_coef\\n%.2f'%y,size=14)\n        plt.ylabel('dice_coef',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(n_e,history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(n_e,history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()","metadata":{"papermill":{"duration":0.166137,"end_time":"2021-04-01T19:08:26.021564","exception":false,"start_time":"2021-04-01T19:08:25.855427","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### WRITE METRICS\nimport json\nfrom datetime import datetime\nM['datetime'] = str(datetime.now())\nfor fm in metrics:\n    M['oof_'+fm] = np.mean(M['val_'+fm])\n    print('OOF '+ fm + ' '+ str(M['oof_'+fm]))\nwith open('metrics.json', 'w') as outfile:\n    json.dump(M, outfile)","metadata":{"papermill":{"duration":0.054589,"end_time":"2021-04-01T19:08:26.104466","exception":false,"start_time":"2021-04-01T19:08:26.049877","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r wandb","metadata":{"papermill":{"duration":0.028296,"end_time":"2021-04-01T19:08:26.163015","exception":false,"start_time":"2021-04-01T19:08:26.134719","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"[1,2,12]\n[4,5,9]","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}