{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Tensorflow HuBMAP - Hacking the Kidney competition starter kit:\n\n* https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-tfrecs (how to create training and inference tfrecords)\n* this notebook (training pipeline)\n* https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm (inference with submission)\n\n\n# Versions\n* V1-V6 init\n* V7: 4-CV efficientunetb0 512x512 (LB .834)\n* V8: loss bce, fixed dice_coe function for tpu (LB .835)\n* V9: efficientunetb1, added oof metrics.json (CV .871, LB .830)\n* V10: efficientunetb4 (CV .874, LB .839) \n* V11: \n    * updated files paths (moved files to train folder)\n    * efficientunetb7 (memory issue on 3-rd fold)\n* V12: P['BATCH_COE'] = 4, efficientunetb7 (CV .858, LB .835)\n* V13: efficientunetb4, P['BATCH_COE'] = 8, P['SEED'] = 1 (just rerun nb V10 with best lb and consume tpu quota ;) ) (CV .877, LB .836)\n* V14: efficientunetb4, add overlapped tiles and P['EPOCHS'] = 30 and P['SEED'] = 0 (CV .8798, LB .843 (THRESHOLD = 0.5, MIN_OVERLAP = 32); LB .846 (THRESHOLD = .4, MIN_OVERLAP = 32); LB .848 (THRESHOLD = .4, MIN_OVERLAP = 300))\n* V15: efficientunetb4, fixed issue with image counting in training filenames, paths to train2 updated, added P['STEPS_COE'], P['TILING'], P['DIM_FROM'] (...)\n* V16: efficientunetb4, competition data update, P['STEPS_COE'] = 1, P['NFOLDS'] = 5\n\n","metadata":{}},{"cell_type":"markdown","source":"# Refferences:\n* @marcosnovaes  https://www.kaggle.com/marcosnovaes/hubmap-looking-at-tfrecords and https://www.kaggle.com/marcosnovaes/hubmap-unet-keras-model-fit-with-tpu\n* @mgornergoogle https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n* @qubvel https://github.com/qubvel/segmentation_models  !! 25 available backbones for each of 4 architectures\n* @kool777, @joshi98kishan https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n* @cdeotte https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n","metadata":{}},{"cell_type":"markdown","source":"# Init - parameters, packages, gcs_paths, tpu","metadata":{}},{"cell_type":"code","source":"P = {}\nP['EPOCHS'] = 30\nP['BACKBONE'] = 'efficientnetb0'\nP['NFOLDS'] = 2\nP['SEED'] = 0\nP['VERBOSE'] = 1\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 16# BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nP['TILING'] = [1024,256] # 1024,512 1024,256 1024,128 1536,512 768,384\nP['DIM'] = P['TILING'][1] \nP['DIM_FROM'] = P['TILING'][0]\n\nP['LR'] = 5e-4\nP['OVERLAPP'] = True\nP['STEPS_COE'] = 1\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install segmentation_models -q\n%matplotlib inline\n\nimport os\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport segmentation_models as sm\nfrom segmentation_models.losses import bce_jaccard_loss\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GCS_PATHS","metadata":{}},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(f'hubmap-tfrecords-1024-{P[\"DIM\"]}')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nALL_TRAINING_FILENAMES","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if P['OVERLAPP']:\n    ALL_TRAINING_FILENAMES2 = tf.io.gfile.glob(GCS_PATH + '/train2/*.tfrec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint('NUM_TRAINING_IMAGES:' )\nif P['OVERLAPP']:\n    print(count_data_items(ALL_TRAINING_FILENAMES2)+count_data_items(ALL_TRAINING_FILENAMES))\nelse:\n    print(count_data_items(ALL_TRAINING_FILENAMES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets pipeline","metadata":{}},{"cell_type":"code","source":"\nimport random\nDIM = P['DIM']\n\n\ndef get_coeff():\n    coeff=random.sample([0,0,0,100],1)\n    return coeff[0]\n\n\n\n\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n   \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    \n    channel_1=tf.reshape(image[:,:,0],(DIM,DIM,1))\n    channel_2=tf.reshape(image[:,:,1],(DIM,DIM,1))\n    channel_3=tf.reshape(image[:,:,2],(DIM,DIM,1))\n    image = tf.concat([channel_3,channel_2,channel_1],2)\n    image=tf.image.rgb_to_grayscale(image)\n     \n    image=tf.reshape(image,(DIM,DIM,1))\n    mask0=tf.cast(mask,tf.float32)\n    image=tf.cast(image,tf.float32)\n    \n    return tf.cast(tf.concat([image,image,image],2), tf.float32),tf.cast((mask0), tf.float32)\n\n \n\ndef _parse_image_function_bis(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    \n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    coeff=0.0\n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n        if tf.random.uniform(()) > 0.5:\n            coeff=100.0\n        if tf.random.uniform(()) <= 0.5:\n            coeff=-100.0\n  \n    channel_1=tf.reshape(image[:,:,0],(DIM,DIM,1))\n    channel_2=tf.reshape(image[:,:,1],(DIM,DIM,1))\n    channel_3=tf.reshape(image[:,:,2],(DIM,DIM,1))\n    image = tf.concat([channel_3,channel_2,channel_1],2)\n    image=tf.image.rgb_to_grayscale(image)\n     \n    image=tf.reshape(image,(DIM,DIM,1))\n    mask0=tf.cast(mask,tf.float32)\n    image=tf.cast(image,tf.float32)\n    coeff=0.0\n        \n    return tf.cast(tf.concat([image+coeff*(1-mask0),image+coeff*(1-mask0),image+coeff*(1-mask0)],2), tf.float32),tf.cast(mask, tf.float32)\n\n\ndef _parse_image_function_bisbis(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    \n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    coeff=0.0\n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n        if tf.random.uniform(()) > 0.5:\n            coeff=100.0\n        if tf.random.uniform(()) <= 0.5:\n            coeff=-100.0\n  \n    channel_1=tf.reshape(image[:,:,0],(DIM,DIM,1))\n    channel_2=tf.reshape(image[:,:,1],(DIM,DIM,1))\n    channel_3=tf.reshape(image[:,:,2],(DIM,DIM,1))\n    image = tf.concat([channel_3,channel_2,channel_1],2)\n    image=tf.image.rgb_to_grayscale(image)\n     \n    image=tf.reshape(image,(DIM,DIM,1))\n    mask0=tf.cast(mask,tf.float32)\n    image=tf.cast(image,tf.float32)\n    \n    \n    \n        \n    return tf.cast(tf.concat([image+coeff*(mask0),image+coeff*(mask0),image+coeff*(mask0)],2), tf.float32),tf.cast(mask, tf.float32)\n\n\n\ndef load_dataset(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\ndef load_dataset_bis(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function_bis(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\ndef load_dataset_bisbis(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function_bisbis(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset_bis():\n    dataset = load_dataset_bis(TRAINING_FILENAMES1)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_training_dataset(TRAINING):\n    dataset = load_dataset(TRAINING)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n  \ndef get_training_dataset_bisbis():\n    dataset = load_dataset_bisbis(TRAINING_FILENAMES1)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered, augment = False)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(output, target, axis = None, smooth=1e-10):\n    output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n    target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = K.flatten(y_true)\n    y_pred_pos = K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.losses import binary_crossentropy\ndef custom_loss():\n\n    # Create a loss function that adds the MSE loss to the mean of all squared activations of a specific layer\n    def loss(y_true,y_pred):\n        return  bce_jaccard_loss(y_true, y_pred)+ (1/400000)*tf.reduce_sum(tf.image.total_variation(y_pred))\n\n   \n    # Return a function\n    return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Input\nfrom keras.models import Model\nfrom keras.layers import Dense\nimport keras\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.layers.convolutional import Conv2D\nfrom keras import regularizers\n\nimage_shape=(DIM,DIM,3)\n\n\n    \n    \ndef final_model(model1,model2):\n    \n\n    inputs0 = Input(shape=image_shape)\n    \n    model1.trainable=False\n    model2.trainable=False\n    \n    #channel_1,channel_2,channel_3=tf.split(inputs0,3,axis=-1)\n    \n    x1 = model1(inputs0)\n    x2 = model2(inputs0)\n   \n    #x1=tf.where(  x1>0.5, x1,1)\n    #x1=tf.where(  x1<=0.5, x1,0)\n\n    x= tf.keras.layers.Concatenate()([x1,x2])\n    x= Conv2D(64, (3,3), activation='relu', padding='same')(x)\n    x= Conv2D(32, (3,3), activation='relu', padding='same')(x)\n    \n    outputs= Dense(1, activation='sigmoid')(x)   \n    \n    model_final = Model(inputs=inputs0, outputs=outputs)\n    return model_final\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model fit","metadata":{}},{"cell_type":"code","source":"from tensorflow.python.keras import backend as K\nM = {}\nmetrics = ['loss','dice_coe','accuracy']\nfor fm in metrics:\n    M['val_'+fm] = []\n\nfold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n    \n    print('#'*35); print('############ FOLD ',fold+1,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    if P['OVERLAPP']:\n        TRAINING_FILENAMES += [ALL_TRAINING_FILENAMES2[fi] for fi in tr_idx]\n    \n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    TRAINING_FILENAMES1,TRAINING_FILENAMES2=train_test_split(TRAINING_FILENAMES,test_size=0.5)\n    STEPS_PER_EPOCH = P['STEPS_COE'] * count_data_items(TRAINING_FILENAMES1) // BATCH_SIZE\n   \n    K.clear_session()\n\n    checkpoint1 = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model1-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True,save_weights_only=True)\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coe',mode = 'max', patience=10, restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.00001)\n        \n\n\n    with strategy.scope():   \n        model1 = sm.Unet('efficientnetb0',encoder_weights=\"imagenet\",activation=\"sigmoid\")\n        \n        model1.compile(tf.keras.optimizers.Adam(lr = P['LR']),\n                      loss = \"binary_crossentropy\",#'focal_tversky',\n                      metrics=[dice_coe,'accuracy'])\n       \n  \n    history = model1.fit(\n        get_training_dataset(TRAINING_FILENAMES1),validation_data = get_validation_dataset(),\n        epochs = 20,callbacks = [checkpoint1, reduce,early_stop],\n        steps_per_epoch = STEPS_PER_EPOCH,\n\n        verbose=P['VERBOSE']\n    )   \n\n    \n\n    STEPS_PER_EPOCH = P['STEPS_COE'] * count_data_items(TRAINING_FILENAMES2) // BATCH_SIZE\n   \n    checkpoint2 = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model2-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True,save_weights_only=True)\n    \n\n    with strategy.scope():   \n        model2 = sm.Unet('efficientnetb4',encoder_weights=\"imagenet\",activation=\"sigmoid\")\n        \n        model2.compile(tf.keras.optimizers.Adam(lr = P['LR']),\n                      loss = \"binary_crossentropy\",#'focal_tversky',\n                      metrics=[dice_coe,'accuracy'])\n       \n  \n    history = model2.fit(\n        get_training_dataset(TRAINING_FILENAMES2),validation_data = get_validation_dataset(),\n        epochs = 20,callbacks = [checkpoint2, reduce,early_stop],\n        steps_per_epoch = STEPS_PER_EPOCH,\n\n        verbose=P['VERBOSE']\n    )   \n\n    \n\n\n\n\n\n    STEPS_PER_EPOCH = P['STEPS_COE'] * count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    \n    \n    checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True,save_weights_only=True)\n    \n    \n    with strategy.scope():   \n        model1= sm.Unet('efficientnetb0',encoder_weights=\"imagenet\",activation=\"sigmoid\")\n        model1.load_weights('/kaggle/working/model1-fold-%i.h5'%fold)\n\n        model2= sm.Unet('efficientnetb4',encoder_weights=\"imagenet\",activation=\"sigmoid\")\n        model2.load_weights('/kaggle/working/model2-fold-%i.h5'%fold)\n    \n        model = final_model(model1,model2)\n        \n        model.compile(tf.keras.optimizers.Adam(lr =P['LR']),\n                      loss = bce_jaccard_loss,#'focal_tversky',\n                      metrics=[dice_coe,'accuracy'])\n    \n    history = model.fit(\n        get_training_dataset(TRAINING_FILENAMES),\n        epochs = 30,callbacks = [checkpoint, reduce,early_stop],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        validation_data = get_validation_dataset(),\n        verbose=P['VERBOSE']\n    )  \n    \n     \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}